diff --git a/build.xml b/build.xml index c1fec89..ed24705 100644 --- a/build.xml +++ b/build.xml @@ -65,7 +65,7 @@ - + @@ -82,7 +82,7 @@ - + @@ -116,7 +116,7 @@ - + @@ -129,7 +129,7 @@ - + @@ -138,6 +138,7 @@ File | Project Structure | Project | Project SDK. + @@ -148,7 +149,7 @@ - + diff --git a/dev-tools/eclipse/dot.classpath b/dev-tools/eclipse/dot.classpath index 46ab697..3f4f947 100644 --- a/dev-tools/eclipse/dot.classpath +++ b/dev-tools/eclipse/dot.classpath @@ -91,15 +91,15 @@ - - + + - - - + + + @@ -170,6 +170,6 @@ - + diff --git a/dev-tools/eclipse/dot.settings/org.eclipse.jdt.ui.prefs b/dev-tools/eclipse/dot.settings/org.eclipse.jdt.ui.prefs index b28a1d8..422052a 100644 --- a/dev-tools/eclipse/dot.settings/org.eclipse.jdt.ui.prefs +++ b/dev-tools/eclipse/dot.settings/org.eclipse.jdt.ui.prefs @@ -2,4 +2,4 @@ eclipse.preferences.version=1 formatter_profile=org.eclipse.jdt.ui.default.eclipse_profile formatter_settings_version=12 -org.eclipse.jdt.ui.text.custom_code_templates= +org.eclipse.jdt.ui.text.custom_code_templates= diff --git a/dev-tools/idea/.idea/libraries/Ant.xml b/dev-tools/idea/.idea/libraries/Ant.xml index b8fc15c..7f367ff 100644 --- a/dev-tools/idea/.idea/libraries/Ant.xml +++ b/dev-tools/idea/.idea/libraries/Ant.xml @@ -1,8 +1,8 @@ - - + + diff --git a/dev-tools/idea/.idea/libraries/JUnit.xml b/dev-tools/idea/.idea/libraries/JUnit.xml index 1494e44..9f57955 100644 --- a/dev-tools/idea/.idea/libraries/JUnit.xml +++ b/dev-tools/idea/.idea/libraries/JUnit.xml @@ -2,7 +2,7 @@ - + diff --git a/dev-tools/idea/.idea/projectCodeStyle.xml b/dev-tools/idea/.idea/projectCodeStyle.xml index e6ba7b6..1913663 100644 --- a/dev-tools/idea/.idea/projectCodeStyle.xml +++ b/dev-tools/idea/.idea/projectCodeStyle.xml @@ -16,6 +16,11 @@ + - + + - + @@ -669,25 +683,24 @@ - + + - - - + - + - + @@ -705,7 +718,7 @@ - + @@ -719,15 +732,48 @@ + + + + + + + + + + + + - + - + + + + + + US-ASCII + ISO-8859-1 + UTF-8 + + + + + + @@ -803,17 +851,29 @@ + + + + + + + + + + + + - + - + - - - - + - - + + @@ -876,29 +940,29 @@ - + 5 slowest tests: - + - + Showing ${max} slowest tests according to local stats. (change with -Dmax=...). - + - + Showing ${max} slowest tests in cached stats. (change with -Dmax=...). - + - + @@ -909,7 +973,7 @@ # - 'tests.class' is a class-filtering shell-like glob pattern, # 'testcase' is an alias of "tests.class=*.${testcase}" # - 'tests.method' is a method-filtering glob pattern. -# 'testmethod' is an alias of "tests.method=${testmethod}" +# 'testmethod' is an alias of "tests.method=${testmethod}*" # # Run a single test case (variants) @@ -930,9 +994,6 @@ ant test "-Dtests.method=*esi*" # Run with a given seed (seed is a hex-encoded long). ant test -Dtests.seed=DEADBEEF -# Repeats a given test N times (note filters). -ant test -Dtests.iters=N -Dtestcase=ClassName -Dtestmethod=mytest - # Repeats _all_ tests of ClassName N times. Every test repetition # will have a different seed. ant test -Dtests.iters=N -Dtestcase=ClassName @@ -942,6 +1003,18 @@ ant test -Dtests.iters=N -Dtestcase=ClassName # seed. ant test -Dtests.iters=N -Dtestcase=ClassName -Dtests.seed=dead:beef +# Repeats a given test N times (note the filters - individual test +# repetitions are given suffixes, ie: testFoo[0], testFoo[1], etc... +# so using testmethod or tests.method ending in a glob is necessary +# to ensure iterations are run). +ant test -Dtests.iters=N -Dtestcase=ClassName -Dtestmethod=mytest +ant test -Dtests.iters=N -Dtestcase=ClassName -Dtests.method=mytest* + +# Repeats N times but skips any tests after the first failure or M +# initial failures. +ant test -Dtests.iters=N -Dtests.failfast=yes -Dtestcase=... +ant test -Dtests.iters=N -Dtests.maxfailures=M -Dtestcase=... + # # Test groups. ---------------------------------------------------- # @@ -990,6 +1063,12 @@ ant -Dtests.showSuccess=true test # Display local averaged stats, if any (30 slowest tests). ant test-times -Dmax=30 +# Display a timestamp alongside each suite/ test. +ant -Dtests.timestamps=on ... + +# Override forked JVM file.encoding +ant -Dtests.file.encoding=XXX ... + # Output test files and reports. ${tests-output}/tests-report.txt - full ASCII tests report ${tests-output}/tests-failures.txt - failures only (if any) @@ -1000,15 +1079,13 @@ ${tests-output}/junit4-*.suites - per-JVM executed suites - + - - - - - - - + + + + + @@ -1610,7 +1687,7 @@ ${tests-output}/junit4-*.suites - per-JVM executed suites - + diff --git a/lucene/core/src/java/org/apache/lucene/LucenePackage.java b/lucene/core/src/java/org/apache/lucene/LucenePackage.java index 75361f4..225e054 100644 --- a/lucene/core/src/java/org/apache/lucene/LucenePackage.java +++ b/lucene/core/src/java/org/apache/lucene/LucenePackage.java @@ -1,6 +1,6 @@ package org.apache.lucene; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java b/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java index 33e5beb..9c63470 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java @@ -78,7 +78,7 @@ public abstract class Analyzer { public final TokenStream tokenStream(final String fieldName, final Reader reader) throws IOException { TokenStreamComponents components = reuseStrategy.getReusableComponents(fieldName); - final Reader r = initReader(reader); + final Reader r = initReader(fieldName, reader); if (components == null) { components = createComponents(fieldName, r); reuseStrategy.setReusableComponents(fieldName, components); @@ -91,7 +91,7 @@ public abstract class Analyzer { /** * Override this if you want to add a CharFilter chain. */ - protected Reader initReader(Reader reader) { + protected Reader initReader(String fieldName, Reader reader) { return reader; } diff --git a/lucene/core/src/java/org/apache/lucene/analysis/AnalyzerWrapper.java b/lucene/core/src/java/org/apache/lucene/analysis/AnalyzerWrapper.java index d9b766a..a7256e9 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/AnalyzerWrapper.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/AnalyzerWrapper.java @@ -86,4 +86,9 @@ public abstract class AnalyzerWrapper extends Analyzer { public final int getOffsetGap(IndexableField field) { return getWrappedAnalyzer(field.name()).getOffsetGap(field); } + + @Override + public final Reader initReader(String fieldName, Reader reader) { + return getWrappedAnalyzer(fieldName).initReader(fieldName, reader); + } } diff --git a/lucene/core/src/java/org/apache/lucene/analysis/CachingTokenFilter.java b/lucene/core/src/java/org/apache/lucene/analysis/CachingTokenFilter.java index 9885ace..a6c1e82 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/CachingTokenFilter.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/CachingTokenFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -61,14 +61,14 @@ public final class CachingTokenFilter extends TokenFilter { } @Override - public final void end() throws IOException { + public final void end() { if (finalState != null) { restoreState(finalState); } } @Override - public void reset() throws IOException { + public void reset() { if(cache != null) { iterator = cache.iterator(); } diff --git a/lucene/core/src/java/org/apache/lucene/analysis/CharReader.java b/lucene/core/src/java/org/apache/lucene/analysis/CharReader.java index 9cb31d5..f07d166 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/CharReader.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/CharReader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/analysis/CharStream.java b/lucene/core/src/java/org/apache/lucene/analysis/CharStream.java index 3a11253..22d710d 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/CharStream.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/CharStream.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/analysis/NumericTokenStream.java b/lucene/core/src/java/org/apache/lucene/analysis/NumericTokenStream.java index 069d5ce..d347027 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/NumericTokenStream.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/NumericTokenStream.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/analysis/Token.java b/lucene/core/src/java/org/apache/lucene/analysis/Token.java index 72fce99..7d33509 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/Token.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/Token.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -24,12 +24,12 @@ import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute; import org.apache.lucene.analysis.tokenattributes.TypeAttribute; -import org.apache.lucene.index.Payload; import org.apache.lucene.index.DocsAndPositionsEnum; // for javadoc import org.apache.lucene.util.Attribute; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.AttributeImpl; import org.apache.lucene.util.AttributeReflector; +import org.apache.lucene.util.BytesRef; /** A Token is an occurrence of a term from the text of a field. It consists of @@ -46,7 +46,7 @@ import org.apache.lucene.util.AttributeReflector; belongs to. For example an end of sentence marker token might be implemented with type "eos". The default token type is "word".

- A Token can optionally have metadata (a.k.a. Payload) in the form of a variable + A Token can optionally have metadata (a.k.a. payload) in the form of a variable length byte array. Use {@link DocsAndPositionsEnum#getPayload()} to retrieve the payloads from the index. @@ -118,7 +118,6 @@ import org.apache.lucene.util.AttributeReflector; {@link CharSequence} interface introduced by the interface {@link org.apache.lucene.analysis.tokenattributes.CharTermAttribute}. This method now only prints the term text, no additional information anymore.

- @see org.apache.lucene.index.Payload */ public class Token extends CharTermAttributeImpl implements TypeAttribute, PositionIncrementAttribute, @@ -127,7 +126,7 @@ public class Token extends CharTermAttributeImpl private int startOffset,endOffset; private String type = DEFAULT_TYPE; private int flags; - private Payload payload; + private BytesRef payload; private int positionIncrement = 1; private int positionLength = 1; @@ -140,6 +139,7 @@ public class Token extends CharTermAttributeImpl * @param start start offset in the source text * @param end end offset in the source text */ public Token(int start, int end) { + checkOffsets(start, end); startOffset = start; endOffset = end; } @@ -150,6 +150,7 @@ public class Token extends CharTermAttributeImpl * @param end end offset in the source text * @param typ the lexical type of this Token */ public Token(int start, int end, String typ) { + checkOffsets(start, end); startOffset = start; endOffset = end; type = typ; @@ -163,6 +164,7 @@ public class Token extends CharTermAttributeImpl * @param flags The bits to set for this token */ public Token(int start, int end, int flags) { + checkOffsets(start, end); startOffset = start; endOffset = end; this.flags = flags; @@ -178,6 +180,7 @@ public class Token extends CharTermAttributeImpl * @param end end offset */ public Token(String text, int start, int end) { + checkOffsets(start, end); append(text); startOffset = start; endOffset = end; @@ -193,6 +196,7 @@ public class Token extends CharTermAttributeImpl * @param typ token type */ public Token(String text, int start, int end, String typ) { + checkOffsets(start, end); append(text); startOffset = start; endOffset = end; @@ -210,6 +214,7 @@ public class Token extends CharTermAttributeImpl * @param flags token type bits */ public Token(String text, int start, int end, int flags) { + checkOffsets(start, end); append(text); startOffset = start; endOffset = end; @@ -227,6 +232,7 @@ public class Token extends CharTermAttributeImpl * @param end */ public Token(char[] startTermBuffer, int termBufferOffset, int termBufferLength, int start, int end) { + checkOffsets(start, end); copyBuffer(startTermBuffer, termBufferOffset, termBufferLength); startOffset = start; endOffset = end; @@ -296,12 +302,6 @@ public class Token extends CharTermAttributeImpl return startOffset; } - /** Set the starting offset. - @see #startOffset() */ - public void setStartOffset(int offset) { - this.startOffset = offset; - } - /** Returns this Token's ending offset, one greater than the position of the last character corresponding to this token in the source text. The length of the token in the source text is (endOffset - startOffset). */ @@ -309,15 +309,10 @@ public class Token extends CharTermAttributeImpl return endOffset; } - /** Set the ending offset. - @see #endOffset() */ - public void setEndOffset(int offset) { - this.endOffset = offset; - } - /** Set the starting and ending offset. @see #startOffset() and #endOffset()*/ public void setOffset(int startOffset, int endOffset) { + checkOffsets(startOffset, endOffset); this.startOffset = startOffset; this.endOffset = endOffset; } @@ -357,14 +352,14 @@ public class Token extends CharTermAttributeImpl /** * Returns this Token's payload. */ - public Payload getPayload() { + public BytesRef getPayload() { return this.payload; } /** * Sets this Token's payload. */ - public void setPayload(Payload payload) { + public void setPayload(BytesRef payload) { this.payload = payload; } @@ -450,11 +445,11 @@ public class Token extends CharTermAttributeImpl /** Shorthand for calling {@link #clear}, * {@link #copyBuffer(char[], int, int)}, - * {@link #setStartOffset}, - * {@link #setEndOffset}, + * {@link #setOffset}, * {@link #setType} * @return this Token instance */ public Token reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, String newType) { + checkOffsets(newStartOffset, newEndOffset); clearNoTermBuffer(); copyBuffer(newTermBuffer, newTermOffset, newTermLength); payload = null; @@ -467,11 +462,11 @@ public class Token extends CharTermAttributeImpl /** Shorthand for calling {@link #clear}, * {@link #copyBuffer(char[], int, int)}, - * {@link #setStartOffset}, - * {@link #setEndOffset} + * {@link #setOffset}, * {@link #setType} on Token.DEFAULT_TYPE * @return this Token instance */ public Token reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset) { + checkOffsets(newStartOffset, newEndOffset); clearNoTermBuffer(); copyBuffer(newTermBuffer, newTermOffset, newTermLength); startOffset = newStartOffset; @@ -482,11 +477,11 @@ public class Token extends CharTermAttributeImpl /** Shorthand for calling {@link #clear}, * {@link #append(CharSequence)}, - * {@link #setStartOffset}, - * {@link #setEndOffset} + * {@link #setOffset}, * {@link #setType} * @return this Token instance */ public Token reinit(String newTerm, int newStartOffset, int newEndOffset, String newType) { + checkOffsets(newStartOffset, newEndOffset); clear(); append(newTerm); startOffset = newStartOffset; @@ -497,11 +492,11 @@ public class Token extends CharTermAttributeImpl /** Shorthand for calling {@link #clear}, * {@link #append(CharSequence, int, int)}, - * {@link #setStartOffset}, - * {@link #setEndOffset} + * {@link #setOffset}, * {@link #setType} * @return this Token instance */ public Token reinit(String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, String newType) { + checkOffsets(newStartOffset, newEndOffset); clear(); append(newTerm, newTermOffset, newTermOffset + newTermLength); startOffset = newStartOffset; @@ -512,11 +507,11 @@ public class Token extends CharTermAttributeImpl /** Shorthand for calling {@link #clear}, * {@link #append(CharSequence)}, - * {@link #setStartOffset}, - * {@link #setEndOffset} + * {@link #setOffset}, * {@link #setType} on Token.DEFAULT_TYPE * @return this Token instance */ public Token reinit(String newTerm, int newStartOffset, int newEndOffset) { + checkOffsets(newStartOffset, newEndOffset); clear(); append(newTerm); startOffset = newStartOffset; @@ -527,11 +522,11 @@ public class Token extends CharTermAttributeImpl /** Shorthand for calling {@link #clear}, * {@link #append(CharSequence, int, int)}, - * {@link #setStartOffset}, - * {@link #setEndOffset} + * {@link #setOffset}, * {@link #setType} on Token.DEFAULT_TYPE * @return this Token instance */ public Token reinit(String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset) { + checkOffsets(newStartOffset, newEndOffset); clear(); append(newTerm, newTermOffset, newTermOffset + newTermLength); startOffset = newStartOffset; @@ -615,6 +610,13 @@ public class Token extends CharTermAttributeImpl reflector.reflect(FlagsAttribute.class, "flags", flags); reflector.reflect(TypeAttribute.class, "type", type); } + + private void checkOffsets(int startOffset, int endOffset) { + if (startOffset < 0 || endOffset < startOffset) { + throw new IllegalArgumentException("startOffset must be non-negative, and endOffset must be >= startOffset, " + + "startOffset=" + startOffset + ",endOffset=" + endOffset); + } + } /** Convenience factory that returns Token as implementation for the basic * attributes and return the default impl (with "Impl" appended) for all other diff --git a/lucene/core/src/java/org/apache/lucene/analysis/TokenFilter.java b/lucene/core/src/java/org/apache/lucene/analysis/TokenFilter.java index ed67df3..00ab0d1 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/TokenFilter.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/TokenFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java b/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java index 389229c..605b3cf 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java b/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java index 8d10aa8..162bd69 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.java index 7081ec8..2db0f58 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis.tokenattributes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java index a293f1d..afcf15a 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis.tokenattributes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttribute.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttribute.java index 14c317f..63116e3 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttribute.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttribute.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis.tokenattributes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttributeImpl.java index eae06b2..c1ad928 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttributeImpl.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttributeImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis.tokenattributes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/KeywordAttribute.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/KeywordAttribute.java index f7ba4e8..7e80d2a 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/KeywordAttribute.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/KeywordAttribute.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis.tokenattributes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/KeywordAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/KeywordAttributeImpl.java index 92581e5..7f2b2f4 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/KeywordAttributeImpl.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/KeywordAttributeImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis.tokenattributes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttribute.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttribute.java index 22d40f7..67612e8 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttribute.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttribute.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis.tokenattributes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttributeImpl.java index 0523d9a..c2ffbf8 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttributeImpl.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttributeImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis.tokenattributes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -47,10 +47,10 @@ public class OffsetAttributeImpl extends AttributeImpl implements OffsetAttribut // tokenizer should call clearAtts before re-using // OffsetAtt - // TODO: check that these are valid! IE, each should be - // >= 0, and endOffset should be >= startOffset. - // Problem is this could "break" existing - // tokenizers/filters. + if (startOffset < 0 || endOffset < startOffset) { + throw new IllegalArgumentException("startOffset must be non-negative, and endOffset must be >= startOffset, " + + "startOffset=" + startOffset + ",endOffset=" + endOffset); + } this.startOffset = startOffset; this.endOffset = endOffset; diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java index 006cc5e..658bc82 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis.tokenattributes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,20 +17,20 @@ package org.apache.lucene.analysis.tokenattributes; * limitations under the License. */ -import org.apache.lucene.index.Payload; import org.apache.lucene.util.Attribute; +import org.apache.lucene.util.BytesRef; /** - * The payload of a Token. See also {@link Payload}. + * The payload of a Token. */ public interface PayloadAttribute extends Attribute { /** * Returns this Token's payload. */ - public Payload getPayload(); + public BytesRef getPayload(); /** * Sets this Token's payload. */ - public void setPayload(Payload payload); + public void setPayload(BytesRef payload); } diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttributeImpl.java index 6d44924..6053e58 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttributeImpl.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttributeImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis.tokenattributes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,14 +17,14 @@ package org.apache.lucene.analysis.tokenattributes; * limitations under the License. */ -import org.apache.lucene.index.Payload; import org.apache.lucene.util.AttributeImpl; +import org.apache.lucene.util.BytesRef; /** - * The payload of a Token. See also {@link Payload}. + * The payload of a Token. */ public class PayloadAttributeImpl extends AttributeImpl implements PayloadAttribute, Cloneable { - private Payload payload; + private BytesRef payload; /** * Initialize this attribute with no payload. @@ -34,21 +34,21 @@ public class PayloadAttributeImpl extends AttributeImpl implements PayloadAttrib /** * Initialize this attribute with the given payload. */ - public PayloadAttributeImpl(Payload payload) { + public PayloadAttributeImpl(BytesRef payload) { this.payload = payload; } /** * Returns this Token's payload. */ - public Payload getPayload() { + public BytesRef getPayload() { return this.payload; } /** * Sets this Token's payload. */ - public void setPayload(Payload payload) { + public void setPayload(BytesRef payload) { this.payload = payload; } diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java index 20999aa..3229c84 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis.tokenattributes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java index 0fed366..2336d08 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis.tokenattributes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttribute.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttribute.java index d5b8466..8589cd1 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttribute.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttribute.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis.tokenattributes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttributeImpl.java index 6791834..995261d 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttributeImpl.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttributeImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis.tokenattributes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java index 2a0d9b0..61f82ad 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis.tokenattributes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TypeAttribute.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TypeAttribute.java index ac260c7..5013c60 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TypeAttribute.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TypeAttribute.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis.tokenattributes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TypeAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TypeAttributeImpl.java index 045b30d..098d78d 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TypeAttributeImpl.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TypeAttributeImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis.tokenattributes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BlockTermState.java b/lucene/core/src/java/org/apache/lucene/codecs/BlockTermState.java index 04440cd..15c77be 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/BlockTermState.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/BlockTermState.java @@ -1,5 +1,5 @@ package org.apache.lucene.codecs; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BlockTermsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/BlockTermsReader.java index d7af049..09b427c 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/BlockTermsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/BlockTermsReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,7 +18,6 @@ package org.apache.lucene.codecs; */ import java.io.IOException; -import java.util.Collection; import java.util.Comparator; import java.util.Iterator; import java.util.TreeMap; @@ -30,7 +29,6 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldsEnum; import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; @@ -41,7 +39,6 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.DoubleBarrelLRUCache; /** Handles a terms dict, but decouples all details of @@ -688,6 +685,9 @@ public class BlockTermsReader extends FieldsProducer { @Override public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException { //System.out.println("BTR.docs this=" + this); + if (needsFreqs && fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY) { + return null; + } decodeMetaData(); //System.out.println("BTR.docs: state.docFreq=" + state.docFreq); return postingsReader.docs(fieldInfo, state, liveDocs, reuse, needsFreqs); @@ -711,7 +711,7 @@ public class BlockTermsReader extends FieldsProducer { } @Override - public void seekExact(BytesRef target, TermState otherState) throws IOException { + public void seekExact(BytesRef target, TermState otherState) { //System.out.println("BTR.seekExact termState target=" + target.utf8ToString() + " " + target + " this=" + this); assert otherState != null && otherState instanceof BlockTermState; assert !doOrd || ((BlockTermState) otherState).ord < numTerms; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BlockTermsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/BlockTermsWriter.java index daa82c4..daf49d4 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/BlockTermsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/BlockTermsWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -31,7 +31,6 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RAMOutputStream; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.RamUsageEstimator; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java index de9a546..49dde51 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -41,7 +41,6 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.StringHelper; @@ -467,7 +466,7 @@ public class BlockTreeTermsReader extends FieldsProducer { } @Override - public int getDocCount() throws IOException { + public int getDocCount() { return docCount; } @@ -864,7 +863,7 @@ public class BlockTreeTermsReader extends FieldsProducer { } @Override - public BytesRef term() throws IOException { + public BytesRef term() { return term; } @@ -885,6 +884,9 @@ public class BlockTreeTermsReader extends FieldsProducer { @Override public DocsEnum docs(Bits skipDocs, DocsEnum reuse, boolean needsFreqs) throws IOException { currentFrame.decodeMetaData(); + if (needsFreqs && fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY) { + return null; + } return postingsReader.docs(fieldInfo, currentFrame.termState, skipDocs, reuse, needsFreqs); } @@ -1154,22 +1156,22 @@ public class BlockTreeTermsReader extends FieldsProducer { } @Override - public boolean seekExact(BytesRef text, boolean useCache) throws IOException { + public boolean seekExact(BytesRef text, boolean useCache) { throw new UnsupportedOperationException(); } @Override - public void seekExact(long ord) throws IOException { + public void seekExact(long ord) { throw new UnsupportedOperationException(); } @Override - public long ord() throws IOException { + public long ord() { throw new UnsupportedOperationException(); } @Override - public SeekStatus seekCeil(BytesRef text, boolean useCache) throws IOException { + public SeekStatus seekCeil(BytesRef text, boolean useCache) { throw new UnsupportedOperationException(); } } @@ -2109,6 +2111,9 @@ public class BlockTreeTermsReader extends FieldsProducer { @Override public DocsEnum docs(Bits skipDocs, DocsEnum reuse, boolean needsFreqs) throws IOException { + if (needsFreqs && fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY) { + return null; + } assert !eof; //if (DEBUG) { //System.out.println("BTTR.docs seg=" + segment); @@ -2139,7 +2144,7 @@ public class BlockTreeTermsReader extends FieldsProducer { } @Override - public void seekExact(BytesRef target, TermState otherState) throws IOException { + public void seekExact(BytesRef target, TermState otherState) { // if (DEBUG) { // System.out.println("BTTR.seekExact termState seg=" + segment + " target=" + target.utf8ToString() + " " + target + " state=" + otherState); // } @@ -2169,7 +2174,7 @@ public class BlockTreeTermsReader extends FieldsProducer { } @Override - public void seekExact(long ord) throws IOException { + public void seekExact(long ord) { throw new UnsupportedOperationException(); } @@ -2346,7 +2351,7 @@ public class BlockTreeTermsReader extends FieldsProducer { // } } - void rewind() throws IOException { + void rewind() { // Force reload: fp = fpOrig; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java index e7f3da0..80156f8 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -31,7 +31,6 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RAMOutputStream; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IntsRef; import org.apache.lucene.util.fst.Builder; @@ -639,7 +638,7 @@ public class BlockTreeTermsWriter extends FieldsConsumer { // for debugging @SuppressWarnings("unused") - private String toString(BytesRef b) { + private String toString(BytesRef b) { try { return b.utf8ToString() + " " + b; } catch (Throwable t) { @@ -896,6 +895,10 @@ public class BlockTreeTermsWriter extends FieldsConsumer { // System.out.println("SAVED to " + dotFileName); // w.close(); // } + } else { + assert sumTotalTermFreq == 0; + assert sumDocFreq == 0; + assert docCount == 0; } } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/Codec.java b/lucene/core/src/java/org/apache/lucene/codecs/Codec.java index 5b46c22..bc8715d 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/Codec.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/Codec.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java new file mode 100644 index 0000000..c9c562e --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java @@ -0,0 +1,153 @@ +package org.apache.lucene.codecs; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import java.io.IOException; + +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexFormatTooNewException; +import org.apache.lucene.index.IndexFormatTooOldException; +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.DataOutput; +import org.apache.lucene.util.BytesRef; + +/** + * Utility class for reading and writing versioned headers. + *

+ * Writing codec headers is useful to ensure that a file is in + * the format you think it is. + * + * @lucene.experimental + */ + +public final class CodecUtil { + private CodecUtil() {} // no instance + + /** + * Constant to identify the start of a codec header. + */ + public final static int CODEC_MAGIC = 0x3fd76c17; + + /** + * Writes a codec header, which records both a string to + * identify the file and a version number. This header can + * be parsed and validated with + * {@link #checkHeader(DataInput, String, int, int) checkHeader()}. + *

+ * CodecHeader --> Magic,CodecName,Version + *

    + *
  • Magic --> {@link DataOutput#writeInt Uint32}. This + * identifies the start of the header. It is always {@value #CODEC_MAGIC}. + *
  • CodecName --> {@link DataOutput#writeString String}. This + * is a string to identify this file. + *
  • Version --> {@link DataOutput#writeInt Uint32}. Records + * the version of the file. + *
+ *

+ * Note that the length of a codec header depends only upon the + * name of the codec, so this length can be computed at any time + * with {@link #headerLength(String)}. + * + * @param out Output stream + * @param codec String to identify this file. It should be simple ASCII, + * less than 128 characters in length. + * @param version Version number + * @throws IOException If there is an I/O error writing to the underlying medium. + */ + public static void writeHeader(DataOutput out, String codec, int version) + throws IOException { + BytesRef bytes = new BytesRef(codec); + if (bytes.length != codec.length() || bytes.length >= 128) { + throw new IllegalArgumentException("codec must be simple ASCII, less than 128 characters in length [got " + codec + "]"); + } + out.writeInt(CODEC_MAGIC); + out.writeString(codec); + out.writeInt(version); + } + + /** + * Computes the length of a codec header. + * + * @param codec Codec name. + * @return length of the entire codec header. + * @see #writeHeader(DataOutput, String, int) + */ + public static int headerLength(String codec) { + return 9+codec.length(); + } + + /** + * Reads and validates a header previously written with + * {@link #writeHeader(DataOutput, String, int)}. + *

+ * When reading a file, supply the expected codec and + * an expected version range (minVersion to maxVersion). + * + * @param in Input stream, positioned at the point where the + * header was previously written. Typically this is located + * at the beginning of the file. + * @param codec The expected codec name. + * @param minVersion The minimum supported expected version number. + * @param maxVersion The maximum supported expected version number. + * @return The actual version found, when a valid header is found + * that matches codec, with an actual version + * where minVersion <= actual <= maxVersion. + * Otherwise an exception is thrown. + * @throws CorruptIndexException If the first four bytes are not + * {@link #CODEC_MAGIC}, or if the actual codec found is + * not codec. + * @throws IndexFormatTooOldException If the actual version is less + * than minVersion. + * @throws IndexFormatTooNewException If the actual version is greater + * than maxVersion. + * @throws IOException If there is an I/O error reading from the underlying medium. + * @see #writeHeader(DataOutput, String, int) + */ + public static int checkHeader(DataInput in, String codec, int minVersion, int maxVersion) + throws IOException { + + // Safety to guard against reading a bogus string: + final int actualHeader = in.readInt(); + if (actualHeader != CODEC_MAGIC) { + throw new CorruptIndexException("codec header mismatch: actual header=" + actualHeader + " vs expected header=" + CODEC_MAGIC + " (resource: " + in + ")"); + } + return checkHeaderNoMagic(in, codec, minVersion, maxVersion); + } + + /** Like {@link + * #checkHeader(DataInput,String,int,int)} except this + * version assumes the first int has already been read + * and validated from the input. */ + public static int checkHeaderNoMagic(DataInput in, String codec, int minVersion, int maxVersion) throws IOException { + final String actualCodec = in.readString(); + if (!actualCodec.equals(codec)) { + throw new CorruptIndexException("codec mismatch: actual codec=" + actualCodec + " vs expected codec=" + codec + " (resource: " + in + ")"); + } + + final int actualVersion = in.readInt(); + if (actualVersion < minVersion) { + throw new IndexFormatTooOldException(in, actualVersion, minVersion, maxVersion); + } + if (actualVersion > maxVersion) { + throw new IndexFormatTooNewException(in, actualVersion, minVersion, maxVersion); + } + + return actualVersion; + } +} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesArraySource.java b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesArraySource.java index 413162d..e3dc8dc 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesArraySource.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesArraySource.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF @@ -106,6 +106,10 @@ public abstract class DocValuesArraySource extends Source { public byte[] getArray() { return values; } + + public double getFloat(int docID) { + return getInt(docID); + } @Override public long getInt(int docID) { @@ -167,6 +171,10 @@ public abstract class DocValuesArraySource extends Source { public short[] getArray() { return values; } + + public double getFloat(int docID) { + return getInt(docID); + } @Override public long getInt(int docID) { @@ -223,6 +231,10 @@ public abstract class DocValuesArraySource extends Source { public int[] getArray() { return values; } + + public double getFloat(int docID) { + return getInt(docID); + } @Override public long getInt(int docID) { diff --git a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java index d001053..4e1e536 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -29,6 +29,7 @@ import org.apache.lucene.document.PackedLongDocValuesField; import org.apache.lucene.document.ShortDocValuesField; import org.apache.lucene.document.SortedBytesDocValuesField; import org.apache.lucene.document.StraightBytesDocValuesField; +import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.DocValues.Source; import org.apache.lucene.index.DocValues.Type; import org.apache.lucene.index.DocValues; @@ -76,7 +77,20 @@ public abstract class DocValuesConsumer { * @throws IOException */ public abstract void finish(int docCount) throws IOException; - + + + /** + * Returns the value size this consumer accepts or -1 iff this + * consumer is value size agnostic ie. accepts variable length values. + *

+ * NOTE: the return value is undefined until the consumer has successfully + * consumed at least one value. + * + * @return the value size this consumer accepts or -1 iff this + * consumer is value size agnostic ie. accepts variable length values. + */ + public abstract int getValueSize(); + /** * Merges the given {@link org.apache.lucene.index.MergeState} into * this {@link DocValuesConsumer}. @@ -93,12 +107,12 @@ public abstract class DocValuesConsumer { assert mergeState != null; boolean hasMerged = false; for(int readerIDX=0;readerIDXn */ - public abstract void visitDocument(int n, StoredFieldVisitor visitor) throws CorruptIndexException, IOException; + public abstract void visitDocument(int n, StoredFieldVisitor visitor) throws IOException; public abstract StoredFieldsReader clone(); } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java index fa4967a..7e67b50 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java @@ -20,6 +20,7 @@ import java.io.Closeable; import java.io.IOException; import org.apache.lucene.document.Document; +import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.IndexableField; @@ -74,9 +75,9 @@ public abstract class StoredFieldsWriter implements Closeable { * merging (bulk-byte copying, etc). */ public int merge(MergeState mergeState) throws IOException { int docCount = 0; - for (MergeState.IndexReaderAndLiveDocs reader : mergeState.readers) { - final int maxDoc = reader.reader.maxDoc(); - final Bits liveDocs = reader.liveDocs; + for (AtomicReader reader : mergeState.readers) { + final int maxDoc = reader.maxDoc(); + final Bits liveDocs = reader.getLiveDocs(); for (int i = 0; i < maxDoc; i++) { if (liveDocs != null && !liveDocs.get(i)) { // skip deleted docs @@ -88,7 +89,7 @@ public abstract class StoredFieldsWriter implements Closeable { // on the fly? // NOTE: it's very important to first assign to doc then pass it to // fieldsWriter.addDocument; see LUCENE-1282 - Document doc = reader.reader.document(i); + Document doc = reader.document(i); addDocument(doc, mergeState.fieldInfos); docCount++; mergeState.checkAbort.work(300); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/TermStats.java b/lucene/core/src/java/org/apache/lucene/codecs/TermStats.java index 95341d1..51904a5 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/TermStats.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/TermStats.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsFormat.java index 3205779..19abb2d 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java index 742150f..22940b4 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java index ed5fdbf..1784d1f 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,6 +21,7 @@ import java.io.Closeable; import java.io.IOException; import java.util.Comparator; +import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -143,9 +144,9 @@ public abstract class TermVectorsWriter implements Closeable { * merging (bulk-byte copying, etc). */ public int merge(MergeState mergeState) throws IOException { int docCount = 0; - for (MergeState.IndexReaderAndLiveDocs reader : mergeState.readers) { - final int maxDoc = reader.reader.maxDoc(); - final Bits liveDocs = reader.liveDocs; + for (AtomicReader reader : mergeState.readers) { + final int maxDoc = reader.maxDoc(); + final Bits liveDocs = reader.getLiveDocs(); for (int docID = 0; docID < maxDoc; docID++) { if (liveDocs != null && !liveDocs.get(docID)) { // skip deleted docs @@ -153,7 +154,7 @@ public abstract class TermVectorsWriter implements Closeable { } // NOTE: it's very important to first assign to vectors then pass it to // termVectorsWriter.addAllDocVectors; see LUCENE-1282 - Fields vectors = reader.reader.getTermVectors(docID); + Fields vectors = reader.getTermVectors(docID); addAllDocVectors(vectors, mergeState.fieldInfos); docCount++; mergeState.checkAbort.work(300); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/TermsConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/TermsConsumer.java index a8f0388..30419c9 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/TermsConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/TermsConsumer.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/TermsIndexReaderBase.java b/lucene/core/src/java/org/apache/lucene/codecs/TermsIndexReaderBase.java index 7f79f0a..c7a2665 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/TermsIndexReaderBase.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/TermsIndexReaderBase.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/TermsIndexWriterBase.java b/lucene/core/src/java/org/apache/lucene/codecs/TermsIndexWriterBase.java index 4c2d2eb..477dfde 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/TermsIndexWriterBase.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/TermsIndexWriterBase.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/VariableGapTermsIndexReader.java b/lucene/core/src/java/org/apache/lucene/codecs/VariableGapTermsIndexReader.java index 3cf89ae..ceb76e1 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/VariableGapTermsIndexReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/VariableGapTermsIndexReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -30,7 +30,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.IntsRef; import org.apache.lucene.util.fst.Builder; import org.apache.lucene.util.fst.BytesRefFSTEnum; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/VariableGapTermsIndexWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/VariableGapTermsIndexWriter.java index 084dfb3..2156fbc 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/VariableGapTermsIndexWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/VariableGapTermsIndexWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -27,7 +27,6 @@ import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IntsRef; import org.apache.lucene.util.fst.Builder; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/appending/AppendingCodec.java b/lucene/core/src/java/org/apache/lucene/codecs/appending/AppendingCodec.java index 9984640..c811057 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/appending/AppendingCodec.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/appending/AppendingCodec.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.appending; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/appending/AppendingPostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/appending/AppendingPostingsFormat.java index 5a3686d..797c7ac 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/appending/AppendingPostingsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/appending/AppendingPostingsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.appending; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/appending/AppendingTermsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/appending/AppendingTermsReader.java index facd143..92198e9 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/appending/AppendingTermsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/appending/AppendingTermsReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.appending; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,12 +20,12 @@ package org.apache.lucene.codecs.appending; import java.io.IOException; import org.apache.lucene.codecs.BlockTreeTermsReader; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.PostingsReaderBase; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.CodecUtil; /** * Reads append-only terms from {@link AppendingTermsWriter} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/appending/AppendingTermsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/appending/AppendingTermsWriter.java index 8a47e02..8025a21 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/appending/AppendingTermsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/appending/AppendingTermsWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.appending; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,10 +20,10 @@ package org.apache.lucene.codecs.appending; import java.io.IOException; import org.apache.lucene.codecs.BlockTreeTermsWriter; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.PostingsWriterBase; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.CodecUtil; /** * Append-only version of {@link BlockTreeTermsWriter} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/intblock/FixedIntBlockIndexInput.java b/lucene/core/src/java/org/apache/lucene/codecs/intblock/FixedIntBlockIndexInput.java index cdc7f16..259ce6a 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/intblock/FixedIntBlockIndexInput.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/intblock/FixedIntBlockIndexInput.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.intblock; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -89,8 +89,7 @@ public abstract class FixedIntBlockIndexInput extends IntIndexInput { private final int blockSize; private final IntsRef bulkResult = new IntsRef(); - public Reader(final IndexInput in, final int[] pending, final BlockReader blockReader) - throws IOException { + public Reader(final IndexInput in, final int[] pending, final BlockReader blockReader) { this.in = in; this.pending = pending; this.blockSize = pending.length; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/intblock/FixedIntBlockIndexOutput.java b/lucene/core/src/java/org/apache/lucene/codecs/intblock/FixedIntBlockIndexOutput.java index a293d06..004e51c 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/intblock/FixedIntBlockIndexOutput.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/intblock/FixedIntBlockIndexOutput.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.intblock; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/intblock/VariableIntBlockIndexInput.java b/lucene/core/src/java/org/apache/lucene/codecs/intblock/VariableIntBlockIndexInput.java index e2619da..0804dd7 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/intblock/VariableIntBlockIndexInput.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/intblock/VariableIntBlockIndexInput.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.intblock; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -92,15 +92,14 @@ public abstract class VariableIntBlockIndexInput extends IntIndexInput { private final BlockReader blockReader; private final IntsRef bulkResult = new IntsRef(); - public Reader(final IndexInput in, final int[] pending, final BlockReader blockReader) - throws IOException { + public Reader(final IndexInput in, final int[] pending, final BlockReader blockReader) { this.in = in; this.pending = pending; bulkResult.ints = pending; this.blockReader = blockReader; } - void seek(final long fp, final int upto) throws IOException { + void seek(final long fp, final int upto) { // TODO: should we do this in real-time, not lazy? pendingFP = fp; pendingUpto = upto; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/intblock/VariableIntBlockIndexOutput.java b/lucene/core/src/java/org/apache/lucene/codecs/intblock/VariableIntBlockIndexOutput.java index 196e019..9d31c3c 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/intblock/VariableIntBlockIndexOutput.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/intblock/VariableIntBlockIndexOutput.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.intblock; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xCodec.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xCodec.java deleted file mode 100644 index 002f6e3..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xCodec.java +++ /dev/null @@ -1,151 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.HashSet; -import java.util.Set; - -import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.DocValuesFormat; -import org.apache.lucene.codecs.FieldInfosFormat; -import org.apache.lucene.codecs.LiveDocsFormat; -import org.apache.lucene.codecs.NormsFormat; -import org.apache.lucene.codecs.PerDocConsumer; -import org.apache.lucene.codecs.PerDocProducer; -import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.SegmentInfoFormat; -import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.TermVectorsFormat; -import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.PerDocWriteState; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentInfoPerCommit; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.util.MutableBits; - -/** - * Supports the Lucene 3.x index format (readonly) - * @deprecated - */ -@Deprecated -public class Lucene3xCodec extends Codec { - public Lucene3xCodec() { - super("Lucene3x"); - } - - private final PostingsFormat postingsFormat = new Lucene3xPostingsFormat(); - - private final StoredFieldsFormat fieldsFormat = new Lucene3xStoredFieldsFormat(); - - private final TermVectorsFormat vectorsFormat = new Lucene3xTermVectorsFormat(); - - private final FieldInfosFormat fieldInfosFormat = new Lucene3xFieldInfosFormat(); - - private final SegmentInfoFormat infosFormat = new Lucene3xSegmentInfoFormat(); - - private final Lucene3xNormsFormat normsFormat = new Lucene3xNormsFormat(); - - /** Extension of compound file for doc store files*/ - static final String COMPOUND_FILE_STORE_EXTENSION = "cfx"; - - // TODO: this should really be a different impl - private final LiveDocsFormat liveDocsFormat = new Lucene40LiveDocsFormat() { - @Override - public void writeLiveDocs(MutableBits bits, Directory dir, SegmentInfoPerCommit info, int newDelCount, IOContext context) throws IOException { - throw new UnsupportedOperationException("this codec can only be used for reading"); - } - }; - - // 3.x doesn't support docvalues - private final DocValuesFormat docValuesFormat = new DocValuesFormat() { - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return null; - } - - @Override - public PerDocProducer docsProducer(SegmentReadState state) throws IOException { - return null; - } - }; - - @Override - public PostingsFormat postingsFormat() { - return postingsFormat; - } - - @Override - public DocValuesFormat docValuesFormat() { - return docValuesFormat; - } - - @Override - public StoredFieldsFormat storedFieldsFormat() { - return fieldsFormat; - } - - @Override - public TermVectorsFormat termVectorsFormat() { - return vectorsFormat; - } - - @Override - public FieldInfosFormat fieldInfosFormat() { - return fieldInfosFormat; - } - - @Override - public SegmentInfoFormat segmentInfoFormat() { - return infosFormat; - } - - @Override - public NormsFormat normsFormat() { - return normsFormat; - } - - @Override - public LiveDocsFormat liveDocsFormat() { - return liveDocsFormat; - } - - /** Returns file names for shared doc stores, if any, else - * null. */ - public static Set getDocStoreFiles(SegmentInfo info) { - if (Lucene3xSegmentInfoFormat.getDocStoreOffset(info) != -1) { - final String dsName = Lucene3xSegmentInfoFormat.getDocStoreSegment(info); - Set files = new HashSet(); - if (Lucene3xSegmentInfoFormat.getDocStoreIsCompoundFile(info)) { - files.add(IndexFileNames.segmentFileName(dsName, "", COMPOUND_FILE_STORE_EXTENSION)); - } else { - files.add(IndexFileNames.segmentFileName(dsName, "", Lucene3xStoredFieldsReader.FIELDS_INDEX_EXTENSION)); - files.add(IndexFileNames.segmentFileName(dsName, "", Lucene3xStoredFieldsReader.FIELDS_EXTENSION)); - files.add(IndexFileNames.segmentFileName(dsName, "", Lucene3xTermVectorsReader.VECTORS_INDEX_EXTENSION)); - files.add(IndexFileNames.segmentFileName(dsName, "", Lucene3xTermVectorsReader.VECTORS_FIELDS_EXTENSION)); - files.add(IndexFileNames.segmentFileName(dsName, "", Lucene3xTermVectorsReader.VECTORS_DOCUMENTS_EXTENSION)); - } - return files; - } else { - return null; - } - } -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosFormat.java deleted file mode 100644 index 966718c..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosFormat.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.codecs.FieldInfosFormat; -import org.apache.lucene.codecs.FieldInfosReader; -import org.apache.lucene.codecs.FieldInfosWriter; - -/** - * Lucene3x ReadOnly FieldInfosFromat implementation - * @deprecated (4.0) This is only used to read indexes created - * before 4.0. - * @lucene.experimental - */ -@Deprecated -class Lucene3xFieldInfosFormat extends FieldInfosFormat { - private final FieldInfosReader reader = new Lucene3xFieldInfosReader(); - - @Override - public FieldInfosReader getFieldInfosReader() throws IOException { - return reader; - } - - @Override - public FieldInfosWriter getFieldInfosWriter() throws IOException { - throw new UnsupportedOperationException("this codec can only be used for reading"); - } -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosReader.java deleted file mode 100644 index ecd77d4..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosReader.java +++ /dev/null @@ -1,115 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Collections; - -import org.apache.lucene.codecs.FieldInfosReader; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.DocValues.Type; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.IndexFormatTooNewException; -import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; - -/** - * @lucene.experimental - * @deprecated - */ -@Deprecated -class Lucene3xFieldInfosReader extends FieldInfosReader { - /** Extension of field infos */ - static final String FIELD_INFOS_EXTENSION = "fnm"; - - // First used in 2.9; prior to 2.9 there was no format header - static final int FORMAT_START = -2; - // First used in 3.4: omit only positional information - static final int FORMAT_OMIT_POSITIONS = -3; - static final int FORMAT_MINIMUM = FORMAT_START; - static final int FORMAT_CURRENT = FORMAT_OMIT_POSITIONS; - static final byte IS_INDEXED = 0x1; - static final byte STORE_TERMVECTOR = 0x2; - static final byte OMIT_NORMS = 0x10; - static final byte STORE_PAYLOADS = 0x20; - static final byte OMIT_TERM_FREQ_AND_POSITIONS = 0x40; - static final byte OMIT_POSITIONS = -128; - - @Override - public FieldInfos read(Directory directory, String segmentName, IOContext iocontext) throws IOException { - final String fileName = IndexFileNames.segmentFileName(segmentName, "", FIELD_INFOS_EXTENSION); - IndexInput input = directory.openInput(fileName, iocontext); - - try { - final int format = input.readVInt(); - - if (format > FORMAT_MINIMUM) { - throw new IndexFormatTooOldException(input, format, FORMAT_MINIMUM, FORMAT_CURRENT); - } - if (format < FORMAT_CURRENT) { - throw new IndexFormatTooNewException(input, format, FORMAT_MINIMUM, FORMAT_CURRENT); - } - - final int size = input.readVInt(); //read in the size - FieldInfo infos[] = new FieldInfo[size]; - - for (int i = 0; i < size; i++) { - String name = input.readString(); - final int fieldNumber = i; - byte bits = input.readByte(); - boolean isIndexed = (bits & IS_INDEXED) != 0; - boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0; - boolean omitNorms = (bits & OMIT_NORMS) != 0; - boolean storePayloads = (bits & STORE_PAYLOADS) != 0; - final IndexOptions indexOptions; - if ((bits & OMIT_TERM_FREQ_AND_POSITIONS) != 0) { - indexOptions = IndexOptions.DOCS_ONLY; - } else if ((bits & OMIT_POSITIONS) != 0) { - if (format <= FORMAT_OMIT_POSITIONS) { - indexOptions = IndexOptions.DOCS_AND_FREQS; - } else { - throw new CorruptIndexException("Corrupt fieldinfos, OMIT_POSITIONS set but format=" + format + " (resource: " + input + ")"); - } - } else { - indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; - } - - // LUCENE-3027: past indices were able to write - // storePayloads=true when omitTFAP is also true, - // which is invalid. We correct that, here: - if (indexOptions != IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { - storePayloads = false; - } - infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, - omitNorms, storePayloads, indexOptions, null, isIndexed && !omitNorms? Type.FIXED_INTS_8 : null, Collections.emptyMap()); - } - - if (input.getFilePointer() != input.length()) { - throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")"); - } - return new FieldInfos(infos); - } finally { - input.close(); - } - } -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFields.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFields.java deleted file mode 100644 index 3c77e4a..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFields.java +++ /dev/null @@ -1,1106 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.TreeMap; - -import org.apache.lucene.codecs.FieldsProducer; -import org.apache.lucene.index.DocsAndPositionsEnum; -import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.FieldsEnum; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.UnicodeUtil; - -/** Exposes flex API on a pre-flex index, as a codec. - * @lucene.experimental - * @deprecated (4.0) - */ -@Deprecated -class Lucene3xFields extends FieldsProducer { - - private static final boolean DEBUG_SURROGATES = false; - - public TermInfosReader tis; - public final TermInfosReader tisNoIndex; - - public final IndexInput freqStream; - public final IndexInput proxStream; - final private FieldInfos fieldInfos; - private final SegmentInfo si; - final TreeMap fields = new TreeMap(); - final Map preTerms = new HashMap(); - private final Directory dir; - private final IOContext context; - private Directory cfsReader; - - public Lucene3xFields(Directory dir, FieldInfos fieldInfos, SegmentInfo info, IOContext context, int indexDivisor) - throws IOException { - - si = info; - - // NOTE: we must always load terms index, even for - // "sequential" scan during merging, because what is - // sequential to merger may not be to TermInfosReader - // since we do the surrogates dance: - if (indexDivisor < 0) { - indexDivisor = -indexDivisor; - } - - boolean success = false; - try { - TermInfosReader r = new TermInfosReader(dir, info.name, fieldInfos, context, indexDivisor); - if (indexDivisor == -1) { - tisNoIndex = r; - } else { - tisNoIndex = null; - tis = r; - } - this.context = context; - this.fieldInfos = fieldInfos; - - // make sure that all index files have been read or are kept open - // so that if an index update removes them we'll still have them - freqStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", Lucene3xPostingsFormat.FREQ_EXTENSION), context); - boolean anyProx = false; - for (FieldInfo fi : fieldInfos) { - if (fi.isIndexed()) { - fields.put(fi.name, fi); - preTerms.put(fi.name, new PreTerms(fi)); - if (fi.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { - anyProx = true; - } - } - } - - if (anyProx) { - proxStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", Lucene3xPostingsFormat.PROX_EXTENSION), context); - } else { - proxStream = null; - } - success = true; - } finally { - // With lock-less commits, it's entirely possible (and - // fine) to hit a FileNotFound exception above. In - // this case, we want to explicitly close any subset - // of things that were opened so that we don't have to - // wait for a GC to do so. - if (!success) { - close(); - } - } - this.dir = dir; - } - - // If this returns, we do the surrogates dance so that the - // terms are sorted by unicode sort order. This should be - // true when segments are used for "normal" searching; - // it's only false during testing, to create a pre-flex - // index, using the test-only PreFlexRW. - protected boolean sortTermsByUnicode() { - return true; - } - - @Override - public FieldsEnum iterator() throws IOException { - return new PreFlexFieldsEnum(); - } - - @Override - public Terms terms(String field) { - return preTerms.get(field); - } - - @Override - public int size() { - return preTerms.size(); - } - - @Override - public long getUniqueTermCount() throws IOException { - return getTermsDict().size(); - } - - synchronized private TermInfosReader getTermsDict() { - if (tis != null) { - return tis; - } else { - return tisNoIndex; - } - } - - @Override - public void close() throws IOException { - if (tis != null) { - tis.close(); - } - if (tisNoIndex != null) { - tisNoIndex.close(); - } - if (cfsReader != null) { - cfsReader.close(); - } - if (freqStream != null) { - freqStream.close(); - } - if (proxStream != null) { - proxStream.close(); - } - } - - private class PreFlexFieldsEnum extends FieldsEnum { - final Iterator it; - FieldInfo current; - - public PreFlexFieldsEnum() throws IOException { - it = fields.values().iterator(); - } - - @Override - public String next() { - if (it.hasNext()) { - current = it.next(); - return current.name; - } else { - return null; - } - } - - @Override - public Terms terms() throws IOException { - return Lucene3xFields.this.terms(current.name); - } - } - - private class PreTerms extends Terms { - final FieldInfo fieldInfo; - PreTerms(FieldInfo fieldInfo) { - this.fieldInfo = fieldInfo; - } - - @Override - public TermsEnum iterator(TermsEnum reuse) throws IOException { - PreTermsEnum termsEnum = new PreTermsEnum(); - termsEnum.reset(fieldInfo); - return termsEnum; - } - - @Override - public Comparator getComparator() { - // Pre-flex indexes always sorted in UTF16 order, but - // we remap on-the-fly to unicode order - if (sortTermsByUnicode()) { - return BytesRef.getUTF8SortedAsUnicodeComparator(); - } else { - return BytesRef.getUTF8SortedAsUTF16Comparator(); - } - } - - @Override - public long size() throws IOException { - return -1; - } - - @Override - public long getSumTotalTermFreq() { - return -1; - } - - @Override - public long getSumDocFreq() throws IOException { - return -1; - } - - @Override - public int getDocCount() throws IOException { - return -1; - } - } - - private class PreTermsEnum extends TermsEnum { - private SegmentTermEnum termEnum; - private FieldInfo fieldInfo; - private String internedFieldName; - private boolean skipNext; - private BytesRef current; - - private SegmentTermEnum seekTermEnum; - - private static final byte UTF8_NON_BMP_LEAD = (byte) 0xf0; - private static final byte UTF8_HIGH_BMP_LEAD = (byte) 0xee; - - // Returns true if the unicode char is "after" the - // surrogates in UTF16, ie >= U+E000 and <= U+FFFF: - private final boolean isHighBMPChar(byte[] b, int idx) { - return (b[idx] & UTF8_HIGH_BMP_LEAD) == UTF8_HIGH_BMP_LEAD; - } - - // Returns true if the unicode char in the UTF8 byte - // sequence starting at idx encodes a char outside of - // BMP (ie what would be a surrogate pair in UTF16): - private final boolean isNonBMPChar(byte[] b, int idx) { - return (b[idx] & UTF8_NON_BMP_LEAD) == UTF8_NON_BMP_LEAD; - } - - private final byte[] scratch = new byte[4]; - private final BytesRef prevTerm = new BytesRef(); - private final BytesRef scratchTerm = new BytesRef(); - private int newSuffixStart; - - // Swap in S, in place of E: - private boolean seekToNonBMP(SegmentTermEnum te, BytesRef term, int pos) throws IOException { - final int savLength = term.length; - - assert term.offset == 0; - - // The 3 bytes starting at downTo make up 1 - // unicode character: - assert isHighBMPChar(term.bytes, pos); - - // NOTE: we cannot make this assert, because - // AutomatonQuery legitimately sends us malformed UTF8 - // (eg the UTF8 bytes with just 0xee) - // assert term.length >= pos + 3: "term.length=" + term.length + " pos+3=" + (pos+3) + " byte=" + Integer.toHexString(term.bytes[pos]) + " term=" + term.toString(); - - // Save the bytes && length, since we need to - // restore this if seek "back" finds no matching - // terms - if (term.bytes.length < 4+pos) { - term.grow(4+pos); - } - - scratch[0] = term.bytes[pos]; - scratch[1] = term.bytes[pos+1]; - scratch[2] = term.bytes[pos+2]; - - term.bytes[pos] = (byte) 0xf0; - term.bytes[pos+1] = (byte) 0x90; - term.bytes[pos+2] = (byte) 0x80; - term.bytes[pos+3] = (byte) 0x80; - term.length = 4+pos; - - if (DEBUG_SURROGATES) { - System.out.println(" try seek term=" + UnicodeUtil.toHexString(term.utf8ToString())); - } - - // Seek "back": - getTermsDict().seekEnum(te, new Term(fieldInfo.name, term), true); - - // Test if the term we seek'd to in fact found a - // surrogate pair at the same position as the E: - Term t2 = te.term(); - - // Cannot be null (or move to next field) because at - // "worst" it'd seek to the same term we are on now, - // unless we are being called from seek - if (t2 == null || t2.field() != internedFieldName) { - return false; - } - - if (DEBUG_SURROGATES) { - System.out.println(" got term=" + UnicodeUtil.toHexString(t2.text())); - } - - // Now test if prefix is identical and we found - // a non-BMP char at the same position: - BytesRef b2 = t2.bytes(); - assert b2.offset == 0; - - boolean matches; - if (b2.length >= term.length && isNonBMPChar(b2.bytes, pos)) { - matches = true; - for(int i=0;i limit) { - - if (isHighBMPChar(prevTerm.bytes, downTo)) { - - if (DEBUG_SURROGATES) { - System.out.println(" found E pos=" + downTo + " vs len=" + prevTerm.length); - } - - if (seekToNonBMP(seekTermEnum, prevTerm, downTo)) { - // TODO: more efficient seek? - getTermsDict().seekEnum(termEnum, seekTermEnum.term(), true); - //newSuffixStart = downTo+4; - newSuffixStart = downTo; - scratchTerm.copyBytes(termEnum.term().bytes()); - didSeek = true; - if (DEBUG_SURROGATES) { - System.out.println(" seek!"); - } - break; - } else { - if (DEBUG_SURROGATES) { - System.out.println(" no seek"); - } - } - } - - // Shorten prevTerm in place so that we don't redo - // this loop if we come back here: - if ((prevTerm.bytes[downTo] & 0xc0) == 0xc0 || (prevTerm.bytes[downTo] & 0x80) == 0) { - prevTerm.length = downTo; - } - - downTo--; - } - - return didSeek; - } - - // Look for seek type 3 ("pop"): if the delta from - // prev -> current was replacing an S with an E, - // we must now seek to beyond that E. This seek - // "finishes" the dance at this character - // position. - private boolean doPop() throws IOException { - - if (DEBUG_SURROGATES) { - System.out.println(" try pop"); - } - - assert newSuffixStart <= prevTerm.length; - assert newSuffixStart < scratchTerm.length || newSuffixStart == 0; - - if (prevTerm.length > newSuffixStart && - isNonBMPChar(prevTerm.bytes, newSuffixStart) && - isHighBMPChar(scratchTerm.bytes, newSuffixStart)) { - - // Seek type 2 -- put 0xFF at this position: - scratchTerm.bytes[newSuffixStart] = (byte) 0xff; - scratchTerm.length = newSuffixStart+1; - - if (DEBUG_SURROGATES) { - System.out.println(" seek to term=" + UnicodeUtil.toHexString(scratchTerm.utf8ToString()) + " " + scratchTerm.toString()); - } - - // TODO: more efficient seek? can we simply swap - // the enums? - getTermsDict().seekEnum(termEnum, new Term(fieldInfo.name, scratchTerm), true); - - final Term t2 = termEnum.term(); - - // We could hit EOF or different field since this - // was a seek "forward": - if (t2 != null && t2.field() == internedFieldName) { - - if (DEBUG_SURROGATES) { - System.out.println(" got term=" + UnicodeUtil.toHexString(t2.text()) + " " + t2.bytes()); - } - - final BytesRef b2 = t2.bytes(); - assert b2.offset == 0; - - - // Set newSuffixStart -- we can't use - // termEnum's since the above seek may have - // done no scanning (eg, term was precisely - // and index term, or, was in the term seek - // cache): - scratchTerm.copyBytes(b2); - setNewSuffixStart(prevTerm, scratchTerm); - - return true; - } else if (newSuffixStart != 0 || scratchTerm.length != 0) { - if (DEBUG_SURROGATES) { - System.out.println(" got term=null (or next field)"); - } - newSuffixStart = 0; - scratchTerm.length = 0; - return true; - } - } - - return false; - } - - // Pre-flex indices store terms in UTF16 sort order, but - // certain queries require Unicode codepoint order; this - // method carefully seeks around surrogates to handle - // this impedance mismatch - - private void surrogateDance() throws IOException { - - if (!unicodeSortOrder) { - return; - } - - // We are invoked after TIS.next() (by UTF16 order) to - // possibly seek to a different "next" (by unicode - // order) term. - - // We scan only the "delta" from the last term to the - // current term, in UTF8 bytes. We look at 1) the bytes - // stripped from the prior term, and then 2) the bytes - // appended to that prior term's prefix. - - // We don't care about specific UTF8 sequences, just - // the "category" of the UTF16 character. Category S - // is a high/low surrogate pair (it non-BMP). - // Category E is any BMP char > UNI_SUR_LOW_END (and < - // U+FFFF). Category A is the rest (any unicode char - // <= UNI_SUR_HIGH_START). - - // The core issue is that pre-flex indices sort the - // characters as ASE, while flex must sort as AES. So - // when scanning, when we hit S, we must 1) seek - // forward to E and enum the terms there, then 2) seek - // back to S and enum all terms there, then 3) seek to - // after E. Three different seek points (1, 2, 3). - - // We can easily detect S in UTF8: if a byte has - // prefix 11110 (0xf0), then that byte and the - // following 3 bytes encode a single unicode codepoint - // in S. Similarly, we can detect E: if a byte has - // prefix 1110111 (0xee), then that byte and the - // following 2 bytes encode a single unicode codepoint - // in E. - - // Note that this is really a recursive process -- - // maybe the char at pos 2 needs to dance, but any - // point in its dance, suddenly pos 4 needs to dance - // so you must finish pos 4 before returning to pos - // 2. But then during pos 4's dance maybe pos 7 needs - // to dance, etc. However, despite being recursive, - // we don't need to hold any state because the state - // can always be derived by looking at prior term & - // current term. - - // TODO: can we avoid this copy? - if (termEnum.term() == null || termEnum.term().field() != internedFieldName) { - scratchTerm.length = 0; - } else { - scratchTerm.copyBytes(termEnum.term().bytes()); - } - - if (DEBUG_SURROGATES) { - System.out.println(" dance"); - System.out.println(" prev=" + UnicodeUtil.toHexString(prevTerm.utf8ToString())); - System.out.println(" " + prevTerm.toString()); - System.out.println(" term=" + UnicodeUtil.toHexString(scratchTerm.utf8ToString())); - System.out.println(" " + scratchTerm.toString()); - } - - // This code assumes TermInfosReader/SegmentTermEnum - // always use BytesRef.offset == 0 - assert prevTerm.offset == 0; - assert scratchTerm.offset == 0; - - // Need to loop here because we may need to do multiple - // pops, and possibly a continue in the end, ie: - // - // cont - // pop, cont - // pop, pop, cont - // - // - - while(true) { - if (doContinue()) { - break; - } else { - if (!doPop()) { - break; - } - } - } - - if (DEBUG_SURROGATES) { - System.out.println(" finish bmp ends"); - } - - doPushes(); - } - - - // Look for seek type 1 ("push"): if the newly added - // suffix contains any S, we must try to seek to the - // corresponding E. If we find a match, we go there; - // else we keep looking for additional S's in the new - // suffix. This "starts" the dance, at this character - // position: - private void doPushes() throws IOException { - - int upTo = newSuffixStart; - if (DEBUG_SURROGATES) { - System.out.println(" try push newSuffixStart=" + newSuffixStart + " scratchLen=" + scratchTerm.length); - } - - while(upTo < scratchTerm.length) { - if (isNonBMPChar(scratchTerm.bytes, upTo) && - (upTo > newSuffixStart || - (upTo >= prevTerm.length || - (!isNonBMPChar(prevTerm.bytes, upTo) && - !isHighBMPChar(prevTerm.bytes, upTo))))) { - - // A non-BMP char (4 bytes UTF8) starts here: - assert scratchTerm.length >= upTo + 4; - - final int savLength = scratchTerm.length; - scratch[0] = scratchTerm.bytes[upTo]; - scratch[1] = scratchTerm.bytes[upTo+1]; - scratch[2] = scratchTerm.bytes[upTo+2]; - - scratchTerm.bytes[upTo] = UTF8_HIGH_BMP_LEAD; - scratchTerm.bytes[upTo+1] = (byte) 0x80; - scratchTerm.bytes[upTo+2] = (byte) 0x80; - scratchTerm.length = upTo+3; - - if (DEBUG_SURROGATES) { - System.out.println(" try seek 1 pos=" + upTo + " term=" + UnicodeUtil.toHexString(scratchTerm.utf8ToString()) + " " + scratchTerm.toString() + " len=" + scratchTerm.length); - } - - // Seek "forward": - // TODO: more efficient seek? - getTermsDict().seekEnum(seekTermEnum, new Term(fieldInfo.name, scratchTerm), true); - - scratchTerm.bytes[upTo] = scratch[0]; - scratchTerm.bytes[upTo+1] = scratch[1]; - scratchTerm.bytes[upTo+2] = scratch[2]; - scratchTerm.length = savLength; - - // Did we find a match? - final Term t2 = seekTermEnum.term(); - - if (DEBUG_SURROGATES) { - if (t2 == null) { - System.out.println(" hit term=null"); - } else { - System.out.println(" hit term=" + UnicodeUtil.toHexString(t2.text()) + " " + (t2==null? null:t2.bytes())); - } - } - - // Since this was a seek "forward", we could hit - // EOF or a different field: - boolean matches; - - if (t2 != null && t2.field() == internedFieldName) { - final BytesRef b2 = t2.bytes(); - assert b2.offset == 0; - if (b2.length >= upTo+3 && isHighBMPChar(b2.bytes, upTo)) { - matches = true; - for(int i=0;i BMP - upTo += 3; - - // NOTE: we keep iterating, now, since this - // can easily "recurse". Ie, after seeking - // forward at a certain char position, we may - // find another surrogate in our [new] suffix - // and must then do another seek (recurse) - } else { - upTo++; - } - } else { - upTo++; - } - } - } - - private boolean unicodeSortOrder; - - void reset(FieldInfo fieldInfo) throws IOException { - //System.out.println("pff.reset te=" + termEnum); - this.fieldInfo = fieldInfo; - internedFieldName = fieldInfo.name.intern(); - final Term term = new Term(internedFieldName); - if (termEnum == null) { - termEnum = getTermsDict().terms(term); - seekTermEnum = getTermsDict().terms(term); - //System.out.println(" term=" + termEnum.term()); - } else { - getTermsDict().seekEnum(termEnum, term, true); - } - skipNext = true; - - unicodeSortOrder = sortTermsByUnicode(); - - final Term t = termEnum.term(); - if (t != null && t.field() == internedFieldName) { - newSuffixStart = 0; - prevTerm.length = 0; - surrogateDance(); - } - } - - @Override - public Comparator getComparator() { - // Pre-flex indexes always sorted in UTF16 order, but - // we remap on-the-fly to unicode order - if (unicodeSortOrder) { - return BytesRef.getUTF8SortedAsUnicodeComparator(); - } else { - return BytesRef.getUTF8SortedAsUTF16Comparator(); - } - } - - @Override - public void seekExact(long ord) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public long ord() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public SeekStatus seekCeil(BytesRef term, boolean useCache) throws IOException { - if (DEBUG_SURROGATES) { - System.out.println("TE.seek target=" + UnicodeUtil.toHexString(term.utf8ToString())); - } - skipNext = false; - final TermInfosReader tis = getTermsDict(); - final Term t0 = new Term(fieldInfo.name, term); - - assert termEnum != null; - - tis.seekEnum(termEnum, t0, useCache); - - final Term t = termEnum.term(); - - if (t != null && t.field() == internedFieldName && term.bytesEquals(t.bytes())) { - // If we found an exact match, no need to do the - // surrogate dance - if (DEBUG_SURROGATES) { - System.out.println(" seek exact match"); - } - current = t.bytes(); - return SeekStatus.FOUND; - } else if (t == null || t.field() != internedFieldName) { - - // TODO: maybe we can handle this like the next() - // into null? set term as prevTerm then dance? - - if (DEBUG_SURROGATES) { - System.out.println(" seek hit EOF"); - } - - // We hit EOF; try end-case surrogate dance: if we - // find an E, try swapping in S, backwards: - scratchTerm.copyBytes(term); - - assert scratchTerm.offset == 0; - - for(int i=scratchTerm.length-1;i>=0;i--) { - if (isHighBMPChar(scratchTerm.bytes, i)) { - if (DEBUG_SURROGATES) { - System.out.println(" found E pos=" + i + "; try seek"); - } - - if (seekToNonBMP(seekTermEnum, scratchTerm, i)) { - - scratchTerm.copyBytes(seekTermEnum.term().bytes()); - getTermsDict().seekEnum(termEnum, seekTermEnum.term(), useCache); - - newSuffixStart = 1+i; - - doPushes(); - - // Found a match - // TODO: faster seek? - current = termEnum.term().bytes(); - return SeekStatus.NOT_FOUND; - } - } - } - - if (DEBUG_SURROGATES) { - System.out.println(" seek END"); - } - - current = null; - return SeekStatus.END; - } else { - - // We found a non-exact but non-null term; this one - // is fun -- just treat it like next, by pretending - // requested term was prev: - prevTerm.copyBytes(term); - - if (DEBUG_SURROGATES) { - System.out.println(" seek hit non-exact term=" + UnicodeUtil.toHexString(t.text())); - } - - final BytesRef br = t.bytes(); - assert br.offset == 0; - - setNewSuffixStart(term, br); - - surrogateDance(); - - final Term t2 = termEnum.term(); - if (t2 == null || t2.field() != internedFieldName) { - // PreFlex codec interns field names; verify: - assert t2 == null || !t2.field().equals(internedFieldName); - current = null; - return SeekStatus.END; - } else { - current = t2.bytes(); - assert !unicodeSortOrder || term.compareTo(current) < 0 : "term=" + UnicodeUtil.toHexString(term.utf8ToString()) + " vs current=" + UnicodeUtil.toHexString(current.utf8ToString()); - return SeekStatus.NOT_FOUND; - } - } - } - - private void setNewSuffixStart(BytesRef br1, BytesRef br2) { - final int limit = Math.min(br1.length, br2.length); - int lastStart = 0; - for(int i=0;i norms = new HashMap(); - // any .nrm or .sNN files we have open at any time. - // TODO: just a list, and double-close() separate norms files? - final Set openFiles = Collections.newSetFromMap(new IdentityHashMap()); - // points to a singleNormFile - IndexInput singleNormStream; - final int maxdoc; - - // note: just like segmentreader in 3.x, we open up all the files here (including separate norms) up front. - // but we just don't do any seeks or reading yet. - public Lucene3xNormsProducer(Directory dir, SegmentInfo info, FieldInfos fields, IOContext context) throws IOException { - Directory separateNormsDir = info.dir; // separate norms are never inside CFS - maxdoc = info.getDocCount(); - String segmentName = info.name; - boolean success = false; - try { - long nextNormSeek = NORMS_HEADER.length; //skip header (header unused for now) - for (FieldInfo fi : fields) { - if (fi.hasNorms()) { - String fileName = getNormFilename(info, fi.number); - Directory d = hasSeparateNorms(info, fi.number) ? separateNormsDir : dir; - - // singleNormFile means multiple norms share this file - boolean singleNormFile = IndexFileNames.matchesExtension(fileName, NORMS_EXTENSION); - IndexInput normInput = null; - long normSeek; - - if (singleNormFile) { - normSeek = nextNormSeek; - if (singleNormStream == null) { - singleNormStream = d.openInput(fileName, context); - openFiles.add(singleNormStream); - } - // All norms in the .nrm file can share a single IndexInput since - // they are only used in a synchronized context. - // If this were to change in the future, a clone could be done here. - normInput = singleNormStream; - } else { - normInput = d.openInput(fileName, context); - openFiles.add(normInput); - // if the segment was created in 3.2 or after, we wrote the header for sure, - // and don't need to do the sketchy file size check. otherwise, we check - // if the size is exactly equal to maxDoc to detect a headerless file. - // NOTE: remove this check in Lucene 5.0! - String version = info.getVersion(); - final boolean isUnversioned = - (version == null || StringHelper.getVersionComparator().compare(version, "3.2") < 0) - && normInput.length() == maxdoc; - if (isUnversioned) { - normSeek = 0; - } else { - normSeek = NORMS_HEADER.length; - } - } - NormsDocValues norm = new NormsDocValues(normInput, normSeek); - norms.put(fi.name, norm); - nextNormSeek += maxdoc; // increment also if some norms are separate - } - } - // TODO: change to a real check? see LUCENE-3619 - assert singleNormStream == null || nextNormSeek == singleNormStream.length() : singleNormStream != null ? "len: " + singleNormStream.length() + " expected: " + nextNormSeek : "null"; - success = true; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(openFiles); - } - } - } - - @Override - public DocValues docValues(String field) throws IOException { - return norms.get(field); - } - - @Override - public void close() throws IOException { - try { - IOUtils.close(openFiles); - } finally { - norms.clear(); - openFiles.clear(); - } - } - - private static String getNormFilename(SegmentInfo info, int number) { - if (hasSeparateNorms(info, number)) { - long gen = Long.parseLong(info.getAttribute(Lucene3xSegmentInfoFormat.NORMGEN_PREFIX + number)); - return IndexFileNames.fileNameFromGeneration(info.name, SEPARATE_NORMS_EXTENSION + number, gen); - } else { - // single file for all norms - return IndexFileNames.segmentFileName(info.name, "", NORMS_EXTENSION); - } - } - - private static boolean hasSeparateNorms(SegmentInfo info, int number) { - String v = info.getAttribute(Lucene3xSegmentInfoFormat.NORMGEN_PREFIX + number); - if (v == null) { - return false; - } else { - assert Long.parseLong(v) != SegmentInfo.NO; - return true; - } - } - - static final class NormSource extends Source { - protected NormSource(byte[] bytes) { - super(Type.FIXED_INTS_8); - this.bytes = bytes; - } - - final byte bytes[]; - - @Override - public BytesRef getBytes(int docID, BytesRef ref) { - ref.bytes = bytes; - ref.offset = docID; - ref.length = 1; - return ref; - } - - @Override - public long getInt(int docID) { - return bytes[docID]; - } - - @Override - public boolean hasArray() { - return true; - } - - @Override - public Object getArray() { - return bytes; - } - - } - - private class NormsDocValues extends DocValues { - private final IndexInput file; - private final long offset; - public NormsDocValues(IndexInput normInput, long normSeek) { - this.file = normInput; - this.offset = normSeek; - } - - @Override - public Source load() throws IOException { - return new NormSource(bytes()); - } - - @Override - public Source getDirectSource() throws IOException { - return getSource(); - } - - @Override - public Type getType() { - return Type.FIXED_INTS_8; - } - - byte[] bytes() throws IOException { - byte[] bytes = new byte[maxdoc]; - // some norms share fds - synchronized(file) { - file.seek(offset); - file.readBytes(bytes, 0, bytes.length, false); - } - // we are done with this file - if (file != singleNormStream) { - openFiles.remove(file); - file.close(); - } - return bytes; - } - - @Override - public int getValueSize() { - return 1; - } - - } -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xPostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xPostingsFormat.java deleted file mode 100644 index 7c7adb4..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xPostingsFormat.java +++ /dev/null @@ -1,64 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.codecs.FieldsConsumer; -import org.apache.lucene.codecs.FieldsProducer; -import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.SegmentReadState; - -/** Codec that reads the pre-flex-indexing postings - * format. It does not provide a writer because newly - * written segments should use the Codec configured on IndexWriter. - * - * @deprecated (4.0) This is only used to read indexes created - * before 4.0. - * @lucene.experimental - */ -@Deprecated -class Lucene3xPostingsFormat extends PostingsFormat { - - /** Extension of terms file */ - public static final String TERMS_EXTENSION = "tis"; - - /** Extension of terms index file */ - public static final String TERMS_INDEX_EXTENSION = "tii"; - - /** Extension of freq postings file */ - public static final String FREQ_EXTENSION = "frq"; - - /** Extension of prox postings file */ - public static final String PROX_EXTENSION = "prx"; - - public Lucene3xPostingsFormat() { - super("Lucene3x"); - } - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - throw new UnsupportedOperationException("this codec can only be used for reading"); - } - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - return new Lucene3xFields(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.termsIndexDivisor); - } -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfoFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfoFormat.java deleted file mode 100644 index ef1f341..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfoFormat.java +++ /dev/null @@ -1,89 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.codecs.SegmentInfoFormat; -import org.apache.lucene.codecs.SegmentInfoReader; -import org.apache.lucene.codecs.SegmentInfoWriter; -import org.apache.lucene.index.SegmentInfo; - -/** - * Lucene3x ReadOnly SegmentInfoFormat implementation - * @deprecated (4.0) This is only used to read indexes created - * before 4.0. - * @lucene.experimental - */ -@Deprecated -public class Lucene3xSegmentInfoFormat extends SegmentInfoFormat { - private final SegmentInfoReader reader = new Lucene3xSegmentInfoReader(); - - /** This format adds optional per-segment String - * diagnostics storage, and switches userData to Map */ - public static final int FORMAT_DIAGNOSTICS = -9; - - /** Each segment records whether it has term vectors */ - public static final int FORMAT_HAS_VECTORS = -10; - - /** Each segment records the Lucene version that created it. */ - public static final int FORMAT_3_1 = -11; - - /** Extension used for saving each SegmentInfo, once a 3.x - * index is first committed to with 4.0. */ - public static final String UPGRADED_SI_EXTENSION = "si"; - public static final String UPGRADED_SI_CODEC_NAME = "Lucene3xSegmentInfo"; - public static final int UPGRADED_SI_VERSION_START = 0; - public static final int UPGRADED_SI_VERSION_CURRENT = UPGRADED_SI_VERSION_START; - - @Override - public SegmentInfoReader getSegmentInfosReader() { - return reader; - } - - @Override - public SegmentInfoWriter getSegmentInfosWriter() { - throw new UnsupportedOperationException("this codec can only be used for reading"); - } - - // only for backwards compat - public static final String DS_OFFSET_KEY = Lucene3xSegmentInfoFormat.class.getSimpleName() + ".dsoffset"; - public static final String DS_NAME_KEY = Lucene3xSegmentInfoFormat.class.getSimpleName() + ".dsname"; - public static final String DS_COMPOUND_KEY = Lucene3xSegmentInfoFormat.class.getSimpleName() + ".dscompound"; - public static final String NORMGEN_KEY = Lucene3xSegmentInfoFormat.class.getSimpleName() + ".normgen"; - public static final String NORMGEN_PREFIX = Lucene3xSegmentInfoFormat.class.getSimpleName() + ".normfield"; - - /** - * @return if this segment shares stored fields & vectors, this - * offset is where in that file this segment's docs begin - */ - public static int getDocStoreOffset(SegmentInfo si) { - String v = si.getAttribute(DS_OFFSET_KEY); - return v == null ? -1 : Integer.parseInt(v); - } - - /** @return name used to derive fields/vectors file we share with other segments */ - public static String getDocStoreSegment(SegmentInfo si) { - String v = si.getAttribute(DS_NAME_KEY); - return v == null ? si.name : v; - } - - /** @return whether doc store files are stored in compound file (*.cfx) */ - public static boolean getDocStoreIsCompoundFile(SegmentInfo si) { - String v = si.getAttribute(DS_COMPOUND_KEY); - return v == null ? false : Boolean.parseBoolean(v); - } -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfoReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfoReader.java deleted file mode 100644 index b79d76c..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfoReader.java +++ /dev/null @@ -1,272 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import org.apache.lucene.codecs.SegmentInfoReader; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.IndexFormatTooNewException; -import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentInfoPerCommit; -import org.apache.lucene.index.SegmentInfos; -import org.apache.lucene.store.CompoundFileDirectory; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.CodecUtil; -import org.apache.lucene.util.IOUtils; - -/** - * Lucene 3x implementation of {@link SegmentInfoReader}. - * @lucene.experimental - * @deprecated - */ -@Deprecated -public class Lucene3xSegmentInfoReader extends SegmentInfoReader { - - public static void readLegacyInfos(SegmentInfos infos, Directory directory, IndexInput input, int format) throws IOException { - infos.version = input.readLong(); // read version - infos.counter = input.readInt(); // read counter - Lucene3xSegmentInfoReader reader = new Lucene3xSegmentInfoReader(); - for (int i = input.readInt(); i > 0; i--) { // read segmentInfos - SegmentInfoPerCommit siPerCommit = reader.readLegacySegmentInfo(directory, format, input); - SegmentInfo si = siPerCommit.info; - - if (si.getVersion() == null) { - // Could be a 3.0 - try to open the doc stores - if it fails, it's a - // 2.x segment, and an IndexFormatTooOldException will be thrown, - // which is what we want. - Directory dir = directory; - if (Lucene3xSegmentInfoFormat.getDocStoreOffset(si) != -1) { - if (Lucene3xSegmentInfoFormat.getDocStoreIsCompoundFile(si)) { - dir = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName( - Lucene3xSegmentInfoFormat.getDocStoreSegment(si), "", - Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION), IOContext.READONCE, false); - } - } else if (si.getUseCompoundFile()) { - dir = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName( - si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), IOContext.READONCE, false); - } - - try { - Lucene3xStoredFieldsReader.checkCodeVersion(dir, Lucene3xSegmentInfoFormat.getDocStoreSegment(si)); - } finally { - // If we opened the directory, close it - if (dir != directory) dir.close(); - } - - // Above call succeeded, so it's a 3.0 segment. Upgrade it so the next - // time the segment is read, its version won't be null and we won't - // need to open FieldsReader every time for each such segment. - si.setVersion("3.0"); - } else if (si.getVersion().equals("2.x")) { - // If it's a 3x index touched by 3.1+ code, then segments record their - // version, whether they are 2.x ones or not. We detect that and throw - // appropriate exception. - throw new IndexFormatTooOldException("segment " + si.name + " in resource " + input, si.getVersion()); - } - infos.add(siPerCommit); - } - - infos.userData = input.readStringStringMap(); - } - - @Override - public SegmentInfo read(Directory directory, String segmentName, IOContext context) throws IOException { - // NOTE: this is NOT how 3.x is really written... - String fileName = IndexFileNames.segmentFileName(segmentName, "", Lucene3xSegmentInfoFormat.UPGRADED_SI_EXTENSION); - - boolean success = false; - - IndexInput input = directory.openInput(fileName, context); - - try { - SegmentInfo si = readUpgradedSegmentInfo(segmentName, directory, input); - success = true; - return si; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(input); - } else { - input.close(); - } - } - } - - private static void addIfExists(Directory dir, Set files, String fileName) throws IOException { - if (dir.fileExists(fileName)) { - files.add(fileName); - } - } - - /** reads from legacy 3.x segments_N */ - private SegmentInfoPerCommit readLegacySegmentInfo(Directory dir, int format, IndexInput input) throws IOException { - // check that it is a format we can understand - if (format > Lucene3xSegmentInfoFormat.FORMAT_DIAGNOSTICS) { - throw new IndexFormatTooOldException(input, format, - Lucene3xSegmentInfoFormat.FORMAT_DIAGNOSTICS, Lucene3xSegmentInfoFormat.FORMAT_3_1); - } - if (format < Lucene3xSegmentInfoFormat.FORMAT_3_1) { - throw new IndexFormatTooNewException(input, format, - Lucene3xSegmentInfoFormat.FORMAT_DIAGNOSTICS, Lucene3xSegmentInfoFormat.FORMAT_3_1); - } - final String version; - if (format <= Lucene3xSegmentInfoFormat.FORMAT_3_1) { - version = input.readString(); - } else { - version = null; - } - - final String name = input.readString(); - - final int docCount = input.readInt(); - final long delGen = input.readLong(); - - final int docStoreOffset = input.readInt(); - final Map attributes = new HashMap(); - - // parse the docstore stuff and shove it into attributes - final String docStoreSegment; - final boolean docStoreIsCompoundFile; - if (docStoreOffset != -1) { - docStoreSegment = input.readString(); - docStoreIsCompoundFile = input.readByte() == SegmentInfo.YES; - attributes.put(Lucene3xSegmentInfoFormat.DS_OFFSET_KEY, Integer.toString(docStoreOffset)); - attributes.put(Lucene3xSegmentInfoFormat.DS_NAME_KEY, docStoreSegment); - attributes.put(Lucene3xSegmentInfoFormat.DS_COMPOUND_KEY, Boolean.toString(docStoreIsCompoundFile)); - } else { - docStoreSegment = name; - docStoreIsCompoundFile = false; - } - - // pre-4.0 indexes write a byte if there is a single norms file - byte b = input.readByte(); - - //System.out.println("version=" + version + " name=" + name + " docCount=" + docCount + " delGen=" + delGen + " dso=" + docStoreOffset + " dss=" + docStoreSegment + " dssCFs=" + docStoreIsCompoundFile + " b=" + b + " format=" + format); - - assert 1 == b : "expected 1 but was: "+ b + " format: " + format; - final int numNormGen = input.readInt(); - final Map normGen; - if (numNormGen == SegmentInfo.NO) { - normGen = null; - } else { - normGen = new HashMap(); - for(int j=0;j diagnostics = input.readStringStringMap(); - - if (format <= Lucene3xSegmentInfoFormat.FORMAT_HAS_VECTORS) { - // NOTE: unused - final int hasVectors = input.readByte(); - } - - // Replicate logic from 3.x's SegmentInfo.files(): - final Set files = new HashSet(); - if (isCompoundFile) { - files.add(IndexFileNames.segmentFileName(name, "", IndexFileNames.COMPOUND_FILE_EXTENSION)); - } else { - addIfExists(dir, files, IndexFileNames.segmentFileName(name, "", Lucene3xFieldInfosReader.FIELD_INFOS_EXTENSION)); - addIfExists(dir, files, IndexFileNames.segmentFileName(name, "", Lucene3xPostingsFormat.FREQ_EXTENSION)); - addIfExists(dir, files, IndexFileNames.segmentFileName(name, "", Lucene3xPostingsFormat.PROX_EXTENSION)); - addIfExists(dir, files, IndexFileNames.segmentFileName(name, "", Lucene3xPostingsFormat.TERMS_EXTENSION)); - addIfExists(dir, files, IndexFileNames.segmentFileName(name, "", Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION)); - addIfExists(dir, files, IndexFileNames.segmentFileName(name, "", Lucene3xNormsProducer.NORMS_EXTENSION)); - } - - if (docStoreOffset != -1) { - if (docStoreIsCompoundFile) { - files.add(IndexFileNames.segmentFileName(docStoreSegment, "", Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION)); - } else { - files.add(IndexFileNames.segmentFileName(docStoreSegment, "", Lucene3xStoredFieldsReader.FIELDS_INDEX_EXTENSION)); - files.add(IndexFileNames.segmentFileName(docStoreSegment, "", Lucene3xStoredFieldsReader.FIELDS_EXTENSION)); - addIfExists(dir, files, IndexFileNames.segmentFileName(docStoreSegment, "", Lucene3xTermVectorsReader.VECTORS_INDEX_EXTENSION)); - addIfExists(dir, files, IndexFileNames.segmentFileName(docStoreSegment, "", Lucene3xTermVectorsReader.VECTORS_FIELDS_EXTENSION)); - addIfExists(dir, files, IndexFileNames.segmentFileName(docStoreSegment, "", Lucene3xTermVectorsReader.VECTORS_DOCUMENTS_EXTENSION)); - } - } else if (!isCompoundFile) { - files.add(IndexFileNames.segmentFileName(name, "", Lucene3xStoredFieldsReader.FIELDS_INDEX_EXTENSION)); - files.add(IndexFileNames.segmentFileName(name, "", Lucene3xStoredFieldsReader.FIELDS_EXTENSION)); - addIfExists(dir, files, IndexFileNames.segmentFileName(name, "", Lucene3xTermVectorsReader.VECTORS_INDEX_EXTENSION)); - addIfExists(dir, files, IndexFileNames.segmentFileName(name, "", Lucene3xTermVectorsReader.VECTORS_FIELDS_EXTENSION)); - addIfExists(dir, files, IndexFileNames.segmentFileName(name, "", Lucene3xTermVectorsReader.VECTORS_DOCUMENTS_EXTENSION)); - } - - // parse the normgen stuff and shove it into attributes - if (normGen != null) { - attributes.put(Lucene3xSegmentInfoFormat.NORMGEN_KEY, Integer.toString(numNormGen)); - for(Map.Entry ent : normGen.entrySet()) { - long gen = ent.getValue(); - if (gen >= SegmentInfo.YES) { - // Definitely a separate norm file, with generation: - files.add(IndexFileNames.fileNameFromGeneration(name, "s" + ent.getKey(), gen)); - attributes.put(Lucene3xSegmentInfoFormat.NORMGEN_PREFIX + ent.getKey(), Long.toString(gen)); - } else if (gen == SegmentInfo.NO) { - // No separate norm - } else { - // We should have already hit indexformat too old exception - assert false; - } - } - } - - SegmentInfo info = new SegmentInfo(dir, version, name, docCount, isCompoundFile, - null, diagnostics, Collections.unmodifiableMap(attributes)); - info.setFiles(files); - - SegmentInfoPerCommit infoPerCommit = new SegmentInfoPerCommit(info, delCount, delGen); - return infoPerCommit; - } - - private SegmentInfo readUpgradedSegmentInfo(String name, Directory dir, IndexInput input) throws IOException { - CodecUtil.checkHeader(input, Lucene3xSegmentInfoFormat.UPGRADED_SI_CODEC_NAME, - Lucene3xSegmentInfoFormat.UPGRADED_SI_VERSION_START, - Lucene3xSegmentInfoFormat.UPGRADED_SI_VERSION_CURRENT); - final String version = input.readString(); - - final int docCount = input.readInt(); - - final Map attributes = input.readStringStringMap(); - - final boolean isCompoundFile = input.readByte() == SegmentInfo.YES; - - final Map diagnostics = input.readStringStringMap(); - - final Set files = input.readStringSet(); - - SegmentInfo info = new SegmentInfo(dir, version, name, docCount, isCompoundFile, - null, diagnostics, Collections.unmodifiableMap(attributes)); - info.setFiles(files); - return info; - } -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSkipListReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSkipListReader.java deleted file mode 100644 index 5d3db46..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSkipListReader.java +++ /dev/null @@ -1,117 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Arrays; - -import org.apache.lucene.codecs.MultiLevelSkipListReader; -import org.apache.lucene.store.IndexInput; - -/** - * @deprecated (4.0) This is only used to read indexes created - * before 4.0. - */ -@Deprecated -final class Lucene3xSkipListReader extends MultiLevelSkipListReader { - private boolean currentFieldStoresPayloads; - private long freqPointer[]; - private long proxPointer[]; - private int payloadLength[]; - - private long lastFreqPointer; - private long lastProxPointer; - private int lastPayloadLength; - - public Lucene3xSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval) { - super(skipStream, maxSkipLevels, skipInterval); - freqPointer = new long[maxSkipLevels]; - proxPointer = new long[maxSkipLevels]; - payloadLength = new int[maxSkipLevels]; - } - - public void init(long skipPointer, long freqBasePointer, long proxBasePointer, int df, boolean storesPayloads) { - super.init(skipPointer, df); - this.currentFieldStoresPayloads = storesPayloads; - lastFreqPointer = freqBasePointer; - lastProxPointer = proxBasePointer; - - Arrays.fill(freqPointer, freqBasePointer); - Arrays.fill(proxPointer, proxBasePointer); - Arrays.fill(payloadLength, 0); - } - - /** Returns the freq pointer of the doc to which the last call of - * {@link MultiLevelSkipListReader#skipTo(int)} has skipped. */ - public long getFreqPointer() { - return lastFreqPointer; - } - - /** Returns the prox pointer of the doc to which the last call of - * {@link MultiLevelSkipListReader#skipTo(int)} has skipped. */ - public long getProxPointer() { - return lastProxPointer; - } - - /** Returns the payload length of the payload stored just before - * the doc to which the last call of {@link MultiLevelSkipListReader#skipTo(int)} - * has skipped. */ - public int getPayloadLength() { - return lastPayloadLength; - } - - @Override - protected void seekChild(int level) throws IOException { - super.seekChild(level); - freqPointer[level] = lastFreqPointer; - proxPointer[level] = lastProxPointer; - payloadLength[level] = lastPayloadLength; - } - - @Override - protected void setLastSkipData(int level) { - super.setLastSkipData(level); - lastFreqPointer = freqPointer[level]; - lastProxPointer = proxPointer[level]; - lastPayloadLength = payloadLength[level]; - } - - @Override - protected int readSkipData(int level, IndexInput skipStream) throws IOException { - int delta; - if (currentFieldStoresPayloads) { - // the current field stores payloads. - // if the doc delta is odd then we have - // to read the current payload length - // because it differs from the length of the - // previous payload - delta = skipStream.readVInt(); - if ((delta & 1) != 0) { - payloadLength[level] = skipStream.readVInt(); - } - delta >>>= 1; - } else { - delta = skipStream.readVInt(); - } - - freqPointer[level] += skipStream.readVInt(); - proxPointer[level] += skipStream.readVInt(); - - return delta; - } -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsFormat.java deleted file mode 100644 index 255343c..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsFormat.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.StoredFieldsReader; -import org.apache.lucene.codecs.StoredFieldsWriter; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; - -/** @deprecated */ -@Deprecated -class Lucene3xStoredFieldsFormat extends StoredFieldsFormat { - - @Override - public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, - FieldInfos fn, IOContext context) throws IOException { - return new Lucene3xStoredFieldsReader(directory, si, fn, context); - } - - @Override - public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, - IOContext context) throws IOException { - throw new UnsupportedOperationException("this codec can only be used for reading"); - } -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java deleted file mode 100644 index 082d26b..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java +++ /dev/null @@ -1,298 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.codecs.StoredFieldsReader; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.IndexFormatTooNewException; -import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.StoredFieldVisitor; -import org.apache.lucene.store.AlreadyClosedException; -import org.apache.lucene.store.CompoundFileDirectory; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.IOUtils; - -import java.io.Closeable; - -/** - * Class responsible for access to stored document fields. - *

- * It uses <segment>.fdt and <segment>.fdx; files. - * - * @deprecated - */ -@Deprecated -final class Lucene3xStoredFieldsReader extends StoredFieldsReader implements Cloneable, Closeable { - private final static int FORMAT_SIZE = 4; - - /** Extension of stored fields file */ - public static final String FIELDS_EXTENSION = "fdt"; - - /** Extension of stored fields index file */ - public static final String FIELDS_INDEX_EXTENSION = "fdx"; - - // Lucene 3.0: Removal of compressed fields - static final int FORMAT_LUCENE_3_0_NO_COMPRESSED_FIELDS = 2; - - // Lucene 3.2: NumericFields are stored in binary format - static final int FORMAT_LUCENE_3_2_NUMERIC_FIELDS = 3; - - // NOTE: if you introduce a new format, make it 1 higher - // than the current one, and always change this if you - // switch to a new format! - public static final int FORMAT_CURRENT = FORMAT_LUCENE_3_2_NUMERIC_FIELDS; - - // when removing support for old versions, leave the last supported version here - static final int FORMAT_MINIMUM = FORMAT_LUCENE_3_0_NO_COMPRESSED_FIELDS; - - // NOTE: bit 0 is free here! You can steal it! - public static final int FIELD_IS_BINARY = 1 << 1; - - // the old bit 1 << 2 was compressed, is now left out - - private static final int _NUMERIC_BIT_SHIFT = 3; - static final int FIELD_IS_NUMERIC_MASK = 0x07 << _NUMERIC_BIT_SHIFT; - - public static final int FIELD_IS_NUMERIC_INT = 1 << _NUMERIC_BIT_SHIFT; - public static final int FIELD_IS_NUMERIC_LONG = 2 << _NUMERIC_BIT_SHIFT; - public static final int FIELD_IS_NUMERIC_FLOAT = 3 << _NUMERIC_BIT_SHIFT; - public static final int FIELD_IS_NUMERIC_DOUBLE = 4 << _NUMERIC_BIT_SHIFT; - - private final FieldInfos fieldInfos; - private final IndexInput fieldsStream; - private final IndexInput indexStream; - private int numTotalDocs; - private int size; - private boolean closed; - private final int format; - - // The docID offset where our docs begin in the index - // file. This will be 0 if we have our own private file. - private int docStoreOffset; - - // when we are inside a compound share doc store (CFX), - // (lucene 3.0 indexes only), we privately open our own fd. - private final CompoundFileDirectory storeCFSReader; - - /** Returns a cloned FieldsReader that shares open - * IndexInputs with the original one. It is the caller's - * job not to close the original FieldsReader until all - * clones are called (eg, currently SegmentReader manages - * this logic). */ - @Override - public Lucene3xStoredFieldsReader clone() { - ensureOpen(); - return new Lucene3xStoredFieldsReader(fieldInfos, numTotalDocs, size, format, docStoreOffset, (IndexInput)fieldsStream.clone(), (IndexInput)indexStream.clone()); - } - - /** Verifies that the code version which wrote the segment is supported. */ - public static void checkCodeVersion(Directory dir, String segment) throws IOException { - final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION); - IndexInput idxStream = dir.openInput(indexStreamFN, IOContext.DEFAULT); - - try { - int format = idxStream.readInt(); - if (format < FORMAT_MINIMUM) - throw new IndexFormatTooOldException(idxStream, format, FORMAT_MINIMUM, FORMAT_CURRENT); - if (format > FORMAT_CURRENT) - throw new IndexFormatTooNewException(idxStream, format, FORMAT_MINIMUM, FORMAT_CURRENT); - } finally { - idxStream.close(); - } - } - - // Used only by clone - private Lucene3xStoredFieldsReader(FieldInfos fieldInfos, int numTotalDocs, int size, int format, int docStoreOffset, - IndexInput fieldsStream, IndexInput indexStream) { - this.fieldInfos = fieldInfos; - this.numTotalDocs = numTotalDocs; - this.size = size; - this.format = format; - this.docStoreOffset = docStoreOffset; - this.fieldsStream = fieldsStream; - this.indexStream = indexStream; - this.storeCFSReader = null; - } - - public Lucene3xStoredFieldsReader(Directory d, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { - final String segment = Lucene3xSegmentInfoFormat.getDocStoreSegment(si); - final int docStoreOffset = Lucene3xSegmentInfoFormat.getDocStoreOffset(si); - final int size = si.getDocCount(); - boolean success = false; - fieldInfos = fn; - try { - if (docStoreOffset != -1 && Lucene3xSegmentInfoFormat.getDocStoreIsCompoundFile(si)) { - d = storeCFSReader = new CompoundFileDirectory(si.dir, - IndexFileNames.segmentFileName(segment, "", Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION), context, false); - } else { - storeCFSReader = null; - } - fieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION), context); - final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION); - indexStream = d.openInput(indexStreamFN, context); - - format = indexStream.readInt(); - - if (format < FORMAT_MINIMUM) - throw new IndexFormatTooOldException(indexStream, format, FORMAT_MINIMUM, FORMAT_CURRENT); - if (format > FORMAT_CURRENT) - throw new IndexFormatTooNewException(indexStream, format, FORMAT_MINIMUM, FORMAT_CURRENT); - - final long indexSize = indexStream.length() - FORMAT_SIZE; - - if (docStoreOffset != -1) { - // We read only a slice out of this shared fields file - this.docStoreOffset = docStoreOffset; - this.size = size; - - // Verify the file is long enough to hold all of our - // docs - assert ((int) (indexSize / 8)) >= size + this.docStoreOffset: "indexSize=" + indexSize + " size=" + size + " docStoreOffset=" + docStoreOffset; - } else { - this.docStoreOffset = 0; - this.size = (int) (indexSize >> 3); - // Verify two sources of "maxDoc" agree: - if (this.size != si.getDocCount()) { - throw new CorruptIndexException("doc counts differ for segment " + segment + ": fieldsReader shows " + this.size + " but segmentInfo shows " + si.getDocCount()); - } - } - numTotalDocs = (int) (indexSize >> 3); - success = true; - } finally { - // With lock-less commits, it's entirely possible (and - // fine) to hit a FileNotFound exception above. In - // this case, we want to explicitly close any subset - // of things that were opened so that we don't have to - // wait for a GC to do so. - if (!success) { - close(); - } - } - } - - /** - * @throws AlreadyClosedException if this FieldsReader is closed - */ - private void ensureOpen() throws AlreadyClosedException { - if (closed) { - throw new AlreadyClosedException("this FieldsReader is closed"); - } - } - - /** - * Closes the underlying {@link org.apache.lucene.store.IndexInput} streams. - * This means that the Fields values will not be accessible. - * - * @throws IOException - */ - public final void close() throws IOException { - if (!closed) { - IOUtils.close(fieldsStream, indexStream, storeCFSReader); - closed = true; - } - } - - private void seekIndex(int docID) throws IOException { - indexStream.seek(FORMAT_SIZE + (docID + docStoreOffset) * 8L); - } - - public final void visitDocument(int n, StoredFieldVisitor visitor) throws CorruptIndexException, IOException { - seekIndex(n); - fieldsStream.seek(indexStream.readLong()); - - final int numFields = fieldsStream.readVInt(); - for (int fieldIDX = 0; fieldIDX < numFields; fieldIDX++) { - int fieldNumber = fieldsStream.readVInt(); - FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber); - - int bits = fieldsStream.readByte() & 0xFF; - assert bits <= (FIELD_IS_NUMERIC_MASK | FIELD_IS_BINARY): "bits=" + Integer.toHexString(bits); - - switch(visitor.needsField(fieldInfo)) { - case YES: - readField(visitor, fieldInfo, bits); - break; - case NO: - skipField(bits); - break; - case STOP: - return; - } - } - } - - private void readField(StoredFieldVisitor visitor, FieldInfo info, int bits) throws IOException { - final int numeric = bits & FIELD_IS_NUMERIC_MASK; - if (numeric != 0) { - switch(numeric) { - case FIELD_IS_NUMERIC_INT: - visitor.intField(info, fieldsStream.readInt()); - return; - case FIELD_IS_NUMERIC_LONG: - visitor.longField(info, fieldsStream.readLong()); - return; - case FIELD_IS_NUMERIC_FLOAT: - visitor.floatField(info, Float.intBitsToFloat(fieldsStream.readInt())); - return; - case FIELD_IS_NUMERIC_DOUBLE: - visitor.doubleField(info, Double.longBitsToDouble(fieldsStream.readLong())); - return; - default: - throw new CorruptIndexException("Invalid numeric type: " + Integer.toHexString(numeric)); - } - } else { - final int length = fieldsStream.readVInt(); - byte bytes[] = new byte[length]; - fieldsStream.readBytes(bytes, 0, length); - if ((bits & FIELD_IS_BINARY) != 0) { - visitor.binaryField(info, bytes, 0, bytes.length); - } else { - visitor.stringField(info, new String(bytes, 0, bytes.length, IOUtils.CHARSET_UTF_8)); - } - } - } - - private void skipField(int bits) throws IOException { - final int numeric = bits & FIELD_IS_NUMERIC_MASK; - if (numeric != 0) { - switch(numeric) { - case FIELD_IS_NUMERIC_INT: - case FIELD_IS_NUMERIC_FLOAT: - fieldsStream.readInt(); - return; - case FIELD_IS_NUMERIC_LONG: - case FIELD_IS_NUMERIC_DOUBLE: - fieldsStream.readLong(); - return; - default: - throw new CorruptIndexException("Invalid numeric type: " + Integer.toHexString(numeric)); - } - } else { - final int length = fieldsStream.readVInt(); - fieldsStream.seek(fieldsStream.getFilePointer() + length); - } - } -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsFormat.java deleted file mode 100644 index 90371b7..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsFormat.java +++ /dev/null @@ -1,80 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.codecs.TermVectorsFormat; -import org.apache.lucene.codecs.TermVectorsReader; -import org.apache.lucene.codecs.TermVectorsWriter; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.store.CompoundFileDirectory; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; - -/** - * Lucene3x ReadOnly TermVectorsFormat implementation - * @deprecated (4.0) This is only used to read indexes created - * before 4.0. - * @lucene.experimental - */ -@Deprecated -class Lucene3xTermVectorsFormat extends TermVectorsFormat { - - @Override - public TermVectorsReader vectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) throws IOException { - final String fileName = IndexFileNames.segmentFileName(Lucene3xSegmentInfoFormat.getDocStoreSegment(segmentInfo), "", Lucene3xTermVectorsReader.VECTORS_FIELDS_EXTENSION); - - // Unfortunately, for 3.x indices, each segment's - // FieldInfos can lie about hasVectors (claim it's true - // when really it's false).... so we have to carefully - // check if the files really exist before trying to open - // them (4.x has fixed this): - final boolean exists; - if (Lucene3xSegmentInfoFormat.getDocStoreOffset(segmentInfo) != -1 && Lucene3xSegmentInfoFormat.getDocStoreIsCompoundFile(segmentInfo)) { - String cfxFileName = IndexFileNames.segmentFileName(Lucene3xSegmentInfoFormat.getDocStoreSegment(segmentInfo), "", Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION); - if (segmentInfo.dir.fileExists(cfxFileName)) { - Directory cfsDir = new CompoundFileDirectory(segmentInfo.dir, cfxFileName, context, false); - try { - exists = cfsDir.fileExists(fileName); - } finally { - cfsDir.close(); - } - } else { - exists = false; - } - } else { - exists = directory.fileExists(fileName); - } - - if (!exists) { - // 3x's FieldInfos sometimes lies and claims a segment - // has vectors when it doesn't: - return null; - } else { - return new Lucene3xTermVectorsReader(directory, segmentInfo, fieldInfos, context); - } - } - - @Override - public TermVectorsWriter vectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context) throws IOException { - throw new UnsupportedOperationException("this codec can only be used for reading"); - } -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java deleted file mode 100644 index d4bf9c6..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java +++ /dev/null @@ -1,701 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Arrays; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Map; - -import org.apache.lucene.codecs.TermVectorsReader; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.DocsAndPositionsEnum; -import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.Fields; -import org.apache.lucene.index.FieldsEnum; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.IndexFormatTooNewException; -import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.store.CompoundFileDirectory; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.IOUtils; - -/** @deprecated */ -@Deprecated -class Lucene3xTermVectorsReader extends TermVectorsReader { - - // NOTE: if you make a new format, it must be larger than - // the current format - - // Changed strings to UTF8 with length-in-bytes not length-in-chars - static final int FORMAT_UTF8_LENGTH_IN_BYTES = 4; - - // NOTE: always change this if you switch to a new format! - // whenever you add a new format, make it 1 larger (positive version logic)! - public static final int FORMAT_CURRENT = FORMAT_UTF8_LENGTH_IN_BYTES; - - // when removing support for old versions, leave the last supported version here - public static final int FORMAT_MINIMUM = FORMAT_UTF8_LENGTH_IN_BYTES; - - //The size in bytes that the FORMAT_VERSION will take up at the beginning of each file - static final int FORMAT_SIZE = 4; - - public static final byte STORE_POSITIONS_WITH_TERMVECTOR = 0x1; - - public static final byte STORE_OFFSET_WITH_TERMVECTOR = 0x2; - - /** Extension of vectors fields file */ - public static final String VECTORS_FIELDS_EXTENSION = "tvf"; - - /** Extension of vectors documents file */ - public static final String VECTORS_DOCUMENTS_EXTENSION = "tvd"; - - /** Extension of vectors index file */ - public static final String VECTORS_INDEX_EXTENSION = "tvx"; - - private FieldInfos fieldInfos; - - private IndexInput tvx; - private IndexInput tvd; - private IndexInput tvf; - private int size; - private int numTotalDocs; - - // The docID offset where our docs begin in the index - // file. This will be 0 if we have our own private file. - private int docStoreOffset; - - // when we are inside a compound share doc store (CFX), - // (lucene 3.0 indexes only), we privately open our own fd. - // TODO: if we are worried, maybe we could eliminate the - // extra fd somehow when you also have vectors... - private final CompoundFileDirectory storeCFSReader; - - private final int format; - - // used by clone - Lucene3xTermVectorsReader(FieldInfos fieldInfos, IndexInput tvx, IndexInput tvd, IndexInput tvf, int size, int numTotalDocs, int docStoreOffset, int format) { - this.fieldInfos = fieldInfos; - this.tvx = tvx; - this.tvd = tvd; - this.tvf = tvf; - this.size = size; - this.numTotalDocs = numTotalDocs; - this.docStoreOffset = docStoreOffset; - this.format = format; - this.storeCFSReader = null; - } - - public Lucene3xTermVectorsReader(Directory d, SegmentInfo si, FieldInfos fieldInfos, IOContext context) - throws CorruptIndexException, IOException { - final String segment = Lucene3xSegmentInfoFormat.getDocStoreSegment(si); - final int docStoreOffset = Lucene3xSegmentInfoFormat.getDocStoreOffset(si); - final int size = si.getDocCount(); - - boolean success = false; - - try { - if (docStoreOffset != -1 && Lucene3xSegmentInfoFormat.getDocStoreIsCompoundFile(si)) { - d = storeCFSReader = new CompoundFileDirectory(si.dir, - IndexFileNames.segmentFileName(segment, "", Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION), context, false); - } else { - storeCFSReader = null; - } - String idxName = IndexFileNames.segmentFileName(segment, "", VECTORS_INDEX_EXTENSION); - tvx = d.openInput(idxName, context); - format = checkValidFormat(tvx); - String fn = IndexFileNames.segmentFileName(segment, "", VECTORS_DOCUMENTS_EXTENSION); - tvd = d.openInput(fn, context); - final int tvdFormat = checkValidFormat(tvd); - fn = IndexFileNames.segmentFileName(segment, "", VECTORS_FIELDS_EXTENSION); - tvf = d.openInput(fn, context); - final int tvfFormat = checkValidFormat(tvf); - - assert format == tvdFormat; - assert format == tvfFormat; - - numTotalDocs = (int) (tvx.length() >> 4); - - if (-1 == docStoreOffset) { - this.docStoreOffset = 0; - this.size = numTotalDocs; - assert size == 0 || numTotalDocs == size; - } else { - this.docStoreOffset = docStoreOffset; - this.size = size; - // Verify the file is long enough to hold all of our - // docs - assert numTotalDocs >= size + docStoreOffset: "numTotalDocs=" + numTotalDocs + " size=" + size + " docStoreOffset=" + docStoreOffset; - } - - this.fieldInfos = fieldInfos; - success = true; - } finally { - // With lock-less commits, it's entirely possible (and - // fine) to hit a FileNotFound exception above. In - // this case, we want to explicitly close any subset - // of things that were opened so that we don't have to - // wait for a GC to do so. - if (!success) { - close(); - } - } - } - - // Not private to avoid synthetic access$NNN methods - void seekTvx(final int docNum) throws IOException { - tvx.seek((docNum + docStoreOffset) * 16L + FORMAT_SIZE); - } - - private int checkValidFormat(IndexInput in) throws CorruptIndexException, IOException - { - int format = in.readInt(); - if (format < FORMAT_MINIMUM) - throw new IndexFormatTooOldException(in, format, FORMAT_MINIMUM, FORMAT_CURRENT); - if (format > FORMAT_CURRENT) - throw new IndexFormatTooNewException(in, format, FORMAT_MINIMUM, FORMAT_CURRENT); - return format; - } - - public void close() throws IOException { - IOUtils.close(tvx, tvd, tvf, storeCFSReader); - } - - /** - * - * @return The number of documents in the reader - */ - int size() { - return size; - } - - private class TVFields extends Fields { - private final int[] fieldNumbers; - private final long[] fieldFPs; - private final Map fieldNumberToIndex = new HashMap(); - - public TVFields(int docID) throws IOException { - seekTvx(docID); - tvd.seek(tvx.readLong()); - - final int fieldCount = tvd.readVInt(); - assert fieldCount >= 0; - if (fieldCount != 0) { - fieldNumbers = new int[fieldCount]; - fieldFPs = new long[fieldCount]; - for(int fieldUpto=0;fieldUpto getComparator() { - if (unicodeSortOrder) { - return BytesRef.getUTF8SortedAsUnicodeComparator(); - } else { - return BytesRef.getUTF8SortedAsUTF16Comparator(); - } - } - } - - static class TermAndPostings { - BytesRef term; - int freq; - int[] positions; - int[] startOffsets; - int[] endOffsets; - } - - private class TVTermsEnum extends TermsEnum { - private boolean unicodeSortOrder; - private final IndexInput origTVF; - private final IndexInput tvf; - private int numTerms; - private int currentTerm; - private boolean storePositions; - private boolean storeOffsets; - - private TermAndPostings[] termAndPostings; - - // NOTE: tvf is pre-positioned by caller - public TVTermsEnum() throws IOException { - this.origTVF = Lucene3xTermVectorsReader.this.tvf; - tvf = (IndexInput) origTVF.clone(); - } - - public boolean canReuse(IndexInput tvf) { - return tvf == origTVF; - } - - public void reset(int numTerms, long tvfFPStart, boolean unicodeSortOrder) throws IOException { - this.numTerms = numTerms; - currentTerm = -1; - tvf.seek(tvfFPStart); - final byte bits = tvf.readByte(); - storePositions = (bits & STORE_POSITIONS_WITH_TERMVECTOR) != 0; - storeOffsets = (bits & STORE_OFFSET_WITH_TERMVECTOR) != 0; - this.unicodeSortOrder = unicodeSortOrder; - readVectors(); - if (unicodeSortOrder) { - Arrays.sort(termAndPostings, new Comparator() { - public int compare(TermAndPostings left, TermAndPostings right) { - return left.term.compareTo(right.term); - } - }); - } - } - - private void readVectors() throws IOException { - termAndPostings = new TermAndPostings[numTerms]; - BytesRef lastTerm = new BytesRef(); - for (int i = 0; i < numTerms; i++) { - TermAndPostings t = new TermAndPostings(); - BytesRef term = new BytesRef(); - term.copyBytes(lastTerm); - final int start = tvf.readVInt(); - final int deltaLen = tvf.readVInt(); - term.length = start + deltaLen; - term.grow(term.length); - tvf.readBytes(term.bytes, start, deltaLen); - t.term = term; - int freq = tvf.readVInt(); - t.freq = freq; - - if (storePositions) { - int positions[] = new int[freq]; - int pos = 0; - for(int posUpto=0;posUpto comparator = getComparator(); - for (int i = 0; i < numTerms; i++) { - int cmp = comparator.compare(text, termAndPostings[i].term); - if (cmp < 0) { - currentTerm = i; - return SeekStatus.NOT_FOUND; - } else if (cmp == 0) { - currentTerm = i; - return SeekStatus.FOUND; - } - } - currentTerm = termAndPostings.length; - return SeekStatus.END; - } - - @Override - public void seekExact(long ord) { - throw new UnsupportedOperationException(); - } - - @Override - public BytesRef next() throws IOException { - if (++currentTerm >= numTerms) { - return null; - } - return term(); - } - - @Override - public BytesRef term() { - return termAndPostings[currentTerm].term; - } - - @Override - public long ord() { - throw new UnsupportedOperationException(); - } - - @Override - public int docFreq() { - return 1; - } - - @Override - public long totalTermFreq() { - return termAndPostings[currentTerm].freq; - } - - @Override - public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs /* ignored */) throws IOException { - TVDocsEnum docsEnum; - if (reuse != null && reuse instanceof TVDocsEnum) { - docsEnum = (TVDocsEnum) reuse; - } else { - docsEnum = new TVDocsEnum(); - } - docsEnum.reset(liveDocs, termAndPostings[currentTerm]); - return docsEnum; - } - - @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException { - if (needsOffsets && !storeOffsets) { - return null; - } - - if (!storePositions && !storeOffsets) { - return null; - } - - TVDocsAndPositionsEnum docsAndPositionsEnum; - if (reuse != null && reuse instanceof TVDocsAndPositionsEnum) { - docsAndPositionsEnum = (TVDocsAndPositionsEnum) reuse; - } else { - docsAndPositionsEnum = new TVDocsAndPositionsEnum(); - } - docsAndPositionsEnum.reset(liveDocs, termAndPostings[currentTerm]); - return docsAndPositionsEnum; - } - - @Override - public Comparator getComparator() { - if (unicodeSortOrder) { - return BytesRef.getUTF8SortedAsUnicodeComparator(); - } else { - return BytesRef.getUTF8SortedAsUTF16Comparator(); - } - } - } - - // NOTE: sort of a silly class, since you can get the - // freq() already by TermsEnum.totalTermFreq - private static class TVDocsEnum extends DocsEnum { - private boolean didNext; - private int doc = -1; - private int freq; - private Bits liveDocs; - - @Override - public int freq() throws IOException { - return freq; - } - - @Override - public int docID() { - return doc; - } - - @Override - public int nextDoc() { - if (!didNext && (liveDocs == null || liveDocs.get(0))) { - didNext = true; - return (doc = 0); - } else { - return (doc = NO_MORE_DOCS); - } - } - - @Override - public int advance(int target) { - if (!didNext && target == 0) { - return nextDoc(); - } else { - return (doc = NO_MORE_DOCS); - } - } - - public void reset(Bits liveDocs, TermAndPostings termAndPostings) { - this.liveDocs = liveDocs; - this.freq = termAndPostings.freq; - this.doc = -1; - didNext = false; - } - } - - private static class TVDocsAndPositionsEnum extends DocsAndPositionsEnum { - private boolean didNext; - private int doc = -1; - private int nextPos; - private Bits liveDocs; - private int[] positions; - private int[] startOffsets; - private int[] endOffsets; - - @Override - public int freq() throws IOException { - if (positions != null) { - return positions.length; - } else { - assert startOffsets != null; - return startOffsets.length; - } - } - - @Override - public int docID() { - return doc; - } - - @Override - public int nextDoc() { - if (!didNext && (liveDocs == null || liveDocs.get(0))) { - didNext = true; - return (doc = 0); - } else { - return (doc = NO_MORE_DOCS); - } - } - - @Override - public int advance(int target) { - if (!didNext && target == 0) { - return nextDoc(); - } else { - return (doc = NO_MORE_DOCS); - } - } - - public void reset(Bits liveDocs, TermAndPostings termAndPostings) { - this.liveDocs = liveDocs; - this.positions = termAndPostings.positions; - this.startOffsets = termAndPostings.startOffsets; - this.endOffsets = termAndPostings.endOffsets; - this.doc = -1; - didNext = false; - nextPos = 0; - } - - @Override - public BytesRef getPayload() { - return null; - } - - @Override - public boolean hasPayload() { - return false; - } - - @Override - public int nextPosition() { - assert (positions != null && nextPos < positions.length) || - startOffsets != null && nextPos < startOffsets.length; - - if (positions != null) { - return positions[nextPos++]; - } else { - nextPos++; - return -1; - } - } - - @Override - public int startOffset() { - assert startOffsets != null; - return startOffsets[nextPos-1]; - } - - @Override - public int endOffset() { - assert endOffsets != null; - return endOffsets[nextPos-1]; - } - } - - @Override - public Fields get(int docID) throws IOException { - if (docID < 0 || docID >= numTotalDocs) { - throw new IllegalArgumentException("doID=" + docID + " is out of bounds [0.." + (numTotalDocs-1) + "]"); - } - if (tvx != null) { - Fields fields = new TVFields(docID); - if (fields.size() == 0) { - // TODO: we can improve writer here, eg write 0 into - // tvx file, so we know on first read from tvx that - // this doc has no TVs - return null; - } else { - return fields; - } - } else { - return null; - } - } - - @Override - public TermVectorsReader clone() { - IndexInput cloneTvx = null; - IndexInput cloneTvd = null; - IndexInput cloneTvf = null; - - // These are null when a TermVectorsReader was created - // on a segment that did not have term vectors saved - if (tvx != null && tvd != null && tvf != null) { - cloneTvx = (IndexInput) tvx.clone(); - cloneTvd = (IndexInput) tvd.clone(); - cloneTvf = (IndexInput) tvf.clone(); - } - - return new Lucene3xTermVectorsReader(fieldInfos, cloneTvx, cloneTvd, cloneTvf, size, numTotalDocs, docStoreOffset, format); - } - - // If this returns, we do the surrogates shuffle so that the - // terms are sorted by unicode sort order. This should be - // true when segments are used for "normal" searching; - // it's only false during testing, to create a pre-flex - // index, using the test-only PreFlexRW. - protected boolean sortTermsByUnicode() { - return true; - } -} - diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/SegmentTermDocs.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/SegmentTermDocs.java deleted file mode 100644 index 0ba8705..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/SegmentTermDocs.java +++ /dev/null @@ -1,229 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.Term; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.Bits; - -/** @deprecated (4.0) - * @lucene.experimental */ -@Deprecated -class SegmentTermDocs { - //protected SegmentReader parent; - private final FieldInfos fieldInfos; - private final TermInfosReader tis; - protected Bits liveDocs; - protected IndexInput freqStream; - protected int count; - protected int df; - int doc = 0; - int freq; - - private int skipInterval; - private int maxSkipLevels; - private Lucene3xSkipListReader skipListReader; - - private long freqBasePointer; - private long proxBasePointer; - - private long skipPointer; - private boolean haveSkipped; - - protected boolean currentFieldStoresPayloads; - protected IndexOptions indexOptions; - - public SegmentTermDocs(IndexInput freqStream, TermInfosReader tis, FieldInfos fieldInfos) { - this.freqStream = (IndexInput) freqStream.clone(); - this.tis = tis; - this.fieldInfos = fieldInfos; - skipInterval = tis.getSkipInterval(); - maxSkipLevels = tis.getMaxSkipLevels(); - } - - public void seek(Term term) throws IOException { - TermInfo ti = tis.get(term); - seek(ti, term); - } - - public void setLiveDocs(Bits liveDocs) { - this.liveDocs = liveDocs; - } - - public void seek(SegmentTermEnum segmentTermEnum) throws IOException { - TermInfo ti; - Term term; - - // use comparison of fieldinfos to verify that termEnum belongs to the same segment as this SegmentTermDocs - if (segmentTermEnum.fieldInfos == fieldInfos) { // optimized case - term = segmentTermEnum.term(); - ti = segmentTermEnum.termInfo(); - } else { // punt case - term = segmentTermEnum.term(); - ti = tis.get(term); - } - - seek(ti, term); - } - - void seek(TermInfo ti, Term term) throws IOException { - count = 0; - FieldInfo fi = fieldInfos.fieldInfo(term.field()); - this.indexOptions = (fi != null) ? fi.getIndexOptions() : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; - currentFieldStoresPayloads = (fi != null) ? fi.hasPayloads() : false; - if (ti == null) { - df = 0; - } else { - df = ti.docFreq; - doc = 0; - freqBasePointer = ti.freqPointer; - proxBasePointer = ti.proxPointer; - skipPointer = freqBasePointer + ti.skipOffset; - freqStream.seek(freqBasePointer); - haveSkipped = false; - } - } - - public void close() throws IOException { - freqStream.close(); - if (skipListReader != null) - skipListReader.close(); - } - - public final int doc() { return doc; } - public final int freq() { - assert indexOptions != IndexOptions.DOCS_ONLY; - return freq; - } - - protected void skippingDoc() throws IOException { - } - - public boolean next() throws IOException { - while (true) { - if (count == df) - return false; - final int docCode = freqStream.readVInt(); - - if (indexOptions == IndexOptions.DOCS_ONLY) { - doc += docCode; - } else { - doc += docCode >>> 1; // shift off low bit - if ((docCode & 1) != 0) // if low bit is set - freq = 1; // freq is one - else { - freq = freqStream.readVInt(); // else read freq - assert freq != 1; - } - } - - count++; - - if (liveDocs == null || liveDocs.get(doc)) { - break; - } - skippingDoc(); - } - return true; - } - - /** Optimized implementation. */ - public int read(final int[] docs, final int[] freqs) - throws IOException { - final int length = docs.length; - if (indexOptions == IndexOptions.DOCS_ONLY) { - return readNoTf(docs, freqs, length); - } else { - int i = 0; - while (i < length && count < df) { - // manually inlined call to next() for speed - final int docCode = freqStream.readVInt(); - doc += docCode >>> 1; // shift off low bit - if ((docCode & 1) != 0) // if low bit is set - freq = 1; // freq is one - else - freq = freqStream.readVInt(); // else read freq - count++; - - if (liveDocs == null || liveDocs.get(doc)) { - docs[i] = doc; - freqs[i] = freq; - ++i; - } - } - return i; - } - } - - private final int readNoTf(final int[] docs, final int[] freqs, final int length) throws IOException { - int i = 0; - while (i < length && count < df) { - // manually inlined call to next() for speed - doc += freqStream.readVInt(); - count++; - - if (liveDocs == null || liveDocs.get(doc)) { - docs[i] = doc; - // Hardware freq to 1 when term freqs were not - // stored in the index - freqs[i] = 1; - ++i; - } - } - return i; - } - - - /** Overridden by SegmentTermPositions to skip in prox stream. */ - protected void skipProx(long proxPointer, int payloadLength) throws IOException {} - - /** Optimized implementation. */ - public boolean skipTo(int target) throws IOException { - // don't skip if the target is close (within skipInterval docs away) - if ((target - skipInterval) >= doc && df >= skipInterval) { // optimized case - if (skipListReader == null) - skipListReader = new Lucene3xSkipListReader((IndexInput) freqStream.clone(), maxSkipLevels, skipInterval); // lazily clone - - if (!haveSkipped) { // lazily initialize skip stream - skipListReader.init(skipPointer, freqBasePointer, proxBasePointer, df, currentFieldStoresPayloads); - haveSkipped = true; - } - - int newCount = skipListReader.skipTo(target); - if (newCount > count) { - freqStream.seek(skipListReader.getFreqPointer()); - skipProx(skipListReader.getProxPointer(), skipListReader.getPayloadLength()); - - doc = skipListReader.getDoc(); - count = newCount; - } - } - - // done skipping, now just scan - do { - if (!next()) - return false; - } while (target > doc); - return true; - } -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/SegmentTermEnum.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/SegmentTermEnum.java deleted file mode 100644 index b9b796b..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/SegmentTermEnum.java +++ /dev/null @@ -1,225 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.index.IndexFormatTooNewException; - -/** - * @deprecated (4.0) No longer used with flex indexing, except for - * reading old segments - * @lucene.experimental */ - -@Deprecated -final class SegmentTermEnum implements Cloneable { - private IndexInput input; - FieldInfos fieldInfos; - long size; - long position = -1; - - // Changed strings to true utf8 with length-in-bytes not - // length-in-chars - public static final int FORMAT_VERSION_UTF8_LENGTH_IN_BYTES = -4; - - // NOTE: always change this if you switch to a new format! - // whenever you add a new format, make it 1 smaller (negative version logic)! - public static final int FORMAT_CURRENT = FORMAT_VERSION_UTF8_LENGTH_IN_BYTES; - - // when removing support for old versions, leave the last supported version here - public static final int FORMAT_MINIMUM = FORMAT_VERSION_UTF8_LENGTH_IN_BYTES; - - private TermBuffer termBuffer = new TermBuffer(); - private TermBuffer prevBuffer = new TermBuffer(); - private TermBuffer scanBuffer = new TermBuffer(); // used for scanning - - TermInfo termInfo = new TermInfo(); - - private int format; - private boolean isIndex = false; - long indexPointer = 0; - int indexInterval; - int skipInterval; - int newSuffixStart; - int maxSkipLevels; - private boolean first = true; - - SegmentTermEnum(IndexInput i, FieldInfos fis, boolean isi) - throws CorruptIndexException, IOException { - input = i; - fieldInfos = fis; - isIndex = isi; - maxSkipLevels = 1; // use single-level skip lists for formats > -3 - - int firstInt = input.readInt(); - if (firstInt >= 0) { - // original-format file, without explicit format version number - format = 0; - size = firstInt; - - // back-compatible settings - indexInterval = 128; - skipInterval = Integer.MAX_VALUE; // switch off skipTo optimization - } else { - // we have a format version number - format = firstInt; - - // check that it is a format we can understand - if (format > FORMAT_MINIMUM) - throw new IndexFormatTooOldException(input, format, FORMAT_MINIMUM, FORMAT_CURRENT); - if (format < FORMAT_CURRENT) - throw new IndexFormatTooNewException(input, format, FORMAT_MINIMUM, FORMAT_CURRENT); - - size = input.readLong(); // read the size - - indexInterval = input.readInt(); - skipInterval = input.readInt(); - maxSkipLevels = input.readInt(); - assert indexInterval > 0: "indexInterval=" + indexInterval + " is negative; must be > 0"; - assert skipInterval > 0: "skipInterval=" + skipInterval + " is negative; must be > 0"; - } - } - - @Override - protected SegmentTermEnum clone() { - SegmentTermEnum clone = null; - try { - clone = (SegmentTermEnum) super.clone(); - } catch (CloneNotSupportedException e) {} - - clone.input = (IndexInput) input.clone(); - clone.termInfo = new TermInfo(termInfo); - - clone.termBuffer = termBuffer.clone(); - clone.prevBuffer = prevBuffer.clone(); - clone.scanBuffer = new TermBuffer(); - - return clone; - } - - final void seek(long pointer, long p, Term t, TermInfo ti) - throws IOException { - input.seek(pointer); - position = p; - termBuffer.set(t); - prevBuffer.reset(); - //System.out.println(" ste doSeek prev=" + prevBuffer.toTerm() + " this=" + this); - termInfo.set(ti); - first = p == -1; - } - - /** Increments the enumeration to the next element. True if one exists.*/ - public final boolean next() throws IOException { - prevBuffer.set(termBuffer); - //System.out.println(" ste setPrev=" + prev() + " this=" + this); - - if (position++ >= size - 1) { - termBuffer.reset(); - //System.out.println(" EOF"); - return false; - } - - termBuffer.read(input, fieldInfos); - newSuffixStart = termBuffer.newSuffixStart; - - termInfo.docFreq = input.readVInt(); // read doc freq - termInfo.freqPointer += input.readVLong(); // read freq pointer - termInfo.proxPointer += input.readVLong(); // read prox pointer - - if (termInfo.docFreq >= skipInterval) - termInfo.skipOffset = input.readVInt(); - - if (isIndex) - indexPointer += input.readVLong(); // read index pointer - - //System.out.println(" ste ret term=" + term()); - return true; - } - - /* Optimized scan, without allocating new terms. - * Return number of invocations to next(). - * - * NOTE: LUCENE-3183: if you pass Term("", "") here then this - * will incorrectly return before positioning the enum, - * and position will be -1; caller must detect this. */ - final int scanTo(Term term) throws IOException { - scanBuffer.set(term); - int count = 0; - if (first) { - // Always force initial next() in case term is - // Term("", "") - next(); - first = false; - count++; - } - while (scanBuffer.compareTo(termBuffer) > 0 && next()) { - count++; - } - return count; - } - - /** Returns the current Term in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - public final Term term() { - return termBuffer.toTerm(); - } - - /** Returns the previous Term enumerated. Initially null.*/ - final Term prev() { - return prevBuffer.toTerm(); - } - - /** Returns the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - final TermInfo termInfo() { - return new TermInfo(termInfo); - } - - /** Sets the argument to the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - final void termInfo(TermInfo ti) { - ti.set(termInfo); - } - - /** Returns the docFreq from the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - public final int docFreq() { - return termInfo.docFreq; - } - - /* Returns the freqPointer from the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - final long freqPointer() { - return termInfo.freqPointer; - } - - /* Returns the proxPointer from the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - final long proxPointer() { - return termInfo.proxPointer; - } - - /** Closes the enumeration to further activity, freeing resources. */ - public final void close() throws IOException { - input.close(); - } -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/SegmentTermPositions.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/SegmentTermPositions.java deleted file mode 100644 index bdb0961..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/SegmentTermPositions.java +++ /dev/null @@ -1,219 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.store.IndexInput; - -/** - * @lucene.experimental - * @deprecated (4.0) - */ -@Deprecated -final class SegmentTermPositions -extends SegmentTermDocs { - private IndexInput proxStream; - private IndexInput proxStreamOrig; - private int proxCount; - private int position; - - // the current payload length - private int payloadLength; - // indicates whether the payload of the current position has - // been read from the proxStream yet - private boolean needToLoadPayload; - - // these variables are being used to remember information - // for a lazy skip - private long lazySkipPointer = -1; - private int lazySkipProxCount = 0; - - /* - SegmentTermPositions(SegmentReader p) { - super(p); - this.proxStream = null; // the proxStream will be cloned lazily when nextPosition() is called for the first time - } - */ - - public SegmentTermPositions(IndexInput freqStream, IndexInput proxStream, TermInfosReader tis, FieldInfos fieldInfos) { - super(freqStream, tis, fieldInfos); - this.proxStreamOrig = proxStream; // the proxStream will be cloned lazily when nextPosition() is called for the first time - } - - @Override - final void seek(TermInfo ti, Term term) throws IOException { - super.seek(ti, term); - if (ti != null) - lazySkipPointer = ti.proxPointer; - - lazySkipProxCount = 0; - proxCount = 0; - payloadLength = 0; - needToLoadPayload = false; - } - - @Override - public final void close() throws IOException { - super.close(); - if (proxStream != null) proxStream.close(); - } - - public final int nextPosition() throws IOException { - if (indexOptions != IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) - // This field does not store positions, payloads - return 0; - // perform lazy skips if necessary - lazySkip(); - proxCount--; - return position += readDeltaPosition(); - } - - private final int readDeltaPosition() throws IOException { - int delta = proxStream.readVInt(); - if (currentFieldStoresPayloads) { - // if the current field stores payloads then - // the position delta is shifted one bit to the left. - // if the LSB is set, then we have to read the current - // payload length - if ((delta & 1) != 0) { - payloadLength = proxStream.readVInt(); - } - delta >>>= 1; - needToLoadPayload = true; - } - return delta; - } - - @Override - protected final void skippingDoc() throws IOException { - // we remember to skip a document lazily - lazySkipProxCount += freq; - } - - @Override - public final boolean next() throws IOException { - // we remember to skip the remaining positions of the current - // document lazily - lazySkipProxCount += proxCount; - - if (super.next()) { // run super - proxCount = freq; // note frequency - position = 0; // reset position - return true; - } - return false; - } - - @Override - public final int read(final int[] docs, final int[] freqs) { - throw new UnsupportedOperationException("TermPositions does not support processing multiple documents in one call. Use TermDocs instead."); - } - - - /** Called by super.skipTo(). */ - @Override - protected void skipProx(long proxPointer, int payloadLength) throws IOException { - // we save the pointer, we might have to skip there lazily - lazySkipPointer = proxPointer; - lazySkipProxCount = 0; - proxCount = 0; - this.payloadLength = payloadLength; - needToLoadPayload = false; - } - - private void skipPositions(int n) throws IOException { - assert indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; - for (int f = n; f > 0; f--) { // skip unread positions - readDeltaPosition(); - skipPayload(); - } - } - - private void skipPayload() throws IOException { - if (needToLoadPayload && payloadLength > 0) { - proxStream.seek(proxStream.getFilePointer() + payloadLength); - } - needToLoadPayload = false; - } - - // It is not always necessary to move the prox pointer - // to a new document after the freq pointer has been moved. - // Consider for example a phrase query with two terms: - // the freq pointer for term 1 has to move to document x - // to answer the question if the term occurs in that document. But - // only if term 2 also matches document x, the positions have to be - // read to figure out if term 1 and term 2 appear next - // to each other in document x and thus satisfy the query. - // So we move the prox pointer lazily to the document - // as soon as positions are requested. - private void lazySkip() throws IOException { - if (proxStream == null) { - // clone lazily - proxStream = (IndexInput)proxStreamOrig.clone(); - } - - // we might have to skip the current payload - // if it was not read yet - skipPayload(); - - if (lazySkipPointer != -1) { - proxStream.seek(lazySkipPointer); - lazySkipPointer = -1; - } - - if (lazySkipProxCount != 0) { - skipPositions(lazySkipProxCount); - lazySkipProxCount = 0; - } - } - - public int getPayloadLength() { - return payloadLength; - } - - public byte[] getPayload(byte[] data, int offset) throws IOException { - if (!needToLoadPayload) { - throw new IOException("Either no payload exists at this term position or an attempt was made to load it more than once."); - } - - // read payloads lazily - byte[] retArray; - int retOffset; - if (data == null || data.length - offset < payloadLength) { - // the array is too small to store the payload data, - // so we allocate a new one - retArray = new byte[payloadLength]; - retOffset = 0; - } else { - retArray = data; - retOffset = offset; - } - proxStream.readBytes(retArray, retOffset, payloadLength); - needToLoadPayload = false; - return retArray; - } - - public boolean isPayloadAvailable() { - return needToLoadPayload && payloadLength > 0; - } - -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermBuffer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermBuffer.java deleted file mode 100644 index 097b001..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermBuffer.java +++ /dev/null @@ -1,128 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Comparator; - -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.FieldInfos; - -/** - * @lucene.experimental - * @deprecated (4.0) - */ -@Deprecated -final class TermBuffer implements Cloneable { - - private String field; - private Term term; // cached - - private BytesRef bytes = new BytesRef(10); - - // Cannot be -1 since (strangely) we write that - // fieldNumber into index for first indexed term: - private int currentFieldNumber = -2; - - private static final Comparator utf8AsUTF16Comparator = BytesRef.getUTF8SortedAsUTF16Comparator(); - - int newSuffixStart; // only valid right after .read is called - - public int compareTo(TermBuffer other) { - if (field == other.field) // fields are interned - // (only by PreFlex codec) - return utf8AsUTF16Comparator.compare(bytes, other.bytes); - else - return field.compareTo(other.field); - } - - public void read(IndexInput input, FieldInfos fieldInfos) - throws IOException { - this.term = null; // invalidate cache - newSuffixStart = input.readVInt(); - int length = input.readVInt(); - int totalLength = newSuffixStart + length; - if (bytes.bytes.length < totalLength) { - bytes.grow(totalLength); - } - bytes.length = totalLength; - input.readBytes(bytes.bytes, newSuffixStart, length); - final int fieldNumber = input.readVInt(); - if (fieldNumber != currentFieldNumber) { - currentFieldNumber = fieldNumber; - // NOTE: too much sneakiness here, seriously this is a negative vint?! - if (currentFieldNumber == -1) { - field = ""; - } else { - assert fieldInfos.fieldInfo(currentFieldNumber) != null : currentFieldNumber; - field = fieldInfos.fieldInfo(currentFieldNumber).name.intern(); - } - } else { - assert field.equals(fieldInfos.fieldInfo(fieldNumber).name) : "currentFieldNumber=" + currentFieldNumber + " field=" + field + " vs " + fieldInfos.fieldInfo(fieldNumber) == null ? "null" : fieldInfos.fieldInfo(fieldNumber).name; - } - } - - public void set(Term term) { - if (term == null) { - reset(); - return; - } - bytes.copyBytes(term.bytes()); - field = term.field().intern(); - currentFieldNumber = -1; - this.term = term; - } - - public void set(TermBuffer other) { - field = other.field; - currentFieldNumber = other.currentFieldNumber; - // dangerous to copy Term over, since the underlying - // BytesRef could subsequently be modified: - term = null; - bytes.copyBytes(other.bytes); - } - - public void reset() { - field = null; - term = null; - currentFieldNumber= -1; - } - - public Term toTerm() { - if (field == null) // unset - return null; - - if (term == null) { - term = new Term(field, BytesRef.deepCopyOf(bytes)); - } - - return term; - } - - @Override - protected TermBuffer clone() { - TermBuffer clone = null; - try { - clone = (TermBuffer)super.clone(); - } catch (CloneNotSupportedException e) {} - clone.bytes = BytesRef.deepCopyOf(bytes); - return clone; - } -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfo.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfo.java deleted file mode 100644 index ad715b4..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfo.java +++ /dev/null @@ -1,63 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** A TermInfo is the record of information stored for a - * term - * @deprecated (4.0) This class is no longer used in flexible - * indexing. */ - -@Deprecated -class TermInfo { - /** The number of documents which contain the term. */ - public int docFreq = 0; - - public long freqPointer = 0; - public long proxPointer = 0; - public int skipOffset; - - public TermInfo() {} - - public TermInfo(int df, long fp, long pp) { - docFreq = df; - freqPointer = fp; - proxPointer = pp; - } - - public TermInfo(TermInfo ti) { - docFreq = ti.docFreq; - freqPointer = ti.freqPointer; - proxPointer = ti.proxPointer; - skipOffset = ti.skipOffset; - } - - public final void set(int docFreq, - long freqPointer, long proxPointer, int skipOffset) { - this.docFreq = docFreq; - this.freqPointer = freqPointer; - this.proxPointer = proxPointer; - this.skipOffset = skipOffset; - } - - public final void set(TermInfo ti) { - docFreq = ti.docFreq; - freqPointer = ti.freqPointer; - proxPointer = ti.proxPointer; - skipOffset = ti.skipOffset; - } -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReader.java deleted file mode 100644 index de3cb84..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReader.java +++ /dev/null @@ -1,351 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Comparator; - -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.Term; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CloseableThreadLocal; -import org.apache.lucene.util.DoubleBarrelLRUCache; - -/** This stores a monotonically increasing set of pairs in a - * Directory. Pairs are accessed either by Term or by ordinal position the - * set - * @deprecated (4.0) This class has been replaced by - * FormatPostingsTermsDictReader, except for reading old segments. - * @lucene.experimental - */ -@Deprecated -final class TermInfosReader { - private final Directory directory; - private final String segment; - private final FieldInfos fieldInfos; - - private final CloseableThreadLocal threadResources = new CloseableThreadLocal(); - private final SegmentTermEnum origEnum; - private final long size; - - private final TermInfosReaderIndex index; - private final int indexLength; - - private final int totalIndexInterval; - - private final static int DEFAULT_CACHE_SIZE = 1024; - - // Just adds term's ord to TermInfo - private final static class TermInfoAndOrd extends TermInfo { - final long termOrd; - public TermInfoAndOrd(TermInfo ti, long termOrd) { - super(ti); - assert termOrd >= 0; - this.termOrd = termOrd; - } - } - - private static class CloneableTerm extends DoubleBarrelLRUCache.CloneableKey { - Term term; - public CloneableTerm(Term t) { - this.term = t; - } - - @Override - public boolean equals(Object other) { - CloneableTerm t = (CloneableTerm) other; - return this.term.equals(t.term); - } - - @Override - public int hashCode() { - return term.hashCode(); - } - - @Override - public CloneableTerm clone() { - return new CloneableTerm(term); - } - } - - private final DoubleBarrelLRUCache termsCache = new DoubleBarrelLRUCache(DEFAULT_CACHE_SIZE); - - /** - * Per-thread resources managed by ThreadLocal - */ - private static final class ThreadResources { - SegmentTermEnum termEnum; - } - - TermInfosReader(Directory dir, String seg, FieldInfos fis, IOContext context, int indexDivisor) - throws CorruptIndexException, IOException { - boolean success = false; - - if (indexDivisor < 1 && indexDivisor != -1) { - throw new IllegalArgumentException("indexDivisor must be -1 (don't load terms index) or greater than 0: got " + indexDivisor); - } - - try { - directory = dir; - segment = seg; - fieldInfos = fis; - - origEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_EXTENSION), - context), fieldInfos, false); - size = origEnum.size; - - - if (indexDivisor != -1) { - // Load terms index - totalIndexInterval = origEnum.indexInterval * indexDivisor; - - final String indexFileName = IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION); - final SegmentTermEnum indexEnum = new SegmentTermEnum(directory.openInput(indexFileName, - context), fieldInfos, true); - - try { - index = new TermInfosReaderIndex(indexEnum, indexDivisor, dir.fileLength(indexFileName), totalIndexInterval); - indexLength = index.length(); - } finally { - indexEnum.close(); - } - } else { - // Do not load terms index: - totalIndexInterval = -1; - index = null; - indexLength = -1; - } - success = true; - } finally { - // With lock-less commits, it's entirely possible (and - // fine) to hit a FileNotFound exception above. In - // this case, we want to explicitly close any subset - // of things that were opened so that we don't have to - // wait for a GC to do so. - if (!success) { - close(); - } - } - } - - public int getSkipInterval() { - return origEnum.skipInterval; - } - - public int getMaxSkipLevels() { - return origEnum.maxSkipLevels; - } - - void close() throws IOException { - if (origEnum != null) - origEnum.close(); - threadResources.close(); - } - - /** Returns the number of term/value pairs in the set. */ - long size() { - return size; - } - - private ThreadResources getThreadResources() { - ThreadResources resources = threadResources.get(); - if (resources == null) { - resources = new ThreadResources(); - resources.termEnum = terms(); - threadResources.set(resources); - } - return resources; - } - - private static final Comparator legacyComparator = - BytesRef.getUTF8SortedAsUTF16Comparator(); - - private final int compareAsUTF16(Term term1, Term term2) { - if (term1.field().equals(term2.field())) { - return legacyComparator.compare(term1.bytes(), term2.bytes()); - } else { - return term1.field().compareTo(term2.field()); - } - } - - /** Returns the TermInfo for a Term in the set, or null. */ - TermInfo get(Term term) throws IOException { - return get(term, false); - } - - /** Returns the TermInfo for a Term in the set, or null. */ - private TermInfo get(Term term, boolean mustSeekEnum) throws IOException { - if (size == 0) return null; - - ensureIndexIsRead(); - TermInfoAndOrd tiOrd = termsCache.get(new CloneableTerm(term)); - ThreadResources resources = getThreadResources(); - - if (!mustSeekEnum && tiOrd != null) { - return tiOrd; - } - - return seekEnum(resources.termEnum, term, tiOrd, true); - } - - public void cacheCurrentTerm(SegmentTermEnum enumerator) { - termsCache.put(new CloneableTerm(enumerator.term()), - new TermInfoAndOrd(enumerator.termInfo, - enumerator.position)); - } - - TermInfo seekEnum(SegmentTermEnum enumerator, Term term, boolean useCache) throws IOException { - if (useCache) { - return seekEnum(enumerator, term, - termsCache.get(new CloneableTerm(term.deepCopyOf())), - useCache); - } else { - return seekEnum(enumerator, term, null, useCache); - } - } - - TermInfo seekEnum(SegmentTermEnum enumerator, Term term, TermInfoAndOrd tiOrd, boolean useCache) throws IOException { - if (size == 0) { - return null; - } - - // optimize sequential access: first try scanning cached enum w/o seeking - if (enumerator.term() != null // term is at or past current - && ((enumerator.prev() != null && compareAsUTF16(term, enumerator.prev())> 0) - || compareAsUTF16(term, enumerator.term()) >= 0)) { - int enumOffset = (int)(enumerator.position/totalIndexInterval)+1; - if (indexLength == enumOffset // but before end of block - || index.compareTo(term, enumOffset) < 0) { - // no need to seek - - final TermInfo ti; - int numScans = enumerator.scanTo(term); - if (enumerator.term() != null && compareAsUTF16(term, enumerator.term()) == 0) { - ti = enumerator.termInfo; - if (numScans > 1) { - // we only want to put this TermInfo into the cache if - // scanEnum skipped more than one dictionary entry. - // This prevents RangeQueries or WildcardQueries to - // wipe out the cache when they iterate over a large numbers - // of terms in order - if (tiOrd == null) { - if (useCache) { - termsCache.put(new CloneableTerm(term.deepCopyOf()), - new TermInfoAndOrd(ti, enumerator.position)); - } - } else { - assert sameTermInfo(ti, tiOrd, enumerator); - assert (int) enumerator.position == tiOrd.termOrd; - } - } - } else { - ti = null; - } - - return ti; - } - } - - // random-access: must seek - final int indexPos; - if (tiOrd != null) { - indexPos = (int) (tiOrd.termOrd / totalIndexInterval); - } else { - // Must do binary search: - indexPos = index.getIndexOffset(term); - } - - index.seekEnum(enumerator, indexPos); - enumerator.scanTo(term); - final TermInfo ti; - - if (enumerator.term() != null && compareAsUTF16(term, enumerator.term()) == 0) { - ti = enumerator.termInfo; - if (tiOrd == null) { - if (useCache) { - termsCache.put(new CloneableTerm(term.deepCopyOf()), - new TermInfoAndOrd(ti, enumerator.position)); - } - } else { - assert sameTermInfo(ti, tiOrd, enumerator); - assert enumerator.position == tiOrd.termOrd; - } - } else { - ti = null; - } - return ti; - } - - // called only from asserts - private boolean sameTermInfo(TermInfo ti1, TermInfo ti2, SegmentTermEnum enumerator) { - if (ti1.docFreq != ti2.docFreq) { - return false; - } - if (ti1.freqPointer != ti2.freqPointer) { - return false; - } - if (ti1.proxPointer != ti2.proxPointer) { - return false; - } - // skipOffset is only valid when docFreq >= skipInterval: - if (ti1.docFreq >= enumerator.skipInterval && - ti1.skipOffset != ti2.skipOffset) { - return false; - } - return true; - } - - private void ensureIndexIsRead() { - if (index == null) { - throw new IllegalStateException("terms index was not loaded when this reader was created"); - } - } - - /** Returns the position of a Term in the set or -1. */ - long getPosition(Term term) throws IOException { - if (size == 0) return -1; - - ensureIndexIsRead(); - int indexOffset = index.getIndexOffset(term); - - SegmentTermEnum enumerator = getThreadResources().termEnum; - index.seekEnum(enumerator, indexOffset); - - while(compareAsUTF16(term, enumerator.term()) > 0 && enumerator.next()) {} - - if (compareAsUTF16(term, enumerator.term()) == 0) - return enumerator.position; - else - return -1; - } - - /** Returns an enumeration of all the Terms and TermInfos in the set. */ - public SegmentTermEnum terms() { - return origEnum.clone(); - } - - /** Returns an enumeration of terms starting at or after the named term. */ - public SegmentTermEnum terms(Term term) throws IOException { - get(term, true); - return getThreadResources().termEnum.clone(); - } -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java deleted file mode 100644 index 212d8c9..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java +++ /dev/null @@ -1,257 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; - -import org.apache.lucene.index.Term; -import org.apache.lucene.util.BitUtil; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.MathUtil; -import org.apache.lucene.util.PagedBytes.PagedBytesDataInput; -import org.apache.lucene.util.PagedBytes.PagedBytesDataOutput; -import org.apache.lucene.util.PagedBytes; -import org.apache.lucene.util.packed.GrowableWriter; -import org.apache.lucene.util.packed.PackedInts; - -/** - * This stores a monotonically increasing set of pairs in an - * index segment. Pairs are accessed either by Term or by ordinal position the - * set. The Terms and TermInfo are actually serialized and stored into a byte - * array and pointers to the position of each are stored in a int array. - * @deprecated - */ -@Deprecated -class TermInfosReaderIndex { - - private static final int MAX_PAGE_BITS = 18; // 256 KB block - private Term[] fields; - private int totalIndexInterval; - private Comparator comparator = BytesRef.getUTF8SortedAsUTF16Comparator(); - private final PagedBytesDataInput dataInput; - private final PackedInts.Reader indexToDataOffset; - private final int indexSize; - private final int skipInterval; - - /** - * Loads the segment information at segment load time. - * - * @param indexEnum - * the term enum. - * @param indexDivisor - * the index divisor. - * @param tiiFileLength - * the size of the tii file, used to approximate the size of the - * buffer. - * @param totalIndexInterval - * the total index interval. - */ - TermInfosReaderIndex(SegmentTermEnum indexEnum, int indexDivisor, long tiiFileLength, int totalIndexInterval) throws IOException { - this.totalIndexInterval = totalIndexInterval; - indexSize = 1 + ((int) indexEnum.size - 1) / indexDivisor; - skipInterval = indexEnum.skipInterval; - // this is only an inital size, it will be GCed once the build is complete - long initialSize = (long) (tiiFileLength * 1.5) / indexDivisor; - PagedBytes dataPagedBytes = new PagedBytes(estimatePageBits(initialSize)); - PagedBytesDataOutput dataOutput = dataPagedBytes.getDataOutput(); - - final int bitEstimate = 1+MathUtil.log(tiiFileLength, 2); - GrowableWriter indexToTerms = new GrowableWriter(bitEstimate, indexSize, PackedInts.DEFAULT); - - String currentField = null; - List fieldStrs = new ArrayList(); - int fieldCounter = -1; - for (int i = 0; indexEnum.next(); i++) { - Term term = indexEnum.term(); - if (currentField == null || !currentField.equals(term.field())) { - currentField = term.field(); - fieldStrs.add(currentField); - fieldCounter++; - } - TermInfo termInfo = indexEnum.termInfo(); - indexToTerms.set(i, dataOutput.getPosition()); - dataOutput.writeVInt(fieldCounter); - dataOutput.writeString(term.text()); - dataOutput.writeVInt(termInfo.docFreq); - if (termInfo.docFreq >= skipInterval) { - dataOutput.writeVInt(termInfo.skipOffset); - } - dataOutput.writeVLong(termInfo.freqPointer); - dataOutput.writeVLong(termInfo.proxPointer); - dataOutput.writeVLong(indexEnum.indexPointer); - for (int j = 1; j < indexDivisor; j++) { - if (!indexEnum.next()) { - break; - } - } - } - - fields = new Term[fieldStrs.size()]; - for (int i = 0; i < fields.length; i++) { - fields[i] = new Term(fieldStrs.get(i)); - } - - dataPagedBytes.freeze(true); - dataInput = dataPagedBytes.getDataInput(); - indexToDataOffset = indexToTerms.getMutable(); - } - - private static int estimatePageBits(long estSize) { - return Math.max(Math.min(64 - BitUtil.nlz(estSize), MAX_PAGE_BITS), 4); - } - - void seekEnum(SegmentTermEnum enumerator, int indexOffset) throws IOException { - PagedBytesDataInput input = dataInput.clone(); - - input.setPosition(indexToDataOffset.get(indexOffset)); - - // read the term - int fieldId = input.readVInt(); - Term field = fields[fieldId]; - Term term = new Term(field.field(), input.readString()); - - // read the terminfo - TermInfo termInfo = new TermInfo(); - termInfo.docFreq = input.readVInt(); - if (termInfo.docFreq >= skipInterval) { - termInfo.skipOffset = input.readVInt(); - } else { - termInfo.skipOffset = 0; - } - termInfo.freqPointer = input.readVLong(); - termInfo.proxPointer = input.readVLong(); - - long pointer = input.readVLong(); - - // perform the seek - enumerator.seek(pointer, ((long) indexOffset * totalIndexInterval) - 1, term, termInfo); - } - - /** - * Binary search for the given term. - * - * @param term - * the term to locate. - * @throws IOException - */ - int getIndexOffset(Term term) throws IOException { - int lo = 0; - int hi = indexSize - 1; - PagedBytesDataInput input = dataInput.clone(); - BytesRef scratch = new BytesRef(); - while (hi >= lo) { - int mid = (lo + hi) >>> 1; - int delta = compareTo(term, mid, input, scratch); - if (delta < 0) - hi = mid - 1; - else if (delta > 0) - lo = mid + 1; - else - return mid; - } - return hi; - } - - /** - * Gets the term at the given position. For testing. - * - * @param termIndex - * the position to read the term from the index. - * @return the term. - * @throws IOException - */ - Term getTerm(int termIndex) throws IOException { - PagedBytesDataInput input = dataInput.clone(); - input.setPosition(indexToDataOffset.get(termIndex)); - - // read the term - int fieldId = input.readVInt(); - Term field = fields[fieldId]; - return new Term(field.field(), input.readString()); - } - - /** - * Returns the number of terms. - * - * @return int. - */ - int length() { - return indexSize; - } - - /** - * The compares the given term against the term in the index specified by the - * term index. ie It returns negative N when term is less than index term; - * - * @param term - * the given term. - * @param termIndex - * the index of the of term to compare. - * @return int. - * @throws IOException - */ - int compareTo(Term term, int termIndex) throws IOException { - return compareTo(term, termIndex, dataInput.clone(), new BytesRef()); - } - - /** - * Compare the fields of the terms first, and if not equals return from - * compare. If equal compare terms. - * - * @param term - * the term to compare. - * @param termIndex - * the position of the term in the input to compare - * @param input - * the input buffer. - * @return int. - * @throws IOException - */ - private int compareTo(Term term, int termIndex, PagedBytesDataInput input, BytesRef reuse) throws IOException { - // if term field does not equal mid's field index, then compare fields - // else if they are equal, compare term's string values... - int c = compareField(term, termIndex, input); - if (c == 0) { - reuse.length = input.readVInt(); - reuse.grow(reuse.length); - input.readBytes(reuse.bytes, 0, reuse.length); - return comparator.compare(term.bytes(), reuse); - } - return c; - } - - /** - * Compares the fields before checking the text of the terms. - * - * @param term - * the given term. - * @param termIndex - * the term that exists in the data block. - * @param input - * the data block. - * @return int. - * @throws IOException - */ - private int compareField(Term term, int termIndex, PagedBytesDataInput input) throws IOException { - input.setPosition(indexToDataOffset.get(termIndex)); - return term.field().compareTo(fields[input.readVInt()].field()); - } -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/package.html b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/package.html deleted file mode 100644 index 747c64a..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/package.html +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - - -Codec to support Lucene 3.x indexes (readonly) - - diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/BitVector.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/BitVector.java index b2fa3ec..f45df60 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/BitVector.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/BitVector.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,12 +20,12 @@ package org.apache.lucene.codecs.lucene40; import java.io.IOException; import java.util.Arrays; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.MutableBits; /** Optimized implementation of a vector of bits. This is more-or-less like diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java index 00d0257..ee5f57a 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesConsumer.java index 7d3b276..6f00416 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesConsumer.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -39,7 +39,7 @@ public class Lucene40DocValuesConsumer extends DocValuesWriterBase { public final static String DOC_VALUES_SEGMENT_SUFFIX = "dv"; - public Lucene40DocValuesConsumer(PerDocWriteState state, String segmentSuffix) throws IOException { + public Lucene40DocValuesConsumer(PerDocWriteState state, String segmentSuffix) { super(state); this.segmentSuffix = segmentSuffix; mainDirectory = state.directory; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesFormat.java index 45f1874..1a08c44 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,6 +19,7 @@ package org.apache.lucene.codecs.lucene40; import java.io.IOException; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PerDocConsumer; import org.apache.lucene.codecs.PerDocProducer; @@ -28,7 +29,6 @@ import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.store.CompoundFileDirectory; // javadocs import org.apache.lucene.store.DataOutput; // javadocs -import org.apache.lucene.util.CodecUtil; // javadocs import org.apache.lucene.util.packed.PackedInts; // javadocs /** diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesProducer.java index 7851f94..efd269b 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesProducer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesProducer.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java index ee1e276..e41baa0 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,13 +19,13 @@ package org.apache.lucene.codecs.lucene40; import java.io.IOException; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.codecs.FieldInfosReader; import org.apache.lucene.codecs.FieldInfosWriter; import org.apache.lucene.index.DocValues; // javadoc import org.apache.lucene.index.DocValues.Type; // javadoc import org.apache.lucene.store.DataOutput; // javadoc -import org.apache.lucene.util.CodecUtil; // javadoc /** * Lucene 4.0 Field Infos format. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java index 11a9927..40b97ed 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.util.Collections; import java.util.Map; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldInfosReader; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.FieldInfo; @@ -14,9 +15,8 @@ import org.apache.lucene.index.DocValues; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.CodecUtil; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -63,7 +63,9 @@ public class Lucene40FieldInfosReader extends FieldInfosReader { boolean omitNorms = (bits & Lucene40FieldInfosWriter.OMIT_NORMS) != 0; boolean storePayloads = (bits & Lucene40FieldInfosWriter.STORE_PAYLOADS) != 0; final IndexOptions indexOptions; - if ((bits & Lucene40FieldInfosWriter.OMIT_TERM_FREQ_AND_POSITIONS) != 0) { + if (!isIndexed) { + indexOptions = null; + } else if ((bits & Lucene40FieldInfosWriter.OMIT_TERM_FREQ_AND_POSITIONS) != 0) { indexOptions = IndexOptions.DOCS_ONLY; } else if ((bits & Lucene40FieldInfosWriter.OMIT_POSITIONS) != 0) { indexOptions = IndexOptions.DOCS_AND_FREQS; @@ -76,7 +78,7 @@ public class Lucene40FieldInfosReader extends FieldInfosReader { // LUCENE-3027: past indices were able to write // storePayloads=true when omitTFAP is also true, // which is invalid. We correct that, here: - if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { + if (isIndexed && indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { storePayloads = false; } // DV Types are packed in one byte diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java index f0a3cbe..e30fcee 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,6 +18,7 @@ package org.apache.lucene.codecs.lucene40; */ import java.io.IOException; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldInfosWriter; import org.apache.lucene.index.DocValues.Type; import org.apache.lucene.index.FieldInfo; @@ -27,7 +28,6 @@ import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.CodecUtil; /** * Lucene 4.0 FieldInfos writer. @@ -61,18 +61,20 @@ public class Lucene40FieldInfosWriter extends FieldInfosWriter { output.writeVInt(infos.size()); for (FieldInfo fi : infos) { IndexOptions indexOptions = fi.getIndexOptions(); - assert indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !fi.hasPayloads(); byte bits = 0x0; - if (fi.isIndexed()) bits |= IS_INDEXED; if (fi.hasVectors()) bits |= STORE_TERMVECTOR; if (fi.omitsNorms()) bits |= OMIT_NORMS; if (fi.hasPayloads()) bits |= STORE_PAYLOADS; - if (indexOptions == IndexOptions.DOCS_ONLY) { - bits |= OMIT_TERM_FREQ_AND_POSITIONS; - } else if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) { - bits |= STORE_OFFSETS_IN_POSTINGS; - } else if (indexOptions == IndexOptions.DOCS_AND_FREQS) { - bits |= OMIT_POSITIONS; + if (fi.isIndexed()) { + bits |= IS_INDEXED; + assert indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !fi.hasPayloads(); + if (indexOptions == IndexOptions.DOCS_ONLY) { + bits |= OMIT_TERM_FREQ_AND_POSITIONS; + } else if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) { + bits |= STORE_OFFSETS_IN_POSTINGS; + } else if (indexOptions == IndexOptions.DOCS_AND_FREQS) { + bits |= OMIT_POSITIONS; + } } output.writeString(fi.name); output.writeVInt(fi.number); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40LiveDocsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40LiveDocsFormat.java index 6e424bf..faaaf3f 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40LiveDocsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40LiveDocsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,6 +20,7 @@ package org.apache.lucene.codecs.lucene40; import java.io.IOException; import java.util.Collection; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.LiveDocsFormat; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfoPerCommit; @@ -27,7 +28,6 @@ import org.apache.lucene.store.DataOutput; // javadocs import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.CodecUtil; // javadocs import org.apache.lucene.util.MutableBits; /** diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java index 08d0645..585c190 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -96,7 +96,7 @@ public class Lucene40NormsFormat extends NormsFormat { public static class Lucene40NormsDocValuesConsumer extends Lucene40DocValuesConsumer { public Lucene40NormsDocValuesConsumer(PerDocWriteState state, - String segmentSuffix) throws IOException { + String segmentSuffix) { super(state, segmentSuffix); } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsBaseFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsBaseFormat.java index 430f0b0..1d2b6ee 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsBaseFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsBaseFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsFormat.java index a6386b6..26f7f02 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,6 +21,7 @@ import java.io.IOException; import org.apache.lucene.codecs.BlockTreeTermsReader; import org.apache.lucene.codecs.BlockTreeTermsWriter; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; import org.apache.lucene.codecs.PostingsFormat; @@ -32,7 +33,6 @@ import org.apache.lucene.index.FieldInfos; // javadocs import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.store.DataOutput; // javadocs -import org.apache.lucene.util.CodecUtil; // javadocs import org.apache.lucene.util.fst.FST; // javadocs /** @@ -159,7 +159,8 @@ import org.apache.lucene.util.fst.FST; // javadocs * with the frequency of the term in that document (except when frequencies are * omitted: {@link IndexOptions#DOCS_ONLY}).

*
    - *
  • FreqFile (.frq) --> <TermFreqs, SkipData> TermCount
    • + *
  • FreqFile (.frq) --> Header, <TermFreqs, SkipData> TermCount
    • + *
  • Header --> {@link CodecUtil#writeHeader CodecHeader}
    • *
  • TermFreqs --> <TermFreq> DocFreq
    • *
  • TermFreq --> DocDelta[, Freq?]
    • *
  • SkipData --> <<SkipLevelLength, SkipLevel> @@ -232,7 +233,8 @@ import org.apache.lucene.util.fst.FST; // javadocs * anything into this file, and if all fields in the index omit positional data * then the .prx file will not exist.

    *
      - *
    • ProxFile (.prx) --> <TermPositions> TermCount
      • + *
    • ProxFile (.prx) --> Header, <TermPositions> TermCount
      • + *
    • Header --> {@link CodecUtil#writeHeader CodecHeader}
      • *
    • TermPositions --> <Positions> DocFreq
      • *
    • Positions --> <PositionDelta,PayloadLength?,OffsetDelta?,OffsetLength?,PayloadData?> Freq
      • *
    • PositionDelta,OffsetDelta,OffsetLength,PayloadLength --> {@link DataOutput#writeVInt VInt}
      • @@ -254,8 +256,8 @@ import org.apache.lucene.util.fst.FST; // javadocs *

      4, 5, 4

      *

      PayloadData is metadata associated with the current term position. If * PayloadLength is stored at the current position, then it indicates the length - * of this Payload. If PayloadLength is not stored, then this Payload has the same - * length as the Payload at the previous position.

      + * of this payload. If PayloadLength is not stored, then this payload has the same + * length as the payload at the previous position.

      *

      OffsetDelta/2 is the difference between this position's startOffset from the * previous occurrence (or zero, if this is the first occurrence in this document). * If OffsetDelta is odd, then the length (endOffset-startOffset) differs from the diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java index a7624dc..5898d30 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,6 +20,7 @@ package org.apache.lucene.codecs.lucene40; import java.io.IOException; import org.apache.lucene.codecs.BlockTermState; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.PostingsReaderBase; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; @@ -36,7 +37,7 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CodecUtil; +import org.apache.lucene.util.IOUtils; /** * Concrete class that reads the 4.0 frq/prox @@ -58,29 +59,35 @@ public class Lucene40PostingsReader extends PostingsReaderBase { // private String segment; public Lucene40PostingsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo segmentInfo, IOContext ioContext, String segmentSuffix) throws IOException { - freqIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene40PostingsFormat.FREQ_EXTENSION), + boolean success = false; + IndexInput freqIn = null; + IndexInput proxIn = null; + try { + freqIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene40PostingsFormat.FREQ_EXTENSION), ioContext); - // TODO: hasProx should (somehow!) become codec private, - // but it's tricky because 1) FIS.hasProx is global (it - // could be all fields that have prox are written by a - // different codec), 2) the field may have had prox in - // the past but all docs w/ that field were deleted. - // Really we'd need to init prxOut lazily on write, and - // then somewhere record that we actually wrote it so we - // know whether to open on read: - if (fieldInfos.hasProx()) { - boolean success = false; - try { + CodecUtil.checkHeader(freqIn, Lucene40PostingsWriter.FRQ_CODEC, Lucene40PostingsWriter.VERSION_START,Lucene40PostingsWriter.VERSION_START); + // TODO: hasProx should (somehow!) become codec private, + // but it's tricky because 1) FIS.hasProx is global (it + // could be all fields that have prox are written by a + // different codec), 2) the field may have had prox in + // the past but all docs w/ that field were deleted. + // Really we'd need to init prxOut lazily on write, and + // then somewhere record that we actually wrote it so we + // know whether to open on read: + if (fieldInfos.hasProx()) { proxIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene40PostingsFormat.PROX_EXTENSION), - ioContext); - success = true; - } finally { - if (!success) { - freqIn.close(); - } + ioContext); + CodecUtil.checkHeader(proxIn, Lucene40PostingsWriter.PRX_CODEC, Lucene40PostingsWriter.VERSION_START,Lucene40PostingsWriter.VERSION_START); + } else { + proxIn = null; + } + this.freqIn = freqIn; + this.proxIn = proxIn; + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(freqIn, proxIn); } - } else { - proxIn = null; } } @@ -88,7 +95,7 @@ public class Lucene40PostingsReader extends PostingsReaderBase { public void init(IndexInput termsIn) throws IOException { // Make sure we are talking to the matching past writer - CodecUtil.checkHeader(termsIn, Lucene40PostingsWriter.CODEC, + CodecUtil.checkHeader(termsIn, Lucene40PostingsWriter.TERMS_CODEC, Lucene40PostingsWriter.VERSION_START, Lucene40PostingsWriter.VERSION_START); skipInterval = termsIn.readInt(); @@ -212,9 +219,7 @@ public class Lucene40PostingsReader extends PostingsReaderBase { @Override public DocsEnum docs(FieldInfo fieldInfo, BlockTermState termState, Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException { - if (needsFreqs && fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY) { - return null; - } else if (canReuse(reuse, liveDocs)) { + if (canReuse(reuse, liveDocs)) { // if (DEBUG) System.out.println("SPR.docs ts=" + termState); return ((SegmentDocsEnumBase) reuse).reset(fieldInfo, (StandardTermState)termState); } @@ -249,9 +254,6 @@ public class Lucene40PostingsReader extends PostingsReaderBase { throws IOException { boolean hasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; - if (needsOffsets && !hasOffsets) { - return null; // not available - } // TODO: refactor if (fieldInfo.hasPayloads() || hasOffsets) { @@ -317,7 +319,7 @@ public class Lucene40PostingsReader extends PostingsReaderBase { protected boolean skipped; protected final Bits liveDocs; - SegmentDocsEnumBase(IndexInput startFreqIn, Bits liveDocs) throws IOException { + SegmentDocsEnumBase(IndexInput startFreqIn, Bits liveDocs) { this.startFreqIn = startFreqIn; this.freqIn = (IndexInput)startFreqIn.clone(); this.liveDocs = liveDocs; @@ -351,7 +353,7 @@ public class Lucene40PostingsReader extends PostingsReaderBase { } @Override - public final int freq() throws IOException { + public final int freq() { assert !indexOmitsTF; return freq; } @@ -497,7 +499,7 @@ public class Lucene40PostingsReader extends PostingsReaderBase { private final class AllDocsSegmentDocsEnum extends SegmentDocsEnumBase { - AllDocsSegmentDocsEnum(IndexInput startFreqIn) throws IOException { + AllDocsSegmentDocsEnum(IndexInput startFreqIn) { super(startFreqIn, null); assert liveDocs == null; } @@ -574,7 +576,7 @@ public class Lucene40PostingsReader extends PostingsReaderBase { private final class LiveDocsSegmentDocsEnum extends SegmentDocsEnumBase { - LiveDocsSegmentDocsEnum(IndexInput startFreqIn, Bits liveDocs) throws IOException { + LiveDocsSegmentDocsEnum(IndexInput startFreqIn, Bits liveDocs) { super(startFreqIn, liveDocs); assert liveDocs != null; } @@ -694,7 +696,7 @@ public class Lucene40PostingsReader extends PostingsReaderBase { Lucene40SkipListReader skipper; private long lazyProxPointer; - public SegmentDocsAndPositionsEnum(IndexInput freqIn, IndexInput proxIn) throws IOException { + public SegmentDocsAndPositionsEnum(IndexInput freqIn, IndexInput proxIn) { startFreqIn = freqIn; this.freqIn = (IndexInput) freqIn.clone(); this.proxIn = (IndexInput) proxIn.clone(); @@ -770,7 +772,7 @@ public class Lucene40PostingsReader extends PostingsReaderBase { } @Override - public int freq() throws IOException { + public int freq() { return freq; } @@ -851,12 +853,12 @@ public class Lucene40PostingsReader extends PostingsReaderBase { } @Override - public int startOffset() throws IOException { + public int startOffset() { return -1; } @Override - public int endOffset() throws IOException { + public int endOffset() { return -1; } @@ -907,7 +909,7 @@ public class Lucene40PostingsReader extends PostingsReaderBase { int offsetLength; int startOffset; - public SegmentFullPositionsEnum(IndexInput freqIn, IndexInput proxIn) throws IOException { + public SegmentFullPositionsEnum(IndexInput freqIn, IndexInput proxIn) { startFreqIn = freqIn; this.freqIn = (IndexInput) freqIn.clone(); this.proxIn = (IndexInput) proxIn.clone(); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsWriter.java index f103897..91c135c 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -24,6 +24,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.PostingsWriterBase; import org.apache.lucene.codecs.TermStats; import org.apache.lucene.index.CorruptIndexException; @@ -35,7 +36,6 @@ import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RAMOutputStream; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.IOUtils; /** @@ -45,7 +45,9 @@ import org.apache.lucene.util.IOUtils; * @lucene.experimental */ public final class Lucene40PostingsWriter extends PostingsWriterBase { - final static String CODEC = "Lucene40PostingsWriter"; + final static String TERMS_CODEC = "Lucene40PostingsWriterTerms"; + final static String FRQ_CODEC = "Lucene40PostingsWriterFrq"; + final static String PRX_CODEC = "Lucene40PostingsWriterPrx"; //private static boolean DEBUG = BlockTreeTermsWriter.DEBUG; @@ -102,7 +104,9 @@ public final class Lucene40PostingsWriter extends PostingsWriterBase { String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, Lucene40PostingsFormat.FREQ_EXTENSION); freqOut = state.directory.createOutput(fileName, state.context); boolean success = false; + IndexOutput proxOut = null; try { + CodecUtil.writeHeader(freqOut, FRQ_CODEC, VERSION_CURRENT); // TODO: this is a best effort, if one of these fields has no postings // then we make an empty prx file, same as if we are wrapped in // per-field postingsformat. maybe... we shouldn't @@ -112,14 +116,16 @@ public final class Lucene40PostingsWriter extends PostingsWriterBase { // prox file fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, Lucene40PostingsFormat.PROX_EXTENSION); proxOut = state.directory.createOutput(fileName, state.context); + CodecUtil.writeHeader(proxOut, PRX_CODEC, VERSION_CURRENT); } else { // Every field omits TF so we will write no prox file proxOut = null; } + this.proxOut = proxOut; success = true; } finally { if (!success) { - IOUtils.closeWhileHandlingException(freqOut); + IOUtils.closeWhileHandlingException(freqOut, proxOut); } } @@ -135,7 +141,7 @@ public final class Lucene40PostingsWriter extends PostingsWriterBase { @Override public void start(IndexOutput termsOut) throws IOException { this.termsOut = termsOut; - CodecUtil.writeHeader(termsOut, CODEC, VERSION_CURRENT); + CodecUtil.writeHeader(termsOut, TERMS_CODEC, VERSION_CURRENT); termsOut.writeInt(skipInterval); // write skipInterval termsOut.writeInt(maxSkipLevels); // write maxSkipLevels termsOut.writeInt(skipMinimum); // write skipMinimum @@ -246,6 +252,7 @@ public final class Lucene40PostingsWriter extends PostingsWriterBase { // and the numbers aren't that much smaller anyways. int offsetDelta = startOffset - lastOffset; int offsetLength = endOffset - startOffset; + assert offsetDelta >= 0 && offsetLength >= 0 : "startOffset=" + startOffset + ",lastOffset=" + lastOffset + ",endOffset=" + endOffset; if (offsetLength != lastOffsetLength) { proxOut.writeVInt(offsetDelta << 1 | 1); proxOut.writeVInt(offsetLength); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java index 572cb03..d70b05d 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,13 +17,13 @@ package org.apache.lucene.codecs.lucene40; * limitations under the License. */ +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.SegmentInfoFormat; import org.apache.lucene.codecs.SegmentInfoReader; import org.apache.lucene.codecs.SegmentInfoWriter; import org.apache.lucene.index.IndexWriter; // javadocs import org.apache.lucene.index.SegmentInfos; // javadocs import org.apache.lucene.store.DataOutput; // javadocs -import org.apache.lucene.util.CodecUtil; // javadocs /** * Lucene 4.0 Segment info format. @@ -72,12 +72,12 @@ public class Lucene40SegmentInfoFormat extends SegmentInfoFormat { private final SegmentInfoWriter writer = new Lucene40SegmentInfoWriter(); @Override - public SegmentInfoReader getSegmentInfosReader() { + public SegmentInfoReader getSegmentInfoReader() { return reader; } @Override - public SegmentInfoWriter getSegmentInfosWriter() { + public SegmentInfoWriter getSegmentInfoWriter() { return writer; } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java index 0d501c3..0aeb8bf 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,13 +22,13 @@ import java.util.Collections; import java.util.Map; import java.util.Set; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.SegmentInfoReader; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.IOUtils; /** diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java index dae3f21..18def28 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,6 +19,7 @@ package org.apache.lucene.codecs.lucene40; import java.io.IOException; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.SegmentInfoWriter; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.IndexFileNames; @@ -26,7 +27,6 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.IOUtils; /** diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SkipListReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SkipListReader.java index c082ec3..8f087ca 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SkipListReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SkipListReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SkipListWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SkipListWriter.java index 5eb2d88..e2af183 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SkipListWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SkipListWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsFormat.java index 8f79e2b..2ebc99f 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,6 +19,7 @@ package org.apache.lucene.codecs.lucene40; import java.io.IOException; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.StoredFieldsReader; import org.apache.lucene.codecs.StoredFieldsWriter; @@ -27,7 +28,6 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.DataOutput; // javadocs import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; -import org.apache.lucene.util.CodecUtil; /** * Lucene 4.0 Stored Fields Format. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java index cc3691c..3cb24de 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,6 +19,7 @@ package org.apache.lucene.codecs.lucene40; import java.io.IOException; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.StoredFieldsReader; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.FieldInfo; @@ -30,7 +31,6 @@ import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.IOUtils; import java.io.Closeable; @@ -136,7 +136,7 @@ public final class Lucene40StoredFieldsReader extends StoredFieldsReader impleme indexStream.seek(HEADER_LENGTH_IDX + docID * 8L); } - public final void visitDocument(int n, StoredFieldVisitor visitor) throws CorruptIndexException, IOException { + public final void visitDocument(int n, StoredFieldVisitor visitor) throws IOException { seekIndex(n); fieldsStream.seek(indexStream.readLong()); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java index 73be17e..ed699cd 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java @@ -18,15 +18,15 @@ package org.apache.lucene.codecs.lucene40; import java.io.IOException; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.StoredFieldsReader; import org.apache.lucene.codecs.StoredFieldsWriter; import org.apache.lucene.document.Document; -import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.MergePolicy.MergeAbortedException; import org.apache.lucene.index.MergeState; import org.apache.lucene.index.SegmentReader; import org.apache.lucene.store.Directory; @@ -35,7 +35,6 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.IOUtils; /** @@ -209,7 +208,7 @@ public final class Lucene40StoredFieldsWriter extends StoredFieldsWriter { } @Override - public void finish(FieldInfos fis, int numDocs) throws IOException { + public void finish(FieldInfos fis, int numDocs) { if (HEADER_LENGTH_IDX+((long) numDocs)*8 != indexStream.getFilePointer()) // This is most likely a bug in Sun JRE 1.6.0_04/_05; // we detect that the bug has struck, here, and @@ -226,7 +225,7 @@ public final class Lucene40StoredFieldsWriter extends StoredFieldsWriter { int rawDocLengths[] = new int[MAX_RAW_MERGE_DOCS]; int idx = 0; - for (MergeState.IndexReaderAndLiveDocs reader : mergeState.readers) { + for (AtomicReader reader : mergeState.readers) { final SegmentReader matchingSegmentReader = mergeState.matchingSegmentReaders[idx++]; Lucene40StoredFieldsReader matchingFieldsReader = null; if (matchingSegmentReader != null) { @@ -237,7 +236,7 @@ public final class Lucene40StoredFieldsWriter extends StoredFieldsWriter { } } - if (reader.liveDocs != null) { + if (reader.getLiveDocs() != null) { docCount += copyFieldsWithDeletions(mergeState, reader, matchingFieldsReader, rawDocLengths); } else { @@ -253,12 +252,12 @@ public final class Lucene40StoredFieldsWriter extends StoredFieldsWriter { when merging stored fields */ private final static int MAX_RAW_MERGE_DOCS = 4192; - private int copyFieldsWithDeletions(MergeState mergeState, final MergeState.IndexReaderAndLiveDocs reader, + private int copyFieldsWithDeletions(MergeState mergeState, final AtomicReader reader, final Lucene40StoredFieldsReader matchingFieldsReader, int rawDocLengths[]) - throws IOException, MergeAbortedException, CorruptIndexException { + throws IOException { int docCount = 0; - final int maxDoc = reader.reader.maxDoc(); - final Bits liveDocs = reader.liveDocs; + final int maxDoc = reader.maxDoc(); + final Bits liveDocs = reader.getLiveDocs(); assert liveDocs != null; if (matchingFieldsReader != null) { // We can bulk-copy because the fieldInfos are "congruent" @@ -298,7 +297,7 @@ public final class Lucene40StoredFieldsWriter extends StoredFieldsWriter { // on the fly? // NOTE: it's very important to first assign to doc then pass it to // fieldsWriter.addDocument; see LUCENE-1282 - Document doc = reader.reader.document(j); + Document doc = reader.document(j); addDocument(doc, mergeState.fieldInfos); docCount++; mergeState.checkAbort.work(300); @@ -307,10 +306,10 @@ public final class Lucene40StoredFieldsWriter extends StoredFieldsWriter { return docCount; } - private int copyFieldsNoDeletions(MergeState mergeState, final MergeState.IndexReaderAndLiveDocs reader, + private int copyFieldsNoDeletions(MergeState mergeState, final AtomicReader reader, final Lucene40StoredFieldsReader matchingFieldsReader, int rawDocLengths[]) - throws IOException, MergeAbortedException, CorruptIndexException { - final int maxDoc = reader.reader.maxDoc(); + throws IOException { + final int maxDoc = reader.maxDoc(); int docCount = 0; if (matchingFieldsReader != null) { // We can bulk-copy because the fieldInfos are "congruent" @@ -325,7 +324,7 @@ public final class Lucene40StoredFieldsWriter extends StoredFieldsWriter { for (; docCount < maxDoc; docCount++) { // NOTE: it's very important to first assign to doc then pass it to // fieldsWriter.addDocument; see LUCENE-1282 - Document doc = reader.reader.document(docCount); + Document doc = reader.document(docCount); addDocument(doc, mergeState.fieldInfos); mergeState.checkAbort.work(300); } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsFormat.java index 69b49fa..89c46e4 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,6 +19,7 @@ package org.apache.lucene.codecs.lucene40; import java.io.IOException; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.TermVectorsFormat; import org.apache.lucene.codecs.TermVectorsReader; import org.apache.lucene.codecs.TermVectorsWriter; @@ -27,7 +28,6 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.DataOutput; // javadocs import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; -import org.apache.lucene.util.CodecUtil; /** * Lucene 4.0 Term Vectors format. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java index 087acbc..0bd9b4d 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,8 +23,8 @@ import java.util.Comparator; import java.util.HashMap; import java.util.Map; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.TermVectorsReader; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; @@ -40,7 +40,6 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.IOUtils; /** @@ -96,7 +95,7 @@ public class Lucene40TermVectorsReader extends TermVectorsReader { } public Lucene40TermVectorsReader(Directory d, SegmentInfo si, FieldInfos fieldInfos, IOContext context) - throws CorruptIndexException, IOException { + throws IOException { final String segment = si.name; final int size = si.getDocCount(); @@ -252,7 +251,7 @@ public class Lucene40TermVectorsReader extends TermVectorsReader { private int fieldUpto; @Override - public String next() throws IOException { + public String next() { if (fieldNumbers != null && fieldUpto < fieldNumbers.length) { return fieldInfos.fieldInfo(fieldNumbers[fieldUpto++]).name; } else { @@ -365,7 +364,7 @@ public class Lucene40TermVectorsReader extends TermVectorsReader { private int[] endOffsets; // NOTE: tvf is pre-positioned by caller - public TVTermsEnum() throws IOException { + public TVTermsEnum() { this.origTVF = Lucene40TermVectorsReader.this.tvf; tvf = (IndexInput) origTVF.clone(); } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java index 5cc2d1b..67b32c2 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,13 +20,14 @@ package org.apache.lucene.codecs.lucene40; import java.io.IOException; import java.util.Comparator; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.TermVectorsReader; import org.apache.lucene.codecs.TermVectorsWriter; +import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.MergePolicy.MergeAbortedException; import org.apache.lucene.index.MergeState; import org.apache.lucene.index.SegmentReader; import org.apache.lucene.store.DataInput; @@ -36,7 +37,6 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.StringHelper; @@ -255,7 +255,7 @@ public final class Lucene40TermVectorsWriter extends TermVectorsWriter { int idx = 0; int numDocs = 0; - for (final MergeState.IndexReaderAndLiveDocs reader : mergeState.readers) { + for (final AtomicReader reader : mergeState.readers) { final SegmentReader matchingSegmentReader = mergeState.matchingSegmentReaders[idx++]; Lucene40TermVectorsReader matchingVectorsReader = null; if (matchingSegmentReader != null) { @@ -265,7 +265,7 @@ public final class Lucene40TermVectorsWriter extends TermVectorsWriter { matchingVectorsReader = (Lucene40TermVectorsReader) vectorsReader; } } - if (reader.liveDocs != null) { + if (reader.getLiveDocs() != null) { numDocs += copyVectorsWithDeletions(mergeState, matchingVectorsReader, reader, rawDocLengths, rawDocLengths2); } else { numDocs += copyVectorsNoDeletions(mergeState, matchingVectorsReader, reader, rawDocLengths, rawDocLengths2); @@ -281,12 +281,12 @@ public final class Lucene40TermVectorsWriter extends TermVectorsWriter { private int copyVectorsWithDeletions(MergeState mergeState, final Lucene40TermVectorsReader matchingVectorsReader, - final MergeState.IndexReaderAndLiveDocs reader, + final AtomicReader reader, int rawDocLengths[], int rawDocLengths2[]) - throws IOException, MergeAbortedException { - final int maxDoc = reader.reader.maxDoc(); - final Bits liveDocs = reader.liveDocs; + throws IOException { + final int maxDoc = reader.maxDoc(); + final Bits liveDocs = reader.getLiveDocs(); int totalNumDocs = 0; if (matchingVectorsReader != null) { // We can bulk-copy because the fieldInfos are "congruent" @@ -323,7 +323,7 @@ public final class Lucene40TermVectorsWriter extends TermVectorsWriter { // NOTE: it's very important to first assign to vectors then pass it to // termVectorsWriter.addAllDocVectors; see LUCENE-1282 - Fields vectors = reader.reader.getTermVectors(docNum); + Fields vectors = reader.getTermVectors(docNum); addAllDocVectors(vectors, mergeState.fieldInfos); totalNumDocs++; mergeState.checkAbort.work(300); @@ -334,11 +334,11 @@ public final class Lucene40TermVectorsWriter extends TermVectorsWriter { private int copyVectorsNoDeletions(MergeState mergeState, final Lucene40TermVectorsReader matchingVectorsReader, - final MergeState.IndexReaderAndLiveDocs reader, + final AtomicReader reader, int rawDocLengths[], int rawDocLengths2[]) - throws IOException, MergeAbortedException { - final int maxDoc = reader.reader.maxDoc(); + throws IOException { + final int maxDoc = reader.maxDoc(); if (matchingVectorsReader != null) { // We can bulk-copy because the fieldInfos are "congruent" int docCount = 0; @@ -353,7 +353,7 @@ public final class Lucene40TermVectorsWriter extends TermVectorsWriter { for (int docNum = 0; docNum < maxDoc; docNum++) { // NOTE: it's very important to first assign to vectors then pass it to // termVectorsWriter.addAllDocVectors; see LUCENE-1282 - Fields vectors = reader.reader.getTermVectors(docNum); + Fields vectors = reader.getTermVectors(docNum); addAllDocVectors(vectors, mergeState.fieldInfos); mergeState.checkAbort.work(300); } @@ -362,7 +362,7 @@ public final class Lucene40TermVectorsWriter extends TermVectorsWriter { } @Override - public void finish(FieldInfos fis, int numDocs) throws IOException { + public void finish(FieldInfos fis, int numDocs) { if (HEADER_LENGTH_INDEX+((long) numDocs)*16 != tvx.getFilePointer()) // This is most likely a bug in Sun JRE 1.6.0_04/_05; // we detect that the bug has struck, here, and @@ -382,7 +382,7 @@ public final class Lucene40TermVectorsWriter extends TermVectorsWriter { } @Override - public Comparator getComparator() throws IOException { + public Comparator getComparator() { return BytesRef.getUTF8SortedAsUnicodeComparator(); } } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java index 1032690..cbec979 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40.values; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.Comparator; import java.util.concurrent.atomic.AtomicLong; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.index.DocValues.SortedSource; import org.apache.lucene.index.DocValues.Source; @@ -41,7 +42,6 @@ import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefHash.TrackingDirectBytesStartArray; import org.apache.lucene.util.BytesRefHash; -import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.Counter; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.PagedBytes; @@ -121,14 +121,11 @@ public final class Bytes { * {@link Type#BYTES_VAR_SORTED}. * @param context I/O Context * @return a new {@link Writer} instance - * @throws IOException - * if the files for the writer can not be created. * @see PackedInts#getReader(org.apache.lucene.store.DataInput) */ public static DocValuesConsumer getWriter(Directory dir, String id, Mode mode, boolean fixedSize, Comparator sortComparator, - Counter bytesUsed, IOContext context, float acceptableOverheadRatio) - throws IOException { + Counter bytesUsed, IOContext context, float acceptableOverheadRatio) { // TODO -- i shouldn't have to specify fixed? can // track itself & do the write thing at write time? if (sortComparator == null) { @@ -244,7 +241,7 @@ public final class Bytes { private final IOContext context; protected BytesWriterBase(Directory dir, String id, String codecNameIdx, String codecNameDat, - int version, Counter bytesUsed, IOContext context, Type type) throws IOException { + int version, Counter bytesUsed, IOContext context, Type type) { super(bytesUsed, type); this.id = id; this.dir = dir; @@ -388,21 +385,19 @@ public final class Bytes { protected long maxBytes = 0; protected DerefBytesWriterBase(Directory dir, String id, String codecNameIdx, String codecNameDat, - int codecVersion, Counter bytesUsed, IOContext context, Type type) - throws IOException { + int codecVersion, Counter bytesUsed, IOContext context, Type type) { this(dir, id, codecNameIdx, codecNameDat, codecVersion, new DirectTrackingAllocator( ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed, context, PackedInts.DEFAULT, type); } protected DerefBytesWriterBase(Directory dir, String id, String codecNameIdx, String codecNameDat, - int codecVersion, Counter bytesUsed, IOContext context, float acceptableOverheadRatio, Type type) - throws IOException { + int codecVersion, Counter bytesUsed, IOContext context, float acceptableOverheadRatio, Type type) { this(dir, id, codecNameIdx, codecNameDat, codecVersion, new DirectTrackingAllocator( ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed, context, acceptableOverheadRatio, type); } protected DerefBytesWriterBase(Directory dir, String id, String codecNameIdx, String codecNameDat, int codecVersion, Allocator allocator, - Counter bytesUsed, IOContext context, float acceptableOverheadRatio, Type type) throws IOException { + Counter bytesUsed, IOContext context, float acceptableOverheadRatio, Type type) { super(dir, id, codecNameIdx, codecNameDat, codecVersion, bytesUsed, context, type); hash = new BytesRefHash(new ByteBlockPool(allocator), BytesRefHash.DEFAULT_CAPACITY, new TrackingDirectBytesStartArray( @@ -473,6 +468,10 @@ public final class Bytes { } } + public int getValueSize() { + return size; + } + // Important that we get docCount, in case there were // some last docs that we didn't see @Override diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DirectSource.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DirectSource.java index 5f38e3c..2acbb56 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DirectSource.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DirectSource.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40.values; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -43,6 +43,11 @@ abstract class DirectSource extends Source { toNumeric = new ShortToLong(); break; case FLOAT_32: + toNumeric = new BytesToFloat(); + break; + case FLOAT_64: + toNumeric = new BytesToDouble(); + break; case FIXED_INTS_32: toNumeric = new IntToLong(); break; @@ -58,10 +63,10 @@ abstract class DirectSource extends Source { public BytesRef getBytes(int docID, BytesRef ref) { try { final int sizeToRead = position(docID); + ref.offset = 0; ref.grow(sizeToRead); data.readBytes(ref.bytes, 0, sizeToRead); ref.length = sizeToRead; - ref.offset = 0; return ref; } catch (IOException ex) { throw new IllegalStateException("failed to get value for docID: " + docID, ex); @@ -103,7 +108,6 @@ abstract class DirectSource extends Source { long toLong(IndexInput input) throws IOException { return input.readByte(); } - } private static final class ShortToLong extends ToNumeric { @@ -118,11 +122,30 @@ abstract class DirectSource extends Source { long toLong(IndexInput input) throws IOException { return input.readInt(); } + } + + private static final class BytesToFloat extends ToNumeric { + @Override + long toLong(IndexInput input) { + throw new UnsupportedOperationException("ints are not supported"); + } double toDouble(IndexInput input) throws IOException { return Float.intBitsToFloat(input.readInt()); } } + + private static final class BytesToDouble extends ToNumeric { + @Override + long toLong(IndexInput input) { + throw new UnsupportedOperationException("ints are not supported"); + } + + double toDouble(IndexInput input) throws IOException { + return Double.longBitsToDouble(input.readLong()); + } + } + private static final class LongToLong extends ToNumeric { @Override @@ -130,8 +153,8 @@ abstract class DirectSource extends Source { return input.readLong(); } - double toDouble(IndexInput input) throws IOException { - return Double.longBitsToDouble(input.readLong()); + double toDouble(IndexInput input) { + throw new UnsupportedOperationException("doubles are not supported"); } } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DocValuesWriterBase.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DocValuesWriterBase.java index 22c3655..b5ec3e1 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DocValuesWriterBase.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DocValuesWriterBase.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40.values; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedDerefBytesImpl.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedDerefBytesImpl.java index c7e8740..5f73318 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedDerefBytesImpl.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedDerefBytesImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40.values; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -46,8 +46,7 @@ class FixedDerefBytesImpl { static final int VERSION_CURRENT = VERSION_START; public static class Writer extends DerefBytesWriterBase { - public Writer(Directory dir, String id, Counter bytesUsed, IOContext context) - throws IOException { + public Writer(Directory dir, String id, Counter bytesUsed, IOContext context) { super(dir, id, CODEC_NAME_IDX, CODEC_NAME_DAT, VERSION_CURRENT, bytesUsed, context, Type.BYTES_FIXED_DEREF); } @@ -67,6 +66,7 @@ class FixedDerefBytesImpl { idxOut.writeInt(numValues); writeIndex(idxOut, docCount, numValues, docToEntry); } + } public static class FixedDerefReader extends BytesReaderBase { @@ -109,8 +109,7 @@ class FixedDerefBytesImpl { @Override public BytesRef getBytes(int docID, BytesRef bytesRef) { - final int id = (int) addresses.get(docID); - return data.fillSlice(bytesRef, (id * size), size); + return data.fillSlice(bytesRef, addresses.get(docID) * size, size); } } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java index e5f0a7d..3e18fbc 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40.values; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -58,7 +58,7 @@ class FixedSortedBytesImpl { private final Comparator comp; public Writer(Directory dir, String id, Comparator comp, - Counter bytesUsed, IOContext context, float acceptableOverheadRatio) throws IOException { + Counter bytesUsed, IOContext context, float acceptableOverheadRatio) { super(dir, id, CODEC_NAME_IDX, CODEC_NAME_DAT, VERSION_CURRENT, bytesUsed, context, acceptableOverheadRatio, Type.BYTES_FIXED_SORTED); this.comp = comp; } @@ -213,10 +213,10 @@ class FixedSortedBytesImpl { public BytesRef getByOrd(int ord, BytesRef bytesRef) { try { datIn.seek(basePointer + size * ord); + bytesRef.offset = 0; bytesRef.grow(size); datIn.readBytes(bytesRef.bytes, 0, size); bytesRef.length = size; - bytesRef.offset = 0; return bytesRef; } catch (IOException ex) { throw new IllegalStateException("failed to getByOrd", ex); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedStraightBytesImpl.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedStraightBytesImpl.java index ced34f3..bc072ad 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedStraightBytesImpl.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedStraightBytesImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40.values; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -62,12 +62,12 @@ class FixedStraightBytesImpl { private final ByteBlockPool pool; protected FixedBytesWriterBase(Directory dir, String id, String codecNameDat, - int version, Counter bytesUsed, IOContext context) throws IOException { + int version, Counter bytesUsed, IOContext context) { this(dir, id, codecNameDat, version, bytesUsed, context, Type.BYTES_FIXED_STRAIGHT); } protected FixedBytesWriterBase(Directory dir, String id, String codecNameDat, - int version, Counter bytesUsed, IOContext context, Type type) throws IOException { + int version, Counter bytesUsed, IOContext context, Type type) { super(dir, id, null, codecNameDat, version, bytesUsed, context, type); pool = new ByteBlockPool(new DirectTrackingAllocator(bytesUsed)); pool.nextBuffer(); @@ -129,17 +129,22 @@ class FixedStraightBytesImpl { out.writeBytes(zeros, zeros.length); } } + + @Override + public int getValueSize() { + return size; + } } static class Writer extends FixedBytesWriterBase { private boolean hasMerged; private IndexOutput datOut; - public Writer(Directory dir, String id, Counter bytesUsed, IOContext context) throws IOException { + public Writer(Directory dir, String id, Counter bytesUsed, IOContext context) { super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context); } - public Writer(Directory dir, String id, String codecNameDat, int version, Counter bytesUsed, IOContext context) throws IOException { + public Writer(Directory dir, String id, String codecNameDat, int version, Counter bytesUsed, IOContext context) { super(dir, id, codecNameDat, version, bytesUsed, context); } @@ -342,7 +347,7 @@ class FixedStraightBytesImpl { @Override public BytesRef getBytes(int docID, BytesRef bytesRef) { - return data.fillSlice(bytesRef, docID * size, size); + return data.fillSlice(bytesRef, size * ((long) docID), size); } } @@ -356,7 +361,7 @@ class FixedStraightBytesImpl { @Override protected int position(int docID) throws IOException { - data.seek(baseOffset + size * docID); + data.seek(baseOffset + size * ((long) docID)); return size; } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Floats.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Floats.java index 4d3e054..c8a96fd 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Floats.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Floats.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40.values; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -47,7 +47,7 @@ public class Floats { protected static final int VERSION_CURRENT = VERSION_START; public static DocValuesConsumer getWriter(Directory dir, String id, Counter bytesUsed, - IOContext context, Type type) throws IOException { + IOContext context, Type type) { return new FloatsWriter(dir, id, bytesUsed, context, type); } @@ -72,7 +72,7 @@ public class Floats { private final int size; private final DocValuesArraySource template; public FloatsWriter(Directory dir, String id, Counter bytesUsed, - IOContext context, Type type) throws IOException { + IOContext context, Type type) { super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context); size = typeToSize(type); this.bytesRef = new BytesRef(size); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Ints.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Ints.java index 22875ad..2d937f6 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Ints.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Ints.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40.values; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -46,7 +46,7 @@ public final class Ints { } public static DocValuesConsumer getWriter(Directory dir, String id, Counter bytesUsed, - Type type, IOContext context) throws IOException { + Type type, IOContext context) { return type == Type.VAR_INTS ? new PackedIntValues.PackedIntsWriter(dir, id, bytesUsed, context) : new IntsWriter(dir, id, bytesUsed, context, type); } @@ -92,12 +92,12 @@ public final class Ints { private final DocValuesArraySource template; public IntsWriter(Directory dir, String id, Counter bytesUsed, - IOContext context, Type valueType) throws IOException { + IOContext context, Type valueType) { this(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, valueType); } protected IntsWriter(Directory dir, String id, String codecName, - int version, Counter bytesUsed, IOContext context, Type valueType) throws IOException { + int version, Counter bytesUsed, IOContext context, Type valueType) { super(dir, id, codecName, version, bytesUsed, context); size = typeToSize(valueType); this.bytesRef = new BytesRef(size); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/PackedIntValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/PackedIntValues.java index f3c5ece..c40534f 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/PackedIntValues.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/PackedIntValues.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40.values; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,6 +18,7 @@ package org.apache.lucene.codecs.lucene40.values; */ import java.io.IOException; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesArraySource; import org.apache.lucene.codecs.lucene40.values.FixedStraightBytesImpl.FixedBytesWriterBase; import org.apache.lucene.index.DocValues.Source; @@ -30,7 +31,6 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.Counter; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.packed.PackedInts; @@ -57,7 +57,7 @@ class PackedIntValues { private int lastDocId = -1; protected PackedIntsWriter(Directory dir, String id, Counter bytesUsed, - IOContext context) throws IOException { + IOContext context) { super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, Type.VAR_INTS); bytesRef = new BytesRef(8); } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarDerefBytesImpl.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarDerefBytesImpl.java index fa46bf6..c766a36 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarDerefBytesImpl.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarDerefBytesImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40.values; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -57,8 +57,7 @@ class VarDerefBytesImpl { * order and merge them in a streamed fashion. */ static class Writer extends DerefBytesWriterBase { - public Writer(Directory dir, String id, Counter bytesUsed, IOContext context) - throws IOException { + public Writer(Directory dir, String id, Counter bytesUsed, IOContext context) { super(dir, id, CODEC_NAME_IDX, CODEC_NAME_DAT, VERSION_CURRENT, bytesUsed, context, Type.BYTES_VAR_DEREF); size = 0; } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java index 1d822e0..0cbf6b9 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40.values; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -60,7 +60,7 @@ final class VarSortedBytesImpl { private final Comparator comp; public Writer(Directory dir, String id, Comparator comp, - Counter bytesUsed, IOContext context, float acceptableOverheadRatio) throws IOException { + Counter bytesUsed, IOContext context, float acceptableOverheadRatio) { super(dir, id, CODEC_NAME_IDX, CODEC_NAME_DAT, VERSION_CURRENT, bytesUsed, context, acceptableOverheadRatio, Type.BYTES_VAR_SORTED); this.comp = comp; size = 0; @@ -239,10 +239,10 @@ final class VarSortedBytesImpl { final long nextOffset = ordToOffsetIndex.get(1+ord); datIn.seek(basePointer + offset); final int length = (int) (nextOffset - offset); + bytesRef.offset = 0; bytesRef.grow(length); datIn.readBytes(bytesRef.bytes, 0, length); bytesRef.length = length; - bytesRef.offset = 0; return bytesRef; } catch (IOException ex) { throw new IllegalStateException("failed", ex); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarStraightBytesImpl.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarStraightBytesImpl.java index f3b8444..126c11d 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarStraightBytesImpl.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarStraightBytesImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40.values; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -64,8 +64,7 @@ class VarStraightBytesImpl { private final ByteBlockPool pool; private IndexOutput datOut; private boolean merge = false; - public Writer(Directory dir, String id, Counter bytesUsed, IOContext context) - throws IOException { + public Writer(Directory dir, String id, Counter bytesUsed, IOContext context) { super(dir, id, CODEC_NAME_IDX, CODEC_NAME_DAT, VERSION_CURRENT, bytesUsed, context, Type.BYTES_VAR_STRAIGHT); pool = new ByteBlockPool(new DirectTrackingAllocator(bytesUsed)); docToAddress = new long[1]; @@ -122,7 +121,7 @@ class VarStraightBytesImpl { final IndexInput cloneIdx = reader.cloneIndex(); try { numDataBytes = cloneIdx.readVLong(); - final ReaderIterator iter = PackedInts.getReaderIterator(cloneIdx); + final ReaderIterator iter = PackedInts.getReaderIterator(cloneIdx, PackedInts.DEFAULT_BUFFER_SIZE); for (int i = 0; i < maxDocs; i++) { long offset = iter.next(); ++lastDocID; @@ -232,6 +231,11 @@ class VarStraightBytesImpl { public long ramBytesUsed() { return bytesUsed.get(); } + + @Override + public int getValueSize() { + return -1; + } } public static class VarStraightReader extends BytesReaderBase { diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java index e384457..1b40e49 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40.values; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -83,11 +83,10 @@ abstract class Writer extends DocValuesConsumer { * docvalues of type {@link Type#BYTES_FIXED_SORTED} and * {@link Type#BYTES_VAR_SORTED}. * @return a new {@link Writer} instance for the given {@link Type} - * @throws IOException - * @see PackedInts#getReader(org.apache.lucene.store.DataInput, float) + * @see PackedInts#getReader(org.apache.lucene.store.DataInput) */ public static DocValuesConsumer create(Type type, String id, Directory directory, - Comparator comp, Counter bytesUsed, IOContext context, float acceptableOverheadRatio) throws IOException { + Comparator comp, Counter bytesUsed, IOContext context, float acceptableOverheadRatio) { if (comp == null) { comp = BytesRef.getUTF8SortedAsUnicodeComparator(); } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java index 63a1ad7..fb25738 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.memory; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -54,6 +54,7 @@ import org.apache.lucene.util.fst.ByteSequenceOutputs; import org.apache.lucene.util.fst.BytesRefFSTEnum; import org.apache.lucene.util.fst.FST; import org.apache.lucene.util.fst.Util; +import org.apache.lucene.util.packed.PackedInts; // TODO: would be nice to somehow allow this to act like // InstantiatedIndex, by never writing to disk; ie you write @@ -81,14 +82,16 @@ import org.apache.lucene.util.fst.Util; public class MemoryPostingsFormat extends PostingsFormat { private final boolean doPackFST; + private final float acceptableOverheadRatio; public MemoryPostingsFormat() { - this(false); + this(false, PackedInts.DEFAULT); } - public MemoryPostingsFormat(boolean doPackFST) { + public MemoryPostingsFormat(boolean doPackFST, float acceptableOverheadRatio) { super("Memory"); this.doPackFST = doPackFST; + this.acceptableOverheadRatio = acceptableOverheadRatio; } @Override @@ -102,13 +105,15 @@ public class MemoryPostingsFormat extends PostingsFormat { private final Builder builder; private final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton(); private final boolean doPackFST; + private final float acceptableOverheadRatio; private int termCount; - public TermsWriter(IndexOutput out, FieldInfo field, boolean doPackFST) { + public TermsWriter(IndexOutput out, FieldInfo field, boolean doPackFST, float acceptableOverheadRatio) { this.out = out; this.field = field; this.doPackFST = doPackFST; - builder = new Builder(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, doPackFST); + this.acceptableOverheadRatio = acceptableOverheadRatio; + builder = new Builder(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, doPackFST, acceptableOverheadRatio); } private class PostingsWriter extends PostingsConsumer { @@ -265,7 +270,7 @@ public class MemoryPostingsFormat extends PostingsFormat { out.writeVInt(docCount); FST fst = builder.finish(); if (doPackFST) { - fst = fst.pack(3, Math.max(10, fst.getNodeCount()/4)); + fst = fst.pack(3, Math.max(10, fst.getNodeCount()/4), acceptableOverheadRatio); } fst.save(out); //System.out.println("finish field=" + field.name + " fp=" + out.getFilePointer()); @@ -290,7 +295,7 @@ public class MemoryPostingsFormat extends PostingsFormat { @Override public TermsConsumer addField(FieldInfo field) { //System.out.println("\naddField field=" + field.name); - return new TermsWriter(out, field, doPackFST); + return new TermsWriter(out, field, doPackFST, acceptableOverheadRatio); } @Override @@ -422,7 +427,7 @@ public class MemoryPostingsFormat extends PostingsFormat { } @Override - public int freq() throws IOException { + public int freq() { assert indexOptions != IndexOptions.DOCS_ONLY; return freq; } @@ -622,7 +627,7 @@ public class MemoryPostingsFormat extends PostingsFormat { } @Override - public int freq() throws IOException { + public int freq() { return freq; } } @@ -642,7 +647,7 @@ public class MemoryPostingsFormat extends PostingsFormat { fstEnum = new BytesRefFSTEnum(fst); } - private void decodeMetaData() throws IOException { + private void decodeMetaData() { if (!didDecode) { buffer.reset(current.output.bytes, 0, current.output.length); docFreq = buffer.readVInt(); @@ -691,7 +696,7 @@ public class MemoryPostingsFormat extends PostingsFormat { } @Override - public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException { + public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) { decodeMetaData(); FSTDocsEnum docsEnum; @@ -709,7 +714,7 @@ public class MemoryPostingsFormat extends PostingsFormat { } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException { + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) { boolean hasOffsets = field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; if (needsOffsets && !hasOffsets) { @@ -752,13 +757,13 @@ public class MemoryPostingsFormat extends PostingsFormat { } @Override - public int docFreq() throws IOException { + public int docFreq() { decodeMetaData(); return docFreq; } @Override - public long totalTermFreq() throws IOException { + public long totalTermFreq() { decodeMetaData(); return totalTermFreq; } @@ -812,17 +817,17 @@ public class MemoryPostingsFormat extends PostingsFormat { } @Override - public long getSumDocFreq() throws IOException { + public long getSumDocFreq() { return sumDocFreq; } @Override - public int getDocCount() throws IOException { + public int getDocCount() { return docCount; } @Override - public long size() throws IOException { + public long size() { return termCount; } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/package.html b/lucene/core/src/java/org/apache/lucene/codecs/package.html index 78dcb95..ceccedb 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/package.html +++ b/lucene/core/src/java/org/apache/lucene/codecs/package.html @@ -21,5 +21,46 @@ Codecs API: API for customization of the encoding and structure of the index. + +

      + The Codec API allows you to customise the way the following pieces of index information are stored: +

        +
      • Postings lists - see {@link org.apache.lucene.codecs.PostingsFormat}
        • +
      • DocValues - see {@link org.apache.lucene.codecs.DocValuesFormat}
        • +
      • Stored fields - see {@link org.apache.lucene.codecs.StoredFieldsFormat}
        • +
      • Term vectors - see {@link org.apache.lucene.codecs.TermVectorsFormat}
        • +
      • FieldInfos - see {@link org.apache.lucene.codecs.FieldInfosFormat}
        • +
      • SegmentInfo - see {@link org.apache.lucene.codecs.SegmentInfoFormat}
        • +
      • Norms - see {@link org.apache.lucene.codecs.NormsFormat}
        • +
      • Live documents - see {@link org.apache.lucene.codecs.LiveDocsFormat}
        • +
      +

      + +

      + Codecs are identified by name through the Java Service Provider Interface. To create your own codec, extend + {@link org.apache.lucene.codecs.Codec} and pass the new codec's name to the super() constructor: +

      +public class MyCodec extends Codec {
+
+    public MyCodec() {
+        super("MyCodecName");
+    }
+
+    ...
+}
+
      +You will need to register the Codec class so that the {@link java.util.ServiceLoader ServiceLoader} can find it, by including a +META-INF/services/org.apache.lucene.codecs.Codec file on your classpath that contains the package-qualified +name of your codec. +

      + +

      + If you just want to customise the {@link org.apache.lucene.codecs.PostingsFormat}, or use different postings + formats for different fields, then you can register your custom postings format in the same way (in + META-INF/services/org.apache.lucene.codecs.PostingsFormat), and then extend the default + {@link org.apache.lucene.codecs.lucene40.Lucene40Codec} and override + {@link org.apache.lucene.codecs.lucene40.Lucene40Codec#getPostingsFormatForField(String)} to return your custom + postings format. +

      diff --git a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java index b39e6b6..ca1b47d 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.perfield; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,8 +17,9 @@ package org.apache.lucene.codecs.perfield; * limitations under the License. */ +import java.io.Closeable; import java.io.IOException; -import java.util.IdentityHashMap; +import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.ServiceLoader; // javadocs @@ -46,7 +47,7 @@ import org.apache.lucene.util.IOUtils; *

      * Files written by each posting format have an additional suffix containing the * format name. For example, in a per-field configuration instead of _1.prx - * filenames would look like _1_Lucene40.prx. + * filenames would look like _1_Lucene40_0.prx. * @see ServiceLoader * @lucene.experimental */ @@ -55,6 +56,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat { public static final String PER_FIELD_NAME = "PerField40"; public static final String PER_FIELD_FORMAT_KEY = PerFieldPostingsFormat.class.getSimpleName() + ".format"; + public static final String PER_FIELD_SUFFIX_KEY = PerFieldPostingsFormat.class.getSimpleName() + ".suffix"; public PerFieldPostingsFormat() { super(PER_FIELD_NAME); @@ -65,14 +67,25 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat { throws IOException { return new FieldsWriter(state); } + + static class FieldsConsumerAndSuffix implements Closeable { + FieldsConsumer consumer; + int suffix; + + @Override + public void close() throws IOException { + consumer.close(); + } + } private class FieldsWriter extends FieldsConsumer { - private final Map formats = new IdentityHashMap(); - + private final Map formats = new HashMap(); + private final Map suffixes = new HashMap(); + private final SegmentWriteState segmentWriteState; - public FieldsWriter(SegmentWriteState state) throws IOException { + public FieldsWriter(SegmentWriteState state) { segmentWriteState = state; } @@ -82,26 +95,48 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat { if (format == null) { throw new IllegalStateException("invalid null PostingsFormat for field=\"" + field.name + "\""); } + final String formatName = format.getName(); - String previousValue = field.putAttribute(PER_FIELD_FORMAT_KEY, format.getName()); + String previousValue = field.putAttribute(PER_FIELD_FORMAT_KEY, formatName); assert previousValue == null; - - FieldsConsumer consumer = formats.get(format); + + Integer suffix; + + FieldsConsumerAndSuffix consumer = formats.get(format); if (consumer == null) { // First time we are seeing this format; create a new instance + + // bump the suffix + suffix = suffixes.get(formatName); + if (suffix == null) { + suffix = 0; + } else { + suffix = suffix + 1; + } + suffixes.put(formatName, suffix); + final String segmentSuffix = getFullSegmentSuffix(field.name, segmentWriteState.segmentSuffix, - format.getName()); - consumer = format.fieldsConsumer(new SegmentWriteState(segmentWriteState, segmentSuffix)); + getSuffix(formatName, Integer.toString(suffix))); + consumer = new FieldsConsumerAndSuffix(); + consumer.consumer = format.fieldsConsumer(new SegmentWriteState(segmentWriteState, segmentSuffix)); + consumer.suffix = suffix; formats.put(format, consumer); + } else { + // we've already seen this format, so just grab its suffix + assert suffixes.containsKey(formatName); + suffix = consumer.suffix; } + + previousValue = field.putAttribute(PER_FIELD_SUFFIX_KEY, Integer.toString(suffix)); + assert previousValue == null; // TODO: we should only provide the "slice" of FIS // that this PF actually sees ... then stuff like // .hasProx could work correctly? // NOTE: .hasProx is already broken in the same way for the non-perfield case, // if there is a fieldinfo with prox that has no postings, you get a 0 byte file. - return consumer.addField(field); + return consumer.consumer.addField(field); } @Override @@ -110,6 +145,10 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat { IOUtils.close(formats.values()); } } + + static String getSuffix(String formatName, String suffix) { + return formatName + "_" + suffix; + } static String getFullSegmentSuffix(String fieldName, String outerSegmentSuffix, String segmentSuffix) { if (outerSegmentSuffix.length() == 0) { @@ -125,7 +164,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat { private class FieldsReader extends FieldsProducer { private final Map fields = new TreeMap(); - private final Map formats = new IdentityHashMap(); + private final Map formats = new HashMap(); public FieldsReader(final SegmentReadState readState) throws IOException { @@ -139,11 +178,14 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat { final String formatName = fi.getAttribute(PER_FIELD_FORMAT_KEY); if (formatName != null) { // null formatName means the field is in fieldInfos, but has no postings! + final String suffix = fi.getAttribute(PER_FIELD_SUFFIX_KEY); + assert suffix != null; PostingsFormat format = PostingsFormat.forName(formatName); - if (!formats.containsKey(format)) { - formats.put(format, format.fieldsProducer(new SegmentReadState(readState, formatName))); + String segmentSuffix = getSuffix(formatName, suffix); + if (!formats.containsKey(segmentSuffix)) { + formats.put(segmentSuffix, format.fieldsProducer(new SegmentReadState(readState, segmentSuffix))); } - fields.put(fieldName, formats.get(format)); + fields.put(fieldName, formats.get(segmentSuffix)); } } } @@ -164,7 +206,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat { } @Override - public String next() throws IOException { + public String next() { if (it.hasNext()) { current = it.next(); } else { diff --git a/lucene/core/src/java/org/apache/lucene/codecs/pulsing/Pulsing40PostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/pulsing/Pulsing40PostingsFormat.java index f99b755..faf8df2 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/pulsing/Pulsing40PostingsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/pulsing/Pulsing40PostingsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.pulsing; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsFormat.java index b443b61..ceb9e84 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.pulsing; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -29,6 +29,7 @@ import org.apache.lucene.codecs.PostingsReaderBase; import org.apache.lucene.codecs.PostingsWriterBase; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.util.IOUtils; /** This postings format "inlines" the postings for terms that have * low docFreq. It wraps another postings format, which is used for @@ -65,33 +66,39 @@ public abstract class PulsingPostingsFormat extends PostingsFormat { @Override public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - PostingsWriterBase docsWriter = wrappedPostingsBaseFormat.postingsWriterBase(state); + PostingsWriterBase docsWriter = null; // Terms that have <= freqCutoff number of docs are // "pulsed" (inlined): - PostingsWriterBase pulsingWriter = new PulsingPostingsWriter(freqCutoff, docsWriter); + PostingsWriterBase pulsingWriter = null; // Terms dict boolean success = false; try { + docsWriter = wrappedPostingsBaseFormat.postingsWriterBase(state); + + // Terms that have <= freqCutoff number of docs are + // "pulsed" (inlined): + pulsingWriter = new PulsingPostingsWriter(freqCutoff, docsWriter); FieldsConsumer ret = new BlockTreeTermsWriter(state, pulsingWriter, minBlockSize, maxBlockSize); success = true; return ret; } finally { if (!success) { - pulsingWriter.close(); + IOUtils.closeWhileHandlingException(docsWriter, pulsingWriter); } } } @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - - PostingsReaderBase docsReader = wrappedPostingsBaseFormat.postingsReaderBase(state); - PostingsReaderBase pulsingReader = new PulsingPostingsReader(docsReader); + PostingsReaderBase docsReader = null; + PostingsReaderBase pulsingReader = null; boolean success = false; try { + docsReader = wrappedPostingsBaseFormat.postingsReaderBase(state); + pulsingReader = new PulsingPostingsReader(docsReader); FieldsProducer ret = new BlockTreeTermsReader( state.dir, state.fieldInfos, state.segmentInfo.name, pulsingReader, @@ -102,7 +109,7 @@ public abstract class PulsingPostingsFormat extends PostingsFormat { return ret; } finally { if (!success) { - pulsingReader.close(); + IOUtils.closeWhileHandlingException(docsReader, pulsingReader); } } } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java index f5026fd..e805e77 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.pulsing; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,6 +22,7 @@ import java.util.IdentityHashMap; import java.util.Map; import org.apache.lucene.codecs.BlockTermState; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.PostingsReaderBase; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; @@ -36,7 +37,6 @@ import org.apache.lucene.util.AttributeImpl; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CodecUtil; /** Concrete class that reads the current doc/freq/skip * postings format @@ -52,7 +52,7 @@ public class PulsingPostingsReader extends PostingsReaderBase { final PostingsReaderBase wrappedPostingsReader; int maxPositions; - public PulsingPostingsReader(PostingsReaderBase wrappedPostingsReader) throws IOException { + public PulsingPostingsReader(PostingsReaderBase wrappedPostingsReader) { this.wrappedPostingsReader = wrappedPostingsReader; } @@ -179,9 +179,6 @@ public class PulsingPostingsReader extends PostingsReaderBase { @Override public DocsEnum docs(FieldInfo field, BlockTermState _termState, Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException { - if (needsFreqs && field.getIndexOptions() == IndexOptions.DOCS_ONLY) { - return null; - } PulsingTermState termState = (PulsingTermState) _termState; if (termState.postingsSize != -1) { PulsingDocsEnum postings; @@ -217,11 +214,6 @@ public class PulsingPostingsReader extends PostingsReaderBase { @Override public DocsAndPositionsEnum docsAndPositions(FieldInfo field, BlockTermState _termState, Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException { - if (field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { - return null; - } else if (needsOffsets && field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) < 0) { - return null; - } final PulsingTermState termState = (PulsingTermState) _termState; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsWriter.java index 80d9265..32afac2 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.pulsing; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.List; import java.util.ArrayList; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.PostingsWriterBase; import org.apache.lucene.codecs.TermStats; import org.apache.lucene.index.FieldInfo; @@ -28,7 +29,6 @@ import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RAMOutputStream; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CodecUtil; // TODO: we now inline based on total TF of the term, // but it might be better to inline by "net bytes used" @@ -92,7 +92,7 @@ public final class PulsingPostingsWriter extends PostingsWriterBase { /** If the total number of positions (summed across all docs * for this term) is <= maxPositions, then the postings are * inlined into terms dict */ - public PulsingPostingsWriter(int maxPositions, PostingsWriterBase wrappedPostingsWriter) throws IOException { + public PulsingPostingsWriter(int maxPositions, PostingsWriterBase wrappedPostingsWriter) { pending = new Position[maxPositions]; for(int i=0;i 0; - zeroBytes = new BytesRef(new byte[fixedSize]); + assert valueSize > 0; + zeroBytes = new BytesRef(new byte[valueSize]); } SimpleTextUtil.write(output, zeroBytes); break; @@ -286,4 +286,9 @@ public class SimpleTextDocValuesConsumer extends DocValuesConsumer { protected Type getType() { return type; } + + @Override + public int getValueSize() { + return valueSize; + } } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesFormat.java index f27bf1e..033136e 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.simpletext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java index 01853cc..efe9ff4 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.simpletext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java index 6ea9ca2..87891f4 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.simpletext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -72,6 +72,15 @@ public class SimpleTextFieldInfosReader extends FieldInfosReader { assert StringHelper.startsWith(scratch, ISINDEXED); boolean isIndexed = Boolean.parseBoolean(readString(ISINDEXED.length, scratch)); + final IndexOptions indexOptions; + if (isIndexed) { + SimpleTextUtil.readLine(input, scratch); + assert StringHelper.startsWith(scratch, INDEXOPTIONS); + indexOptions = IndexOptions.valueOf(readString(INDEXOPTIONS.length, scratch)); + } else { + indexOptions = null; + } + SimpleTextUtil.readLine(input, scratch); assert StringHelper.startsWith(scratch, STORETV); boolean storeTermVector = Boolean.parseBoolean(readString(STORETV.length, scratch)); @@ -95,10 +104,6 @@ public class SimpleTextFieldInfosReader extends FieldInfosReader { final DocValues.Type docValuesType = docValuesType(dvType); SimpleTextUtil.readLine(input, scratch); - assert StringHelper.startsWith(scratch, INDEXOPTIONS); - IndexOptions indexOptions = IndexOptions.valueOf(readString(INDEXOPTIONS.length, scratch)); - - SimpleTextUtil.readLine(input, scratch); assert StringHelper.startsWith(scratch, NUM_ATTS); int numAtts = Integer.parseInt(readString(NUM_ATTS.length, scratch)); Map atts = new HashMap(); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java index be618d6..d8e4072 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.simpletext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -68,8 +68,6 @@ public class SimpleTextFieldInfosWriter extends FieldInfosWriter { SimpleTextUtil.writeNewline(out); for (FieldInfo fi : infos) { - assert fi.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !fi.hasPayloads(); - SimpleTextUtil.write(out, NAME); SimpleTextUtil.write(out, fi.name, scratch); SimpleTextUtil.writeNewline(out); @@ -82,6 +80,13 @@ public class SimpleTextFieldInfosWriter extends FieldInfosWriter { SimpleTextUtil.write(out, Boolean.toString(fi.isIndexed()), scratch); SimpleTextUtil.writeNewline(out); + if (fi.isIndexed()) { + assert fi.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !fi.hasPayloads(); + SimpleTextUtil.write(out, INDEXOPTIONS); + SimpleTextUtil.write(out, fi.getIndexOptions().toString(), scratch); + SimpleTextUtil.writeNewline(out); + } + SimpleTextUtil.write(out, STORETV); SimpleTextUtil.write(out, Boolean.toString(fi.hasVectors()), scratch); SimpleTextUtil.writeNewline(out); @@ -101,11 +106,7 @@ public class SimpleTextFieldInfosWriter extends FieldInfosWriter { SimpleTextUtil.write(out, DOCVALUES); SimpleTextUtil.write(out, getDocValuesType(fi.getDocValuesType()), scratch); SimpleTextUtil.writeNewline(out); - - SimpleTextUtil.write(out, INDEXOPTIONS); - SimpleTextUtil.write(out, fi.getIndexOptions().toString(), scratch); - SimpleTextUtil.writeNewline(out); - + Map atts = fi.attributes(); int numAtts = atts == null ? 0 : atts.size(); SimpleTextUtil.write(out, NUM_ATTS); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java index 02f6d96..7d829c3 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.simpletext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -105,7 +105,7 @@ class SimpleTextFieldsReader extends FieldsProducer { private boolean ended; private final BytesRefFSTEnum>> fstEnum; - public SimpleTextTermsEnum(FST>> fst, IndexOptions indexOptions) throws IOException { + public SimpleTextTermsEnum(FST>> fst, IndexOptions indexOptions) { this.indexOptions = indexOptions; fstEnum = new BytesRefFSTEnum>>(fst); } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsWriter.java index df34531..5a59399 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.simpletext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -104,7 +104,7 @@ class SimpleTextFieldsWriter extends FieldsConsumer { private final boolean writeOffsets; // for assert: - private int lastEndOffset = -1; + private int lastStartOffset = 0; public SimpleTextPostingsWriter(FieldInfo field) { this.indexOptions = field.getIndexOptions(); @@ -133,7 +133,7 @@ class SimpleTextFieldsWriter extends FieldsConsumer { newline(); } - lastEndOffset = -1; + lastStartOffset = 0; } public PostingsConsumer reset(BytesRef term) { @@ -152,8 +152,8 @@ class SimpleTextFieldsWriter extends FieldsConsumer { if (writeOffsets) { assert endOffset >= startOffset; - assert startOffset >= lastEndOffset: "startOffset=" + startOffset + " lastEndOffset=" + lastEndOffset; - lastEndOffset = endOffset; + assert startOffset >= lastStartOffset: "startOffset=" + startOffset + " lastStartOffset=" + lastStartOffset; + lastStartOffset = startOffset; write(START_OFFSET); write(Integer.toString(startOffset)); newline(); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextLiveDocsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextLiveDocsFormat.java index 1604fef..4d9007d 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextLiveDocsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextLiveDocsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.simpletext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -99,7 +99,7 @@ public class SimpleTextLiveDocsFormat extends LiveDocsFormat { } } - private int parseIntAt(BytesRef bytes, int offset, CharsRef scratch) throws IOException { + private int parseIntAt(BytesRef bytes, int offset, CharsRef scratch) { UnicodeUtil.UTF8toUTF16(bytes.bytes, bytes.offset+offset, bytes.length-offset, scratch); return ArrayUtil.parseInt(scratch.chars, 0, scratch.length); } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java index 5bd573c..5ba5f74 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.simpletext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -95,8 +95,7 @@ public class SimpleTextNormsFormat extends NormsFormat { public static class SimpleTextNormsPerDocConsumer extends SimpleTextPerDocConsumer { - public SimpleTextNormsPerDocConsumer(PerDocWriteState state) - throws IOException { + public SimpleTextNormsPerDocConsumer(PerDocWriteState state) { super(state, NORMS_SEG_SUFFIX); } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocConsumer.java index abadbc8..d95e09b 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocConsumer.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.simpletext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF @@ -32,8 +32,7 @@ class SimpleTextPerDocConsumer extends PerDocConsumer { protected final PerDocWriteState state; protected final String segmentSuffix; - public SimpleTextPerDocConsumer(PerDocWriteState state, String segmentSuffix) - throws IOException { + public SimpleTextPerDocConsumer(PerDocWriteState state, String segmentSuffix) { this.state = state; this.segmentSuffix = segmentSuffix; } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocProducer.java index 50076f0..266a1fe 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocProducer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocProducer.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.simpletext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPostingsFormat.java index 61c5728..4f7cfe0 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPostingsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPostingsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.simpletext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java index b214d7d..53440e7 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.simpletext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -34,12 +34,12 @@ public class SimpleTextSegmentInfoFormat extends SegmentInfoFormat { public static final String SI_EXTENSION = "si"; @Override - public SegmentInfoReader getSegmentInfosReader() { + public SegmentInfoReader getSegmentInfoReader() { return reader; } @Override - public SegmentInfoWriter getSegmentInfosWriter() { + public SegmentInfoWriter getSegmentInfoWriter() { return writer; } } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoReader.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoReader.java index 994576a..c020f14 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.simpletext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java index 8f690c7..cbad776 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.simpletext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsFormat.java index dadc0a7..13c73b4 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.simpletext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java index 6406d3a..dd18138 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.simpletext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.ArrayList; import org.apache.lucene.codecs.StoredFieldsReader; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.IndexFileNames; @@ -88,7 +87,7 @@ public class SimpleTextStoredFieldsReader extends StoredFieldsReader { } @Override - public void visitDocument(int n, StoredFieldVisitor visitor) throws CorruptIndexException, IOException { + public void visitDocument(int n, StoredFieldVisitor visitor) throws IOException { in.seek(offsets.get(n)); readLine(); assert StringHelper.startsWith(scratch, NUM); @@ -181,7 +180,7 @@ public class SimpleTextStoredFieldsReader extends StoredFieldsReader { SimpleTextUtil.readLine(in, scratch); } - private int parseIntAt(int offset) throws IOException { + private int parseIntAt(int offset) { UnicodeUtil.UTF8toUTF16(scratch.bytes, scratch.offset+offset, scratch.length-offset, scratchUTF16); return ArrayUtil.parseInt(scratchUTF16.chars, 0, scratchUTF16.length); } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java index 717f612..3d38879 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.simpletext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsFormat.java index 1afa799..a0fe06d 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.simpletext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java index 6ff892e..5fc92d6 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.simpletext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -204,7 +204,7 @@ public class SimpleTextTermVectorsReader extends TermVectorsReader { SimpleTextUtil.readLine(in, scratch); } - private int parseIntAt(int offset) throws IOException { + private int parseIntAt(int offset) { UnicodeUtil.UTF8toUTF16(scratch.bytes, scratch.offset+offset, scratch.length-offset, scratchUTF16); return ArrayUtil.parseInt(scratchUTF16.chars, 0, scratchUTF16.length); } @@ -217,7 +217,7 @@ public class SimpleTextTermVectorsReader extends TermVectorsReader { private class SimpleTVFields extends Fields { private final SortedMap fields; - SimpleTVFields(SortedMap fields) throws IOException { + SimpleTVFields(SortedMap fields) { this.fields = fields; } @@ -228,7 +228,7 @@ public class SimpleTextTermVectorsReader extends TermVectorsReader { private Map.Entry current = null; @Override - public String next() throws IOException { + public String next() { if (!iterator.hasNext()) { return null; } else { @@ -238,7 +238,7 @@ public class SimpleTextTermVectorsReader extends TermVectorsReader { } @Override - public Terms terms() throws IOException { + public Terms terms() { return current.getValue(); } }; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java index 282186a..0f4a913 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.simpletext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextUtil.java index cebc6df..c0c7787 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextUtil.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextUtil.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.simpletext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/document/ByteDocValuesField.java b/lucene/core/src/java/org/apache/lucene/document/ByteDocValuesField.java index 8291286..915f5f1 100644 --- a/lucene/core/src/java/org/apache/lucene/document/ByteDocValuesField.java +++ b/lucene/core/src/java/org/apache/lucene/document/ByteDocValuesField.java @@ -1,6 +1,6 @@ package org.apache.lucene.document; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -32,6 +32,8 @@ import org.apache.lucene.index.DocValues; *

      * If you also need to store the value, you should add a * separate {@link StoredField} instance. + * + * @see DocValues for further information * */ public class ByteDocValuesField extends Field { diff --git a/lucene/core/src/java/org/apache/lucene/document/CompressionTools.java b/lucene/core/src/java/org/apache/lucene/document/CompressionTools.java index 52d7cf9..5fcee26 100644 --- a/lucene/core/src/java/org/apache/lucene/document/CompressionTools.java +++ b/lucene/core/src/java/org/apache/lucene/document/CompressionTools.java @@ -1,6 +1,6 @@ package org.apache.lucene.document; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/document/DateTools.java b/lucene/core/src/java/org/apache/lucene/document/DateTools.java index 9603443..cf9fdd7 100644 --- a/lucene/core/src/java/org/apache/lucene/document/DateTools.java +++ b/lucene/core/src/java/org/apache/lucene/document/DateTools.java @@ -1,6 +1,6 @@ package org.apache.lucene.document; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/document/DerefBytesDocValuesField.java b/lucene/core/src/java/org/apache/lucene/document/DerefBytesDocValuesField.java index 1e9e68a..99d6829 100644 --- a/lucene/core/src/java/org/apache/lucene/document/DerefBytesDocValuesField.java +++ b/lucene/core/src/java/org/apache/lucene/document/DerefBytesDocValuesField.java @@ -1,6 +1,6 @@ package org.apache.lucene.document; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -37,6 +37,8 @@ import org.apache.lucene.util.BytesRef; *

      * If you also need to store the value, you should add a * separate {@link StoredField} instance. + * + * @see DocValues for further information * */ public class DerefBytesDocValuesField extends Field { diff --git a/lucene/core/src/java/org/apache/lucene/document/Document.java b/lucene/core/src/java/org/apache/lucene/document/Document.java index f3322f4..da8424b 100644 --- a/lucene/core/src/java/org/apache/lucene/document/Document.java +++ b/lucene/core/src/java/org/apache/lucene/document/Document.java @@ -1,6 +1,6 @@ package org.apache.lucene.document; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/document/DocumentStoredFieldVisitor.java b/lucene/core/src/java/org/apache/lucene/document/DocumentStoredFieldVisitor.java index bd2ede8..275dc94 100644 --- a/lucene/core/src/java/org/apache/lucene/document/DocumentStoredFieldVisitor.java +++ b/lucene/core/src/java/org/apache/lucene/document/DocumentStoredFieldVisitor.java @@ -1,6 +1,6 @@ package org.apache.lucene.document; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/document/DoubleDocValuesField.java b/lucene/core/src/java/org/apache/lucene/document/DoubleDocValuesField.java index b0021b6..1a0c225 100644 --- a/lucene/core/src/java/org/apache/lucene/document/DoubleDocValuesField.java +++ b/lucene/core/src/java/org/apache/lucene/document/DoubleDocValuesField.java @@ -1,6 +1,6 @@ package org.apache.lucene.document; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -32,6 +32,8 @@ import org.apache.lucene.index.DocValues; *

      * If you also need to store the value, you should add a * separate {@link StoredField} instance. + * + * @see DocValues for further information * */ public class DoubleDocValuesField extends Field { diff --git a/lucene/core/src/java/org/apache/lucene/document/DoubleField.java b/lucene/core/src/java/org/apache/lucene/document/DoubleField.java index 98720bb..32a0146 100644 --- a/lucene/core/src/java/org/apache/lucene/document/DoubleField.java +++ b/lucene/core/src/java/org/apache/lucene/document/DoubleField.java @@ -1,6 +1,6 @@ package org.apache.lucene.document; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -30,14 +30,14 @@ import org.apache.lucene.util.NumericUtils; * for efficient range filtering and sorting. Here's an example usage: * * 

      - * document.add(new DoubleField(name, 6.0));
+ * document.add(new DoubleField(name, 6.0, Field.Store.NO));
  *
      * * For optimal performance, re-use the DoubleField and * {@link Document} instance for more than one document: * * 
      - *  DoubleField field = new DoubleField(name, 0.0);
+ *  DoubleField field = new DoubleField(name, 0.0, Field.Store.NO);
  *  Document document = new Document();
  *  document.add(field);
  * 
@@ -59,10 +59,6 @@ import org.apache.lucene.util.NumericUtils;
  * {@link org.apache.lucene.search.SortField.Type#DOUBLE}. DoubleField 
  * values can also be loaded directly from {@link FieldCache}.
      
  *
- * 
      By default, a DoubleField's value is not stored but
- * is indexed for range filtering and sorting.  You can use
- * {@link StoredField} to also store the value.
- *
  * 
      You may add the same field name as an DoubleField to
  * the same document more than once.  Range querying and
  * filtering will be the logical OR of all values; so a range query
@@ -118,21 +114,32 @@ import org.apache.lucene.util.NumericUtils;
 
 public final class DoubleField extends Field {
   
-  public static final FieldType TYPE = new FieldType();
+  public static final FieldType TYPE_NOT_STORED = new FieldType();
+  static {
+    TYPE_NOT_STORED.setIndexed(true);
+    TYPE_NOT_STORED.setTokenized(true);
+    TYPE_NOT_STORED.setOmitNorms(true);
+    TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
+    TYPE_NOT_STORED.setNumericType(FieldType.NumericType.DOUBLE);
+    TYPE_NOT_STORED.freeze();
+  }
+
+  public static final FieldType TYPE_STORED = new FieldType();
   static {
-    TYPE.setIndexed(true);
-    TYPE.setTokenized(true);
-    TYPE.setOmitNorms(true);
-    TYPE.setIndexOptions(IndexOptions.DOCS_ONLY);
-    TYPE.setNumericType(FieldType.NumericType.DOUBLE);
-    TYPE.freeze();
+    TYPE_STORED.setIndexed(true);
+    TYPE_STORED.setTokenized(true);
+    TYPE_STORED.setOmitNorms(true);
+    TYPE_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
+    TYPE_STORED.setNumericType(FieldType.NumericType.DOUBLE);
+    TYPE_STORED.setStored(true);
+    TYPE_STORED.freeze();
   }
 
-  /** Creates an LongField with the provided value
+  /** Creates a stored or un-stored DoubleField with the provided value
    *  and default precisionStep {@link
    *  NumericUtils#PRECISION_STEP_DEFAULT} (4). */
-  public DoubleField(String name, double value) {
-    super(name, TYPE);
+  public DoubleField(String name, double value, Store stored) {
+    super(name, stored == Store.YES ? TYPE_STORED : TYPE_NOT_STORED);
     fieldsData = Double.valueOf(value);
   }
   
diff --git a/lucene/core/src/java/org/apache/lucene/document/Field.java b/lucene/core/src/java/org/apache/lucene/document/Field.java
index 9c50dd3..47cd3e4 100644
--- a/lucene/core/src/java/org/apache/lucene/document/Field.java
+++ b/lucene/core/src/java/org/apache/lucene/document/Field.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.document;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -469,7 +469,7 @@ public class Field implements IndexableField {
         boolean used;
 
         @Override
-        public boolean incrementToken() throws IOException {
+        public boolean incrementToken() {
           if (used) {
             return false;
           }
@@ -480,7 +480,7 @@ public class Field implements IndexableField {
         }
 
         @Override
-        public void reset() throws IOException {
+        public void reset() {
           used = false;
         }
       };
@@ -497,16 +497,7 @@ public class Field implements IndexableField {
     throw new IllegalArgumentException("Field must have either TokenStream, String, Reader or Number value");
   }
 
-  
-  //
-  // Deprecated transition API below:
-  //
-
-  /** Specifies whether and how a field should be stored.
-   *
-   *  @deprecated This is here only to ease transition from
-   *  the pre-4.0 APIs. */
-  @Deprecated
+  /** Specifies whether and how a field should be stored. */
   public static enum Store {
 
     /** Store the original field value in the index. This is useful for short texts
@@ -514,429 +505,9 @@ public class Field implements IndexableField {
      * value is stored in its original form, i.e. no analyzer is used before it is
      * stored.
      */
-    YES {
-      @Override
-      public boolean isStored() { return true; }
-    },
+    YES,
 
     /** Do not store the field value in the index. */
-    NO {
-      @Override
-      public boolean isStored() { return false; }
-    };
-
-    public abstract boolean isStored();
-  }
-
-  /** Specifies whether and how a field should be indexed.
-   *
-   *  @deprecated This is here only to ease transition from
-   *  the pre-4.0 APIs. */
-  @Deprecated
-  public static enum Index {
-
-    /** Do not index the field value. This field can thus not be searched,
-     * but one can still access its contents provided it is
-     * {@link Field.Store stored}. */
-    NO {
-      @Override
-      public boolean isIndexed()  { return false; }
-      @Override
-      public boolean isAnalyzed() { return false; }
-      @Override
-      public boolean omitNorms()  { return true;  }   
-    },
-
-    /** Index the tokens produced by running the field's
-     * value through an Analyzer.  This is useful for
-     * common text. */
-    ANALYZED {
-      @Override
-      public boolean isIndexed()  { return true;  }
-      @Override
-      public boolean isAnalyzed() { return true;  }
-      @Override
-      public boolean omitNorms()  { return false; }   	
-    },
-
-    /** Index the field's value without using an Analyzer, so it can be searched.
-     * As no analyzer is used the value will be stored as a single term. This is
-     * useful for unique Ids like product numbers.
-     */
-    NOT_ANALYZED {
-      @Override
-      public boolean isIndexed()  { return true;  }
-      @Override
-      public boolean isAnalyzed() { return false; }
-      @Override
-      public boolean omitNorms()  { return false; }   	
-    },
-
-    /** Expert: Index the field's value without an Analyzer,
-     * and also disable the indexing of norms.  Note that you
-     * can also separately enable/disable norms by calling
-     * {@link FieldType#setOmitNorms}.  No norms means that
-     * index-time field and document boosting and field
-     * length normalization are disabled.  The benefit is
-     * less memory usage as norms take up one byte of RAM
-     * per indexed field for every document in the index,
-     * during searching.  Note that once you index a given
-     * field with norms enabled, disabling norms will
-     * have no effect.  In other words, for this to have the
-     * above described effect on a field, all instances of
-     * that field must be indexed with NOT_ANALYZED_NO_NORMS
-     * from the beginning. */
-    NOT_ANALYZED_NO_NORMS {
-      @Override
-      public boolean isIndexed()  { return true;  }
-      @Override
-      public boolean isAnalyzed() { return false; }
-      @Override
-      public boolean omitNorms()  { return true;  }   	
-    },
-
-    /** Expert: Index the tokens produced by running the
-     *  field's value through an Analyzer, and also
-     *  separately disable the storing of norms.  See
-     *  {@link #NOT_ANALYZED_NO_NORMS} for what norms are
-     *  and why you may want to disable them. */
-    ANALYZED_NO_NORMS {
-      @Override
-      public boolean isIndexed()  { return true;  }
-      @Override
-      public boolean isAnalyzed() { return true;  }
-      @Override
-      public boolean omitNorms()  { return true;  }   	
-    };
-
-    /** Get the best representation of the index given the flags. */
-    public static Index toIndex(boolean indexed, boolean analyzed) {
-      return toIndex(indexed, analyzed, false);
-    }
-
-    /** Expert: Get the best representation of the index given the flags. */
-    public static Index toIndex(boolean indexed, boolean analyzed, boolean omitNorms) {
-
-      // If it is not indexed nothing else matters
-      if (!indexed) {
-        return Index.NO;
-      }
-
-      // typical, non-expert
-      if (!omitNorms) {
-        if (analyzed) {
-          return Index.ANALYZED;
-        }
-        return Index.NOT_ANALYZED;
-      }
-
-      // Expert: Norms omitted
-      if (analyzed) {
-        return Index.ANALYZED_NO_NORMS;
-      }
-      return Index.NOT_ANALYZED_NO_NORMS;
-    }
-
-    public abstract boolean isIndexed();
-    public abstract boolean isAnalyzed();
-    public abstract boolean omitNorms();  	
-  }
-
-  /** Specifies whether and how a field should have term vectors.
-   *
-   *  @deprecated This is here only to ease transition from
-   *  the pre-4.0 APIs. */
-  @Deprecated
-  public static enum TermVector {
-    
-    /** Do not store term vectors. 
-     */
-    NO {
-      @Override
-      public boolean isStored()      { return false; }
-      @Override
-      public boolean withPositions() { return false; }
-      @Override
-      public boolean withOffsets()   { return false; }
-    },
-    
-    /** Store the term vectors of each document. A term vector is a list
-     * of the document's terms and their number of occurrences in that document. */
-    YES {
-      @Override
-      public boolean isStored()      { return true;  }
-      @Override
-      public boolean withPositions() { return false; }
-      @Override
-      public boolean withOffsets()   { return false; }
-    },
-    
-    /**
-     * Store the term vector + token position information
-     * 
-     * @see #YES
-     */ 
-    WITH_POSITIONS {
-      @Override
-      public boolean isStored()      { return true;  }
-      @Override
-      public boolean withPositions() { return true;  }
-      @Override
-      public boolean withOffsets()   { return false; }
-    },
-    
-    /**
-     * Store the term vector + Token offset information
-     * 
-     * @see #YES
-     */ 
-    WITH_OFFSETS {
-      @Override
-      public boolean isStored()      { return true;  }
-      @Override
-      public boolean withPositions() { return false; }
-      @Override
-      public boolean withOffsets()   { return true;  }
-    },
-    
-    /**
-     * Store the term vector + Token position and offset information
-     * 
-     * @see #YES
-     * @see #WITH_POSITIONS
-     * @see #WITH_OFFSETS
-     */ 
-    WITH_POSITIONS_OFFSETS {
-      @Override
-      public boolean isStored()      { return true;  }
-      @Override
-      public boolean withPositions() { return true;  }
-      @Override
-      public boolean withOffsets()   { return true;  }
-    };
-
-    /** Get the best representation of a TermVector given the flags. */
-    public static TermVector toTermVector(boolean stored, boolean withOffsets, boolean withPositions) {
-
-      // If it is not stored, nothing else matters.
-      if (!stored) {
-        return TermVector.NO;
-      }
-
-      if (withOffsets) {
-        if (withPositions) {
-          return Field.TermVector.WITH_POSITIONS_OFFSETS;
-        }
-        return Field.TermVector.WITH_OFFSETS;
-      }
-
-      if (withPositions) {
-        return Field.TermVector.WITH_POSITIONS;
-      }
-      return Field.TermVector.YES;
-    }
-
-    public abstract boolean isStored();
-    public abstract boolean withPositions();
-    public abstract boolean withOffsets();
-  }
-
-  /** Translates the pre-4.0 enums for specifying how a
-   *  field should be indexed into the 4.0 {@link FieldType}
-   *  approach.
-   *
-   * @deprecated This is here only to ease transition from
-   * the pre-4.0 APIs.
-   */
-  @Deprecated
-  public static final FieldType translateFieldType(Store store, Index index, TermVector termVector) {
-    final FieldType ft = new FieldType();
-
-    ft.setStored(store == Store.YES);
-
-    switch(index) {
-    case ANALYZED:
-      ft.setIndexed(true);
-      ft.setTokenized(true);
-      break;
-    case ANALYZED_NO_NORMS:
-      ft.setIndexed(true);
-      ft.setTokenized(true);
-      ft.setOmitNorms(true);
-      break;
-    case NOT_ANALYZED:
-      ft.setIndexed(true);
-      break;
-    case NOT_ANALYZED_NO_NORMS:
-      ft.setIndexed(true);
-      ft.setOmitNorms(true);
-      break;
-    case NO:
-      break;
-    }
-
-    switch(termVector) {
-    case NO:
-      break;
-    case YES:
-      ft.setStoreTermVectors(true);
-      break;
-    case WITH_POSITIONS:
-      ft.setStoreTermVectors(true);
-      ft.setStoreTermVectorPositions(true);
-      break;
-    case WITH_OFFSETS:
-      ft.setStoreTermVectors(true);
-      ft.setStoreTermVectorOffsets(true);
-      break;
-    case WITH_POSITIONS_OFFSETS:
-      ft.setStoreTermVectors(true);
-      ft.setStoreTermVectorPositions(true);
-      ft.setStoreTermVectorOffsets(true);
-      break;
-    }
-    ft.freeze();
-    return ft;
-  }
-
-  /**
-   * Create a field by specifying its name, value and how it will
-   * be saved in the index. Term vectors will not be stored in the index.
-   * 
-   * @param name The name of the field
-   * @param value The string to process
-   * @param store Whether value should be stored in the index
-   * @param index Whether the field should be indexed, and if so, if it should
-   *  be tokenized before indexing 
-   * @throws NullPointerException if name or value is null
-   * @throws IllegalArgumentException if the field is neither stored nor indexed 
-   *
-   * @deprecated Use {@link StringField}, {@link TextField} instead. */
-  @Deprecated
-  public Field(String name, String value, Store store, Index index) {
-    this(name, value, translateFieldType(store, index, TermVector.NO));
-  }
-
-  /**
-   * Create a field by specifying its name, value and how it will
-   * be saved in the index.
-   * 
-   * @param name The name of the field
-   * @param value The string to process
-   * @param store Whether value should be stored in the index
-   * @param index Whether the field should be indexed, and if so, if it should
-   *  be tokenized before indexing 
-   * @param termVector Whether term vector should be stored
-   * @throws NullPointerException if name or value is null
-   * @throws IllegalArgumentException in any of the following situations:
-   * 
         
-   *  
      • the field is neither stored nor indexed
        •  
-   *  
      • the field is not indexed but termVector is TermVector.YES
        • 
-   * 
       
-   *
-   * @deprecated Use {@link StringField}, {@link TextField} instead. */
-  @Deprecated
-  public Field(String name, String value, Store store, Index index, TermVector termVector) {  
-    this(name, value, translateFieldType(store, index, termVector));
-  }
-
-  /**
-   * Create a tokenized and indexed field that is not stored. Term vectors will
-   * not be stored.  The Reader is read only when the Document is added to the index,
-   * i.e. you may not close the Reader until {@link IndexWriter#addDocument}
-   * has been called.
-   * 
-   * @param name The name of the field
-   * @param reader The reader with the content
-   * @throws NullPointerException if name or reader is null
-   *
-   * @deprecated Use {@link TextField} instead.
-   */
-  @Deprecated
-  public Field(String name, Reader reader) {
-    this(name, reader, TermVector.NO);
-  }
-
-  /**
-   * Create a tokenized and indexed field that is not stored, optionally with 
-   * storing term vectors.  The Reader is read only when the Document is added to the index,
-   * i.e. you may not close the Reader until {@link IndexWriter#addDocument}
-   * has been called.
-   * 
-   * @param name The name of the field
-   * @param reader The reader with the content
-   * @param termVector Whether term vector should be stored
-   * @throws NullPointerException if name or reader is null
-   *
-   * @deprecated Use {@link TextField} instead.
-   */ 
-  @Deprecated
-  public Field(String name, Reader reader, TermVector termVector) {
-    this(name, reader, translateFieldType(Store.NO, Index.ANALYZED, termVector));
-  }
-
-  /**
-   * Create a tokenized and indexed field that is not stored. Term vectors will
-   * not be stored. This is useful for pre-analyzed fields.
-   * The TokenStream is read only when the Document is added to the index,
-   * i.e. you may not close the TokenStream until {@link IndexWriter#addDocument}
-   * has been called.
-   * 
-   * @param name The name of the field
-   * @param tokenStream The TokenStream with the content
-   * @throws NullPointerException if name or tokenStream is null
-   *
-   * @deprecated Use {@link TextField} instead
-   */ 
-  @Deprecated
-  public Field(String name, TokenStream tokenStream) {
-    this(name, tokenStream, TermVector.NO);
-  }
-
-  /**
-   * Create a tokenized and indexed field that is not stored, optionally with 
-   * storing term vectors.  This is useful for pre-analyzed fields.
-   * The TokenStream is read only when the Document is added to the index,
-   * i.e. you may not close the TokenStream until {@link IndexWriter#addDocument}
-   * has been called.
-   * 
-   * @param name The name of the field
-   * @param tokenStream The TokenStream with the content
-   * @param termVector Whether term vector should be stored
-   * @throws NullPointerException if name or tokenStream is null
-   *
-   * @deprecated Use {@link TextField} instead
-   */ 
-  @Deprecated
-  public Field(String name, TokenStream tokenStream, TermVector termVector) {
-    this(name, tokenStream, translateFieldType(Store.NO, Index.ANALYZED, termVector));
-  }
-
-  /**
-   * Create a stored field with binary value. Optionally the value may be compressed.
-   * 
-   * @param name The name of the field
-   * @param value The binary value
-   *
-   * @deprecated Use {@link StoredField} instead.
-   */
-  @Deprecated
-  public Field(String name, byte[] value) {
-    this(name, value, translateFieldType(Store.YES, Index.NO, TermVector.NO));
-  }
-
-  /**
-   * Create a stored field with binary value. Optionally the value may be compressed.
-   * 
-   * @param name The name of the field
-   * @param value The binary value
-   * @param offset Starting offset in value where this Field's bytes are
-   * @param length Number of bytes to use for this Field, starting at offset
-   *
-   * @deprecated Use {@link StoredField} instead.
-   */
-  @Deprecated
-  public Field(String name, byte[] value, int offset, int length) {
-    this(name, value, offset, length, translateFieldType(Store.YES, Index.NO, TermVector.NO));
+    NO
   }
 }
diff --git a/lucene/core/src/java/org/apache/lucene/document/FieldType.java b/lucene/core/src/java/org/apache/lucene/document/FieldType.java
index dd9b6ac..07cf30d 100644
--- a/lucene/core/src/java/org/apache/lucene/document/FieldType.java
+++ b/lucene/core/src/java/org/apache/lucene/document/FieldType.java
@@ -35,7 +35,7 @@ public class FieldType implements IndexableFieldType {
 
   private boolean indexed;
   private boolean stored;
-  private boolean tokenized;
+  private boolean tokenized = true;
   private boolean storeTermVectors;
   private boolean storeTermVectorOffsets;
   private boolean storeTermVectorPositions;
diff --git a/lucene/core/src/java/org/apache/lucene/document/FloatDocValuesField.java b/lucene/core/src/java/org/apache/lucene/document/FloatDocValuesField.java
index 3cee1fa..6add5e5 100644
--- a/lucene/core/src/java/org/apache/lucene/document/FloatDocValuesField.java
+++ b/lucene/core/src/java/org/apache/lucene/document/FloatDocValuesField.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.document;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -32,6 +32,7 @@ import org.apache.lucene.index.DocValues;
  * 
      
  * If you also need to store the value, you should add a
  * separate {@link StoredField} instance.
+ * @see DocValues for further information
  * */
 
 public class FloatDocValuesField extends Field {
diff --git a/lucene/core/src/java/org/apache/lucene/document/FloatField.java b/lucene/core/src/java/org/apache/lucene/document/FloatField.java
index 8a3dbf7..dcb5ea7 100644
--- a/lucene/core/src/java/org/apache/lucene/document/FloatField.java
+++ b/lucene/core/src/java/org/apache/lucene/document/FloatField.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.document;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -30,14 +30,14 @@ import org.apache.lucene.util.NumericUtils;
  * for efficient range filtering and sorting. Here's an example usage:
  * 
  * 
      - * document.add(new FloatField(name, 6.0F));
+ * document.add(new FloatField(name, 6.0F, Field.Store.NO));
  * 
      
  * 
  * For optimal performance, re-use the FloatField and
  * {@link Document} instance for more than one document:
  * 
  * 
      - *  FloatField field = new FloatField(name, 0.0F);
+ *  FloatField field = new FloatField(name, 0.0F, Field.Store.NO);
  *  Document document = new Document();
  *  document.add(field);
  * 
@@ -59,10 +59,6 @@ import org.apache.lucene.util.NumericUtils;
  * {@link org.apache.lucene.search.SortField.Type#FLOAT}. FloatField 
  * values can also be loaded directly from {@link FieldCache}.
      
  *
- * 
      By default, a FloatField's value is not stored but
- * is indexed for range filtering and sorting.  You can use
- * {@link StoredField} to also store the value.
- *
  * 
      You may add the same field name as an FloatField to
  * the same document more than once.  Range querying and
  * filtering will be the logical OR of all values; so a range query
@@ -118,21 +114,32 @@ import org.apache.lucene.util.NumericUtils;
 
 public final class FloatField extends Field {
   
-  public static final FieldType TYPE = new FieldType();
+  public static final FieldType TYPE_NOT_STORED = new FieldType();
+  static {
+    TYPE_NOT_STORED.setIndexed(true);
+    TYPE_NOT_STORED.setTokenized(true);
+    TYPE_NOT_STORED.setOmitNorms(true);
+    TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
+    TYPE_NOT_STORED.setNumericType(FieldType.NumericType.FLOAT);
+    TYPE_NOT_STORED.freeze();
+  }
+
+  public static final FieldType TYPE_STORED = new FieldType();
   static {
-    TYPE.setIndexed(true);
-    TYPE.setTokenized(true);
-    TYPE.setOmitNorms(true);
-    TYPE.setIndexOptions(IndexOptions.DOCS_ONLY);
-    TYPE.setNumericType(FieldType.NumericType.FLOAT);
-    TYPE.freeze();
+    TYPE_STORED.setIndexed(true);
+    TYPE_STORED.setTokenized(true);
+    TYPE_STORED.setOmitNorms(true);
+    TYPE_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
+    TYPE_STORED.setNumericType(FieldType.NumericType.FLOAT);
+    TYPE_STORED.setStored(true);
+    TYPE_STORED.freeze();
   }
 
-  /** Creates an LongField with the provided value
+  /** Creates a stored or un-stored FloatField with the provided value
    *  and default precisionStep {@link
    *  NumericUtils#PRECISION_STEP_DEFAULT} (4). */
-  public FloatField(String name, float value) {
-    super(name, TYPE);
+  public FloatField(String name, float value, Store stored) {
+    super(name, stored == Store.YES ? TYPE_STORED : TYPE_NOT_STORED);
     fieldsData = Float.valueOf(value);
   }
   
diff --git a/lucene/core/src/java/org/apache/lucene/document/IntDocValuesField.java b/lucene/core/src/java/org/apache/lucene/document/IntDocValuesField.java
index f769af3..7b10662 100644
--- a/lucene/core/src/java/org/apache/lucene/document/IntDocValuesField.java
+++ b/lucene/core/src/java/org/apache/lucene/document/IntDocValuesField.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.document;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -32,6 +32,7 @@ import org.apache.lucene.index.DocValues;
  * 
      
  * If you also need to store the value, you should add a
  * separate {@link StoredField} instance.
+ * @see DocValues for further information
  * */
 
 public class IntDocValuesField extends Field {
diff --git a/lucene/core/src/java/org/apache/lucene/document/IntField.java b/lucene/core/src/java/org/apache/lucene/document/IntField.java
index 27c3358..648da43 100644
--- a/lucene/core/src/java/org/apache/lucene/document/IntField.java
+++ b/lucene/core/src/java/org/apache/lucene/document/IntField.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.document;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -30,14 +30,14 @@ import org.apache.lucene.util.NumericUtils;
  * for efficient range filtering and sorting. Here's an example usage:
  * 
  * 
      - * document.add(new IntField(name, 6));
+ * document.add(new IntField(name, 6, Field.Store.NO));
  * 
      
  * 
  * For optimal performance, re-use the IntField and
  * {@link Document} instance for more than one document:
  * 
  * 
      - *  IntField field = new IntField(name, 6);
+ *  IntField field = new IntField(name, 6, Field.Store.NO);
  *  Document document = new Document();
  *  document.add(field);
  * 
@@ -59,10 +59,6 @@ import org.apache.lucene.util.NumericUtils;
  * {@link org.apache.lucene.search.SortField.Type#INT}. IntField 
  * values can also be loaded directly from {@link FieldCache}.
      
  *
- * 
      By default, a IntField's value is not stored but
- * is indexed for range filtering and sorting.  You can use
- * {@link StoredField} to also store the value.
- *
  * 
      You may add the same field name as an IntField to
  * the same document more than once.  Range querying and
  * filtering will be the logical OR of all values; so a range query
@@ -118,21 +114,32 @@ import org.apache.lucene.util.NumericUtils;
 
 public final class IntField extends Field {
   
-  public static final FieldType TYPE = new FieldType();
+  public static final FieldType TYPE_NOT_STORED = new FieldType();
+  static {
+    TYPE_NOT_STORED.setIndexed(true);
+    TYPE_NOT_STORED.setTokenized(true);
+    TYPE_NOT_STORED.setOmitNorms(true);
+    TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
+    TYPE_NOT_STORED.setNumericType(FieldType.NumericType.INT);
+    TYPE_NOT_STORED.freeze();
+  }
+
+  public static final FieldType TYPE_STORED = new FieldType();
   static {
-    TYPE.setIndexed(true);
-    TYPE.setTokenized(true);
-    TYPE.setOmitNorms(true);
-    TYPE.setIndexOptions(IndexOptions.DOCS_ONLY);
-    TYPE.setNumericType(FieldType.NumericType.INT);
-    TYPE.freeze();
+    TYPE_STORED.setIndexed(true);
+    TYPE_STORED.setTokenized(true);
+    TYPE_STORED.setOmitNorms(true);
+    TYPE_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
+    TYPE_STORED.setNumericType(FieldType.NumericType.INT);
+    TYPE_STORED.setStored(true);
+    TYPE_STORED.freeze();
   }
 
-  /** Creates an IntField with the provided value
+  /** Creates a stored or un-stored IntField with the provided value
    *  and default precisionStep {@link
    *  NumericUtils#PRECISION_STEP_DEFAULT} (4). */
-  public IntField(String name, int value) {
-    super(name, TYPE);
+  public IntField(String name, int value, Store stored) {
+    super(name, stored == Store.YES ? TYPE_STORED : TYPE_NOT_STORED);
     fieldsData = Integer.valueOf(value);
   }
   
diff --git a/lucene/core/src/java/org/apache/lucene/document/LongDocValuesField.java b/lucene/core/src/java/org/apache/lucene/document/LongDocValuesField.java
index 7b629c0..0b78eb9 100644
--- a/lucene/core/src/java/org/apache/lucene/document/LongDocValuesField.java
+++ b/lucene/core/src/java/org/apache/lucene/document/LongDocValuesField.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.document;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -32,6 +32,7 @@ import org.apache.lucene.index.DocValues;
  * 
      
  * If you also need to store the value, you should add a
  * separate {@link StoredField} instance.
+ * @see DocValues for further information
  * */
 
 public class LongDocValuesField extends Field {
diff --git a/lucene/core/src/java/org/apache/lucene/document/LongField.java b/lucene/core/src/java/org/apache/lucene/document/LongField.java
index b673a64..fb41558 100644
--- a/lucene/core/src/java/org/apache/lucene/document/LongField.java
+++ b/lucene/core/src/java/org/apache/lucene/document/LongField.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.document;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -30,14 +30,14 @@ import org.apache.lucene.util.NumericUtils;
  * for efficient range filtering and sorting. Here's an example usage:
  * 
  * 
      - * document.add(new LongField(name, 6L));
+ * document.add(new LongField(name, 6L, Field.Store.NO));
  * 
      
  * 
  * For optimal performance, re-use the LongField and
  * {@link Document} instance for more than one document:
  * 
  * 
      - *  LongField field = new LongField(name, 0L);
+ *  LongField field = new LongField(name, 0L, Field.Store.NO);
  *  Document document = new Document();
  *  document.add(field);
  * 
@@ -69,10 +69,6 @@ import org.apache.lucene.util.NumericUtils;
  * {@link org.apache.lucene.search.SortField.Type#LONG}. LongField 
  * values can also be loaded directly from {@link FieldCache}.
      
  *
- * 
      By default, a LongField's value is not stored but
- * is indexed for range filtering and sorting.  You can use
- * {@link StoredField} to also store the value.
- *
  * 
      You may add the same field name as an LongField to
  * the same document more than once.  Range querying and
  * filtering will be the logical OR of all values; so a range query
@@ -128,21 +124,32 @@ import org.apache.lucene.util.NumericUtils;
 
 public final class LongField extends Field {
   
-  public static final FieldType TYPE = new FieldType();
+  public static final FieldType TYPE_NOT_STORED = new FieldType();
+  static {
+    TYPE_NOT_STORED.setIndexed(true);
+    TYPE_NOT_STORED.setTokenized(true);
+    TYPE_NOT_STORED.setOmitNorms(true);
+    TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
+    TYPE_NOT_STORED.setNumericType(FieldType.NumericType.LONG);
+    TYPE_NOT_STORED.freeze();
+  }
+
+  public static final FieldType TYPE_STORED = new FieldType();
   static {
-    TYPE.setIndexed(true);
-    TYPE.setTokenized(true);
-    TYPE.setOmitNorms(true);
-    TYPE.setIndexOptions(IndexOptions.DOCS_ONLY);
-    TYPE.setNumericType(FieldType.NumericType.LONG);
-    TYPE.freeze();
+    TYPE_STORED.setIndexed(true);
+    TYPE_STORED.setTokenized(true);
+    TYPE_STORED.setOmitNorms(true);
+    TYPE_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
+    TYPE_STORED.setNumericType(FieldType.NumericType.LONG);
+    TYPE_STORED.setStored(true);
+    TYPE_STORED.freeze();
   }
 
-  /** Creates an LongField with the provided value
+  /** Creates a stored or un-stored LongField with the provided value
    *  and default precisionStep {@link
    *  NumericUtils#PRECISION_STEP_DEFAULT} (4). */
-  public LongField(String name, long value) {
-    super(name, TYPE);
+  public LongField(String name, long value, Store stored) {
+    super(name, stored == Store.YES ? TYPE_STORED : TYPE_NOT_STORED);
     fieldsData = Long.valueOf(value);
   }
   
diff --git a/lucene/core/src/java/org/apache/lucene/document/PackedLongDocValuesField.java b/lucene/core/src/java/org/apache/lucene/document/PackedLongDocValuesField.java
index c29e0d9..eceff09 100644
--- a/lucene/core/src/java/org/apache/lucene/document/PackedLongDocValuesField.java
+++ b/lucene/core/src/java/org/apache/lucene/document/PackedLongDocValuesField.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.document;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -35,6 +35,8 @@ import org.apache.lucene.index.AtomicReader;      // javadocs
  * 
      
  * If you also need to store the value, you should add a
  * separate {@link StoredField} instance.
+ * 
+ * @see DocValues for further information
  * */
 
 public class PackedLongDocValuesField extends Field {
diff --git a/lucene/core/src/java/org/apache/lucene/document/ShortDocValuesField.java b/lucene/core/src/java/org/apache/lucene/document/ShortDocValuesField.java
index 6d50f6e..2b08803 100644
--- a/lucene/core/src/java/org/apache/lucene/document/ShortDocValuesField.java
+++ b/lucene/core/src/java/org/apache/lucene/document/ShortDocValuesField.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.document;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -32,6 +32,8 @@ import org.apache.lucene.index.DocValues;
  * 
      
  * If you also need to store the value, you should add a
  * separate {@link StoredField} instance.
+ * 
+ * @see DocValues for further information
  * */
 
 public class ShortDocValuesField extends Field {
diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedBytesDocValuesField.java b/lucene/core/src/java/org/apache/lucene/document/SortedBytesDocValuesField.java
index 70c0a80..07892ce 100644
--- a/lucene/core/src/java/org/apache/lucene/document/SortedBytesDocValuesField.java
+++ b/lucene/core/src/java/org/apache/lucene/document/SortedBytesDocValuesField.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.document;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -33,6 +33,8 @@ import org.apache.lucene.util.BytesRef;
  * 
      
  * If you also need to store the value, you should add a
  * separate {@link StoredField} instance.
+ * 
+ * @see DocValues for further information
  * */
 
 public class SortedBytesDocValuesField extends Field {
diff --git a/lucene/core/src/java/org/apache/lucene/document/StoredField.java b/lucene/core/src/java/org/apache/lucene/document/StoredField.java
index 100980a..692768a 100644
--- a/lucene/core/src/java/org/apache/lucene/document/StoredField.java
+++ b/lucene/core/src/java/org/apache/lucene/document/StoredField.java
@@ -4,7 +4,7 @@ import org.apache.lucene.index.IndexReader; // javadocs
 import org.apache.lucene.search.IndexSearcher; // javadocs
 import org.apache.lucene.util.BytesRef;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/document/StraightBytesDocValuesField.java b/lucene/core/src/java/org/apache/lucene/document/StraightBytesDocValuesField.java
index 50f7931..a59dd47 100644
--- a/lucene/core/src/java/org/apache/lucene/document/StraightBytesDocValuesField.java
+++ b/lucene/core/src/java/org/apache/lucene/document/StraightBytesDocValuesField.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.document;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -36,6 +36,8 @@ import org.apache.lucene.util.BytesRef;
  * 
      
  * If you also need to store the value, you should add a
  * separate {@link StoredField} instance.
+ * 
+ * @see DocValues for further information
  * */
 
 public class StraightBytesDocValuesField extends Field {
diff --git a/lucene/core/src/java/org/apache/lucene/document/StringField.java b/lucene/core/src/java/org/apache/lucene/document/StringField.java
index 6127272..0629c7d 100644
--- a/lucene/core/src/java/org/apache/lucene/document/StringField.java
+++ b/lucene/core/src/java/org/apache/lucene/document/StringField.java
@@ -2,7 +2,7 @@ package org.apache.lucene.document;
 
 import org.apache.lucene.index.FieldInfo.IndexOptions;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -23,38 +23,36 @@ import org.apache.lucene.index.FieldInfo.IndexOptions;
  *  String value is indexed as a single token.  For example
  *  this might be used for a 'country' field or an 'id'
  *  field, or any field that you intend to use for sorting
- *  or access through the field cache.
- *
- *  
      This field's value is not stored by default; use the
- *  {@link StringField#TYPE_STORED} type (pass it to new
- *  Field) to store the value. */
+ *  or access through the field cache. */
 
 public final class StringField extends Field {
 
   /** Indexed, not tokenized, omits norms, indexes
    *  DOCS_ONLY, not stored. */
-  public static final FieldType TYPE_UNSTORED = new FieldType();
+  public static final FieldType TYPE_NOT_STORED = new FieldType();
 
   /** Indexed, not tokenized, omits norms, indexes
    *  DOCS_ONLY, stored */
   public static final FieldType TYPE_STORED = new FieldType();
 
   static {
-    TYPE_UNSTORED.setIndexed(true);
-    TYPE_UNSTORED.setOmitNorms(true);
-    TYPE_UNSTORED.setIndexOptions(IndexOptions.DOCS_ONLY);
-    TYPE_UNSTORED.freeze();
+    TYPE_NOT_STORED.setIndexed(true);
+    TYPE_NOT_STORED.setOmitNorms(true);
+    TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
+    TYPE_NOT_STORED.setTokenized(false);
+    TYPE_NOT_STORED.freeze();
 
     TYPE_STORED.setIndexed(true);
-    TYPE_STORED.setStored(true);
     TYPE_STORED.setOmitNorms(true);
     TYPE_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
+    TYPE_STORED.setStored(true);
+    TYPE_STORED.setTokenized(false);
     TYPE_STORED.freeze();
   }
 
-  /** Creates a new un-stored StringField */
-  public StringField(String name, String value) {
-    super(name, value, TYPE_UNSTORED);
+  /** Creates a new StringField. */
+  public StringField(String name, String value, Store stored) {
+    super(name, value, stored == Store.YES ? TYPE_STORED : TYPE_NOT_STORED);
   }
 
   @Override
diff --git a/lucene/core/src/java/org/apache/lucene/document/TextField.java b/lucene/core/src/java/org/apache/lucene/document/TextField.java
index 7ab8737..d30fca2 100644
--- a/lucene/core/src/java/org/apache/lucene/document/TextField.java
+++ b/lucene/core/src/java/org/apache/lucene/document/TextField.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.document;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -23,45 +23,41 @@ import org.apache.lucene.analysis.TokenStream;
 
 /** A field that is indexed and tokenized, without term
  *  vectors.  For example this would be used on a 'body'
- *  field, that contains the bulk of a document's text.
- * 
- *  This field's value is not stored by default; use the
- *  {@link TextField#TYPE_STORED} type (pass it to new
- *  Field) to store the value. */
+ *  field, that contains the bulk of a document's text. */
 
 public final class TextField extends Field {
 
   /* Indexed, tokenized, not stored. */
-  public static final FieldType TYPE_UNSTORED = new FieldType();
+  public static final FieldType TYPE_NOT_STORED = new FieldType();
 
   /* Indexed, tokenized, stored. */
   public static final FieldType TYPE_STORED = new FieldType();
 
   static {
-    TYPE_UNSTORED.setIndexed(true);
-    TYPE_UNSTORED.setTokenized(true);
-    TYPE_UNSTORED.freeze();
+    TYPE_NOT_STORED.setIndexed(true);
+    TYPE_NOT_STORED.setTokenized(true);
+    TYPE_NOT_STORED.freeze();
 
     TYPE_STORED.setIndexed(true);
-    TYPE_STORED.setStored(true);
     TYPE_STORED.setTokenized(true);
+    TYPE_STORED.setStored(true);
     TYPE_STORED.freeze();
   }
 
   // TODO: add sugar for term vectors...?
 
-  /** Creates a new un-stored TextField */
-  public TextField(String name, Reader reader) {
-    super(name, reader, TextField.TYPE_UNSTORED);
+  /** Creates a new TextField with Reader value. */
+  public TextField(String name, Reader reader, Store store) {
+    super(name, reader, store == Store.YES ? TYPE_STORED : TYPE_NOT_STORED);
   }
 
-  /** Creates a new un-stored TextField */
-  public TextField(String name, String value) {
-    super(name, value, TextField.TYPE_UNSTORED);
+  /** Creates a new TextField with String value. */
+  public TextField(String name, String value, Store store) {
+    super(name, value, store == Store.YES ? TYPE_STORED : TYPE_NOT_STORED);
   }
   
-  /** Creates a new un-stored TextField */
+  /** Creates a new un-stored TextField with TokenStream value. */
   public TextField(String name, TokenStream stream) {
-    super(name, stream, TextField.TYPE_UNSTORED);
+    super(name, stream, TYPE_NOT_STORED);
   }
 }
diff --git a/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java b/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java
index 108afc7..445e6d2 100644
--- a/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -22,7 +22,6 @@ import java.io.IOException;
 import org.apache.lucene.search.SearcherManager; // javadocs
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.ReaderUtil;         // for javadocs
 
 /** {@code AtomicReader} is an abstract class, providing an interface for accessing an
  index.  Search of an index is done entirely through this abstract interface,
@@ -60,19 +59,6 @@ public abstract class AtomicReader extends IndexReader {
     return readerContext;
   }
 
-  /** 
-   * Returns true if there are norms stored for this field.
-   * @deprecated (4.0) use {@link #getFieldInfos()} and check {@link FieldInfo#hasNorms()} 
-   *                   for the field instead.
-   */
-  @Deprecated
-  public final boolean hasNorms(String field) throws IOException {
-    ensureOpen();
-    // note: using normValues(field) != null would potentially cause i/o
-    FieldInfo fi = getFieldInfos().fieldInfo(field);
-    return fi != null && fi.hasNorms();
-  }
-
   /**
    * Returns {@link Fields} for this reader.
    * This method may return null if the reader has no
diff --git a/lucene/core/src/java/org/apache/lucene/index/AtomicReaderContext.java b/lucene/core/src/java/org/apache/lucene/index/AtomicReaderContext.java
index 38f3127..66ac94d 100644
--- a/lucene/core/src/java/org/apache/lucene/index/AtomicReaderContext.java
+++ b/lucene/core/src/java/org/apache/lucene/index/AtomicReaderContext.java
@@ -1,6 +1,9 @@
 package org.apache.lucene.index;
 
-/**
+import java.util.Collections;
+import java.util.List;
+
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -28,7 +31,7 @@ public final class AtomicReaderContext extends IndexReaderContext {
   public final int docBase;
   
   private final AtomicReader reader;
-  private final AtomicReaderContext[] leaves;
+  private final List leaves;
   
   /**
    * Creates a new {@link AtomicReaderContext} 
@@ -39,7 +42,7 @@ public final class AtomicReaderContext extends IndexReaderContext {
     this.ord = leafOrd;
     this.docBase = leafDocBase;
     this.reader = reader;
-    this.leaves = isTopLevel ? new AtomicReaderContext[] { this } : null;
+    this.leaves = isTopLevel ? Collections.singletonList(this) : null;
   }
   
   AtomicReaderContext(AtomicReader atomicReader) {
@@ -47,12 +50,15 @@ public final class AtomicReaderContext extends IndexReaderContext {
   }
   
   @Override
-  public AtomicReaderContext[] leaves() {
+  public List leaves() {
+    if (!isTopLevel)
+      throw new UnsupportedOperationException("This is not a top-level context.");
+    assert leaves != null;
     return leaves;
   }
   
   @Override
-  public IndexReaderContext[] children() {
+  public List children() {
     return null;
   }
   
diff --git a/lucene/core/src/java/org/apache/lucene/index/AutomatonTermsEnum.java b/lucene/core/src/java/org/apache/lucene/index/AutomatonTermsEnum.java
index f41c98c..980fce8 100644
--- a/lucene/core/src/java/org/apache/lucene/index/AutomatonTermsEnum.java
+++ b/lucene/core/src/java/org/apache/lucene/index/AutomatonTermsEnum.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -75,7 +75,7 @@ class AutomatonTermsEnum extends FilteredTermsEnum {
    * 
      
    * @param compiled CompiledAutomaton
    */
-  public AutomatonTermsEnum(TermsEnum tenum, CompiledAutomaton compiled) throws IOException {
+  public AutomatonTermsEnum(TermsEnum tenum, CompiledAutomaton compiled) {
     super(tenum);
     this.finite = compiled.finite;
     this.runAutomaton = compiled.runAutomaton;
diff --git a/lucene/core/src/java/org/apache/lucene/index/BaseCompositeReader.java b/lucene/core/src/java/org/apache/lucene/index/BaseCompositeReader.java
index d97e0b6..ddd0781 100644
--- a/lucene/core/src/java/org/apache/lucene/index/BaseCompositeReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/BaseCompositeReader.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -18,9 +18,11 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
 
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.ReaderUtil;
 
 /** Base class for implementing {@link CompositeReader}s based on an array
  * of sub-readers. The implementing class has to add code for
@@ -47,12 +49,16 @@ import org.apache.lucene.util.ReaderUtil;
  * @lucene.internal
  */
 public abstract class BaseCompositeReader extends CompositeReader {
-  protected final R[] subReaders;
-  protected final int[] starts;       // 1st docno for each reader
+  private final R[] subReaders;
+  private final int[] starts;       // 1st docno for each reader
   private final int maxDoc;
   private final int numDocs;
   private final boolean hasDeletions;
-  
+
+  /** List view solely for {@link #getSequentialSubReaders()},
+   * for effectiveness the array is used internally. */
+  private final List subReadersList;
+
   /**
    * Constructs a {@code BaseCompositeReader} on the given subReaders.
    * @param subReaders the wrapped sub-readers. This array is returned by
@@ -61,8 +67,9 @@ public abstract class BaseCompositeReader extends Composi
    * cloned and not protected for modification, the subclass is responsible 
    * to do this.
    */
-  protected BaseCompositeReader(R[] subReaders) throws IOException {
+  protected BaseCompositeReader(R[] subReaders) {
     this.subReaders = subReaders;
+    this.subReadersList = Collections.unmodifiableList(Arrays.asList(subReaders));
     starts = new int[subReaders.length + 1];    // build starts array
     int maxDoc = 0, numDocs = 0;
     boolean hasDeletions = false;
@@ -70,6 +77,9 @@ public abstract class BaseCompositeReader extends Composi
       starts[i] = maxDoc;
       final IndexReader r = subReaders[i];
       maxDoc += r.maxDoc();      // compute maxDocs
+      if (maxDoc < 0 /* overflow */) {
+        throw new IllegalArgumentException("Too many documents, composite IndexReaders cannot exceed " + Integer.MAX_VALUE);
+      }
       numDocs += r.numDocs();    // compute numDocs
       if (r.hasDeletions()) {
         hasDeletions = true;
@@ -102,7 +112,7 @@ public abstract class BaseCompositeReader extends Composi
   }
 
   @Override
-  public final void document(int docID, StoredFieldVisitor visitor) throws CorruptIndexException, IOException {
+  public final void document(int docID, StoredFieldVisitor visitor) throws IOException {
     ensureOpen();
     final int i = readerIndex(docID);                          // find subreader num
     subReaders[i].document(docID - starts[i], visitor);    // dispatch to subreader
@@ -132,8 +142,16 @@ public abstract class BaseCompositeReader extends Composi
     return ReaderUtil.subIndex(docID, this.starts);
   }
   
+  /** Helper method for subclasses to get the docBase of the given sub-reader index. */
+  protected final int readerBase(int readerIndex) {
+    if (readerIndex < 0 || readerIndex >= subReaders.length) {
+      throw new IllegalArgumentException("readerIndex must be >= 0 and < getSequentialSubReaders().size()");
+    }
+    return this.starts[readerIndex];
+  }
+  
   @Override
-  public final R[] getSequentialSubReaders() {
-    return subReaders;
+  public final List getSequentialSubReaders() {
+    return subReadersList;
   }
 }
diff --git a/lucene/core/src/java/org/apache/lucene/index/BitsSlice.java b/lucene/core/src/java/org/apache/lucene/index/BitsSlice.java
new file mode 100644
index 0000000..7870072
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/BitsSlice.java
@@ -0,0 +1,51 @@
+package org.apache.lucene.index;
+
+import org.apache.lucene.util.Bits;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Exposes a slice of an existing Bits as a new Bits.
+ *
+ * @lucene.internal
+ */
+final class BitsSlice implements Bits {
+  private final Bits parent;
+  private final int start;
+  private final int length;
+
+  // start is inclusive; end is exclusive (length = end-start)
+  public BitsSlice(Bits parent, ReaderSlice slice) {
+    this.parent = parent;
+    this.start = slice.start;
+    this.length = slice.length;
+    assert length >= 0: "length=" + length;
+  }
+    
+  public boolean get(int doc) {
+    if (doc >= length) {
+      throw new RuntimeException("doc " + doc + " is out of bounds 0 .. " + (length-1));
+    }
+    assert doc < length: "doc=" + doc + " length=" + length;
+    return parent.get(doc+start);
+  }
+
+  public int length() {
+    return length;
+  }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/index/BufferedDeletes.java b/lucene/core/src/java/org/apache/lucene/index/BufferedDeletes.java
index 8c1d805..afd1eef 100644
--- a/lucene/core/src/java/org/apache/lucene/index/BufferedDeletes.java
+++ b/lucene/core/src/java/org/apache/lucene/index/BufferedDeletes.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -93,7 +93,7 @@ class BufferedDeletes {
     } else {
       String s = "gen=" + gen;
       if (numTermDeletes.get() != 0) {
-        s += " " + numTermDeletes.get() + " deleted terms (unique count=" + terms.size() + ") terms=" + terms.keySet();
+        s += " " + numTermDeletes.get() + " deleted terms (unique count=" + terms.size() + ")";
       }
       if (queries.size() != 0) {
         s += " " + queries.size() + " deleted queries";
diff --git a/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java b/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java
index 2aedf3c..7bcf822 100644
--- a/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java
+++ b/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/ByteSliceReader.java b/lucene/core/src/java/org/apache/lucene/index/ByteSliceReader.java
index 99241db..dd20ca5 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ByteSliceReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ByteSliceReader.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/ByteSliceWriter.java b/lucene/core/src/java/org/apache/lucene/index/ByteSliceWriter.java
index 5c8b921..c0a3c2c 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ByteSliceWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ByteSliceWriter.java
@@ -3,7 +3,7 @@ package org.apache.lucene.index;
 import org.apache.lucene.store.DataOutput;
 import org.apache.lucene.util.ByteBlockPool;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
index 4ee5c87..c6281bb 100644
--- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
+++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -34,8 +34,6 @@ import org.apache.lucene.document.FieldType; // for javadocs
 import org.apache.lucene.index.DocValues.SortedSource;
 import org.apache.lucene.index.DocValues.Source;
 import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.store.IOContext;
@@ -643,12 +641,10 @@ public class CheckIndex {
       }
       for (FieldInfo info : fieldInfos) {
         if (info.hasNorms()) {
-          assert reader.hasNorms(info.name); // deprecated path
           DocValues dv = reader.normValues(info.name);
           checkDocValues(dv, info.name, info.getNormType(), reader.maxDoc());
           ++status.totFields;
         } else {
-          assert !reader.hasNorms(info.name); // deprecated path
           if (reader.normValues(info.name) != null) {
             throw new RuntimeException("field: " + info.name + " should omit norms but has them!");
           }
@@ -671,7 +667,7 @@ public class CheckIndex {
    * checks Fields api is consistent with itself.
    * searcher is optional, to verify with queries. Can be null.
    */
-  private Status.TermIndexStatus checkFields(Fields fields, Bits liveDocs, int maxDoc, FieldInfos fieldInfos, IndexSearcher searcher, boolean doPrint) throws IOException {
+  private Status.TermIndexStatus checkFields(Fields fields, Bits liveDocs, int maxDoc, FieldInfos fieldInfos, boolean doPrint) throws IOException {
     // TODO: we should probably return our own stats thing...?!
     
     final Status.TermIndexStatus status = new Status.TermIndexStatus();
@@ -685,6 +681,7 @@ public class CheckIndex {
     DocsEnum docs = null;
     DocsEnum docsAndFreqs = null;
     DocsAndPositionsEnum postings = null;
+    DocsAndPositionsEnum offsets = null;
     
     String lastField = null;
     final FieldsEnum fieldsEnum = fields.iterator();
@@ -760,6 +757,7 @@ public class CheckIndex {
         docs = termsEnum.docs(liveDocs, docs, false);
         docsAndFreqs = termsEnum.docs(liveDocs, docsAndFreqs, true);
         postings = termsEnum.docsAndPositions(liveDocs, postings, false);
+        offsets = termsEnum.docsAndPositions(liveDocs, offsets, true);
         
         if (hasOrd) {
           long ord = -1;
@@ -783,19 +781,29 @@ public class CheckIndex {
         final DocsEnum docsAndFreqs2;
         final boolean hasPositions;
         final boolean hasFreqs;
-        if (postings != null) {
+        final boolean hasOffsets;
+        if (offsets != null) {
+          docs2 = postings = offsets;
+          docsAndFreqs2 = postings = offsets;
+          hasOffsets = true;
+          hasPositions = true;
+          hasFreqs = true;
+        } else if (postings != null) {
           docs2 = postings;
           docsAndFreqs2 = postings;
+          hasOffsets = false;
           hasPositions = true;
           hasFreqs = true;
         } else if (docsAndFreqs != null) {
           docs2 = docsAndFreqs;
           docsAndFreqs2 = docsAndFreqs;
+          hasOffsets = false;
           hasPositions = false;
           hasFreqs = true;
         } else {
           docs2 = docs;
           docsAndFreqs2 = null;
+          hasOffsets = false;
           hasPositions = false;
           hasFreqs = false;
         }
@@ -830,6 +838,7 @@ public class CheckIndex {
           lastDoc = doc;
           
           int lastPos = -1;
+          int lastOffset = 0;
           if (hasPositions) {
             for(int j=0;j totDocCountNoDeletes) {
+              throw new RuntimeException("more postings with deletes=" + totDocCount + " than without=" + totDocCountNoDeletes);
+            }
+            
+            if (totDocCountNoDeletes != totDocFreq) {
+              throw new RuntimeException("docfreqs=" + totDocFreq + " != recomputed docfreqs=" + totDocCountNoDeletes);
+            }
           }
         }
       }
@@ -1121,7 +1185,6 @@ public class CheckIndex {
     Status.TermIndexStatus status;
     final int maxDoc = reader.maxDoc();
     final Bits liveDocs = reader.getLiveDocs();
-    final IndexSearcher is = new IndexSearcher(reader);
 
     try {
       if (infoStream != null) {
@@ -1129,13 +1192,12 @@ public class CheckIndex {
       }
 
       final Fields fields = reader.fields();
-      status = checkFields(fields, liveDocs, maxDoc, fieldInfos, is, true);
+      status = checkFields(fields, liveDocs, maxDoc, fieldInfos, true);
       if (liveDocs != null) {
         if (infoStream != null) {
           infoStream.print("    test (ignoring deletes): terms, freq, prox...");
         }
-        // TODO: can we make a IS that ignores all deletes?
-        checkFields(fields, null, maxDoc, fieldInfos, null, true);
+        checkFields(fields, null, maxDoc, fieldInfos, true);
       }
     } catch (Throwable e) {
       msg("ERROR: " + e);
@@ -1352,10 +1414,10 @@ public class CheckIndex {
 
         if (tfv != null) {
           // First run with no deletions:
-          checkFields(tfv, null, 1, fieldInfos, null, false);
+          checkFields(tfv, null, 1, fieldInfos, false);
 
           // Again, with the one doc deleted:
-          checkFields(tfv, onlyDocIsDeleted, 1, fieldInfos, null, false);
+          checkFields(tfv, onlyDocIsDeleted, 1, fieldInfos, false);
 
           // Only agg stats if the doc is live:
           final boolean doStats = liveDocs == null || liveDocs.get(j);
@@ -1669,6 +1731,7 @@ public class CheckIndex {
                          "              You can't use this with the -fix option\n" +
                          "  -dir-impl X: use a specific " + FSDirectory.class.getSimpleName() + " implementation. " +
                          		"If no package is specified the " + FSDirectory.class.getPackage().getName() + " package will be used.\n" +
+                         "\n" +
                          "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" +
                          "documents (perhaps many) to be permanently removed from the index.  Always make\n" +
                          "a backup copy of your index before running this!  Do not run this tool on an index\n" +
diff --git a/lucene/core/src/java/org/apache/lucene/index/CoalescedDeletes.java b/lucene/core/src/java/org/apache/lucene/index/CoalescedDeletes.java
index 4c89a76..b1dd8d0 100644
--- a/lucene/core/src/java/org/apache/lucene/index/CoalescedDeletes.java
+++ b/lucene/core/src/java/org/apache/lucene/index/CoalescedDeletes.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/CompositeReader.java b/lucene/core/src/java/org/apache/lucene/index/CompositeReader.java
index 47b4822..277adf6 100644
--- a/lucene/core/src/java/org/apache/lucene/index/CompositeReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/CompositeReader.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -17,6 +17,8 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
+import java.util.List;
+
 import org.apache.lucene.search.SearcherManager; // javadocs
 import org.apache.lucene.store.*;
 
@@ -28,12 +30,12 @@ import org.apache.lucene.store.*;
   by wrapping composite readers with {@link SlowCompositeReaderWrapper}.
  
  
      IndexReader instances for indexes on disk are usually constructed
- with a call to one of the static DirectoryReader,open() methods,
+ with a call to one of the static DirectoryReader.open() methods,
  e.g. {@link DirectoryReader#open(Directory)}. {@link DirectoryReader} implements
  the {@code CompositeReader} interface, it is not possible to directly get postings.
  
       Concrete subclasses of IndexReader are usually constructed with a call to
  one of the static open() methods, e.g. {@link
- #open(Directory)}.
+ DirectoryReader#open(Directory)}.
 
  
       For efficiency, in this API documents are often referred to via
  document numbers, non-negative integers which each name a unique
@@ -63,12 +65,12 @@ public abstract class CompositeReader extends IndexReader {
     final StringBuilder buffer = new StringBuilder();
     buffer.append(getClass().getSimpleName());
     buffer.append('(');
-    final IndexReader[] subReaders = getSequentialSubReaders();
+    final List subReaders = getSequentialSubReaders();
     assert subReaders != null;
-    if (subReaders.length > 0) {
-      buffer.append(subReaders[0]);
-      for (int i = 1; i < subReaders.length; ++i) {
-        buffer.append(" ").append(subReaders[i]);
+    if (!subReaders.isEmpty()) {
+      buffer.append(subReaders.get(0));
+      for (int i = 1, c = subReaders.size(); i < c; ++i) {
+        buffer.append(" ").append(subReaders.get(i));
       }
     }
     buffer.append(')');
@@ -81,11 +83,8 @@ public abstract class CompositeReader extends IndexReader {
    *  If this method returns an empty array, that means this
    *  reader is a null reader (for example a MultiReader
    *  that has no sub readers).
-   *  
      Warning: Don't modify the returned array!
-   *  Doing so will corrupt the internal structure of this
-   *  {@code CompositeReader}.
    */
-  public abstract IndexReader[] getSequentialSubReaders();
+  public abstract List getSequentialSubReaders();
 
   @Override
   public final CompositeReaderContext getTopReaderContext() {
diff --git a/lucene/core/src/java/org/apache/lucene/index/CompositeReaderContext.java b/lucene/core/src/java/org/apache/lucene/index/CompositeReaderContext.java
index 308bf32..2ea78ab 100644
--- a/lucene/core/src/java/org/apache/lucene/index/CompositeReaderContext.java
+++ b/lucene/core/src/java/org/apache/lucene/index/CompositeReaderContext.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -17,16 +17,18 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
-import java.io.IOException;
-import org.apache.lucene.util.ReaderUtil;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
 
 /**
  * {@link IndexReaderContext} for {@link CompositeReader} instance.
  * @lucene.experimental
  */
 public final class CompositeReaderContext extends IndexReaderContext {
-  private final IndexReaderContext[] children;
-  private final AtomicReaderContext[] leaves;
+  private final List children;
+  private final List leaves;
   private final CompositeReader reader;
   
   static CompositeReaderContext create(CompositeReader reader) {
@@ -38,34 +40,37 @@ public final class CompositeReaderContext extends IndexReaderContext {
    * not top-level readers in the current context
    */
   CompositeReaderContext(CompositeReaderContext parent, CompositeReader reader,
-      int ordInParent, int docbaseInParent, IndexReaderContext[] children) {
+      int ordInParent, int docbaseInParent, List children) {
     this(parent, reader, ordInParent, docbaseInParent, children, null);
   }
   
   /**
    * Creates a {@link CompositeReaderContext} for top-level readers with parent set to null
    */
-  CompositeReaderContext(CompositeReader reader, IndexReaderContext[] children, AtomicReaderContext[] leaves) {
+  CompositeReaderContext(CompositeReader reader, List children, List leaves) {
     this(null, reader, 0, 0, children, leaves);
   }
   
   private CompositeReaderContext(CompositeReaderContext parent, CompositeReader reader,
-      int ordInParent, int docbaseInParent, IndexReaderContext[] children,
-      AtomicReaderContext[] leaves) {
+      int ordInParent, int docbaseInParent, List children,
+      List leaves) {
     super(parent, ordInParent, docbaseInParent);
-    this.children = children;
-    this.leaves = leaves;
+    this.children = Collections.unmodifiableList(children);
+    this.leaves = leaves == null ? null : Collections.unmodifiableList(leaves);
     this.reader = reader;
   }
 
   @Override
-  public AtomicReaderContext[] leaves() {
+  public List leaves() throws UnsupportedOperationException {
+    if (!isTopLevel)
+      throw new UnsupportedOperationException("This is not a top-level context.");
+    assert leaves != null;
     return leaves;
   }
   
   
   @Override
-  public IndexReaderContext[] children() {
+  public List children() {
     return children;
   }
   
@@ -76,13 +81,11 @@ public final class CompositeReaderContext extends IndexReaderContext {
   
   private static final class Builder {
     private final CompositeReader reader;
-    private final AtomicReaderContext[] leaves;
-    private int leafOrd = 0;
+    private final List leaves = new ArrayList();
     private int leafDocBase = 0;
     
     public Builder(CompositeReader reader) {
       this.reader = reader;
-      leaves = new AtomicReaderContext[numLeaves(reader)];
     }
     
     public CompositeReaderContext build() {
@@ -92,14 +95,14 @@ public final class CompositeReaderContext extends IndexReaderContext {
     private IndexReaderContext build(CompositeReaderContext parent, IndexReader reader, int ord, int docBase) {
       if (reader instanceof AtomicReader) {
         final AtomicReader ar = (AtomicReader) reader;
-        final AtomicReaderContext atomic = new AtomicReaderContext(parent, ar, ord, docBase, leafOrd, leafDocBase);
-        leaves[leafOrd++] = atomic;
+        final AtomicReaderContext atomic = new AtomicReaderContext(parent, ar, ord, docBase, leaves.size(), leafDocBase);
+        leaves.add(atomic);
         leafDocBase += reader.maxDoc();
         return atomic;
       } else {
         final CompositeReader cr = (CompositeReader) reader;
-        final IndexReader[] sequentialSubReaders = cr.getSequentialSubReaders();
-        final IndexReaderContext[] children = new IndexReaderContext[sequentialSubReaders.length];
+        final List sequentialSubReaders = cr.getSequentialSubReaders();
+        final List children = Arrays.asList(new IndexReaderContext[sequentialSubReaders.size()]);
         final CompositeReaderContext newParent;
         if (parent == null) {
           newParent = new CompositeReaderContext(cr, children, leaves);
@@ -107,30 +110,15 @@ public final class CompositeReaderContext extends IndexReaderContext {
           newParent = new CompositeReaderContext(parent, cr, ord, docBase, children);
         }
         int newDocBase = 0;
-        for (int i = 0; i < sequentialSubReaders.length; i++) {
-          children[i] = build(newParent, sequentialSubReaders[i], i, newDocBase);
-          newDocBase += sequentialSubReaders[i].maxDoc();
+        for (int i = 0, c = sequentialSubReaders.size(); i < c; i++) {
+          final IndexReader r = sequentialSubReaders.get(i);
+          children.set(i, build(newParent, r, i, newDocBase));
+          newDocBase += r.maxDoc();
         }
+        assert newDocBase == cr.maxDoc();
         return newParent;
       }
     }
-    
-    private int numLeaves(IndexReader reader) {
-      final int[] numLeaves = new int[1];
-      try {
-        new ReaderUtil.Gather(reader) {
-          @Override
-          protected void add(int base, AtomicReader r) {
-            numLeaves[0]++;
-          }
-        }.run();
-      } catch (IOException ioe) {
-        // won't happen
-        throw new RuntimeException(ioe);
-      }
-      return numLeaves[0];
-    }
-    
   }
 
 }
\ No newline at end of file
diff --git a/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java b/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
index 95e4c23..05aa9b4 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -398,7 +398,7 @@ public class ConcurrentMergeScheduler extends MergeScheduler {
     MergePolicy.OneMerge runningMerge;
     private volatile boolean done;
 
-    public MergeThread(IndexWriter writer, MergePolicy.OneMerge startMerge) throws IOException {
+    public MergeThread(IndexWriter writer, MergePolicy.OneMerge startMerge) {
       this.tWriter = writer;
       this.startMerge = startMerge;
     }
diff --git a/lucene/core/src/java/org/apache/lucene/index/CorruptIndexException.java b/lucene/core/src/java/org/apache/lucene/index/CorruptIndexException.java
index 800d1d5..8fea843 100644
--- a/lucene/core/src/java/org/apache/lucene/index/CorruptIndexException.java
+++ b/lucene/core/src/java/org/apache/lucene/index/CorruptIndexException.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java b/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java
index d561a84..328ee47 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -22,7 +22,6 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.Map;
 
 import org.apache.lucene.search.SearcherManager; // javadocs
 import org.apache.lucene.store.Directory;
@@ -57,10 +56,9 @@ public abstract class DirectoryReader extends BaseCompositeReader
   /** Returns a IndexReader reading the index in the given
    *  Directory
    * @param directory the index directory
-   * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
    */
-  public static DirectoryReader open(final Directory directory) throws CorruptIndexException, IOException {
+  public static DirectoryReader open(final Directory directory) throws IOException {
     return StandardDirectoryReader.open(directory, null, DEFAULT_TERMS_INDEX_DIVISOR);
   }
   
@@ -77,10 +75,9 @@ public abstract class DirectoryReader extends BaseCompositeReader
    *  memory usage, at the expense of higher latency when
    *  loading a TermInfo.  The default value is 1.  Set this
    *  to -1 to skip loading the terms index entirely.
-   * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
    */
-  public static DirectoryReader open(final Directory directory, int termInfosIndexDivisor) throws CorruptIndexException, IOException {
+  public static DirectoryReader open(final Directory directory, int termInfosIndexDivisor) throws IOException {
     return StandardDirectoryReader.open(directory, null, termInfosIndexDivisor);
   }
   
@@ -103,17 +100,16 @@ public abstract class DirectoryReader extends BaseCompositeReader
    *
    * @lucene.experimental
    */
-  public static DirectoryReader open(final IndexWriter writer, boolean applyAllDeletes) throws CorruptIndexException, IOException {
+  public static DirectoryReader open(final IndexWriter writer, boolean applyAllDeletes) throws IOException {
     return writer.getReader(applyAllDeletes);
   }
 
   /** Expert: returns an IndexReader reading the index in the given
    *  {@link IndexCommit}.
    * @param commit the commit point to open
-   * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
    */
-  public static DirectoryReader open(final IndexCommit commit) throws CorruptIndexException, IOException {
+  public static DirectoryReader open(final IndexCommit commit) throws IOException {
     return StandardDirectoryReader.open(commit.getDirectory(), commit, DEFAULT_TERMS_INDEX_DIVISOR);
   }
 
@@ -130,10 +126,9 @@ public abstract class DirectoryReader extends BaseCompositeReader
    *  memory usage, at the expense of higher latency when
    *  loading a TermInfo.  The default value is 1.  Set this
    *  to -1 to skip loading the terms index entirely.
-   * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
    */
-  public static DirectoryReader open(final IndexCommit commit, int termInfosIndexDivisor) throws CorruptIndexException, IOException {
+  public static DirectoryReader open(final IndexCommit commit, int termInfosIndexDivisor) throws IOException {
     return StandardDirectoryReader.open(commit.getDirectory(), commit, termInfosIndexDivisor);
   }
 
@@ -253,7 +248,7 @@ public abstract class DirectoryReader extends BaseCompositeReader
    *  one commit point.  But if you're using a custom {@link
    *  IndexDeletionPolicy} then there could be many commits.
    *  Once you have a given commit, you can open a reader on
-   *  it by calling {@link IndexReader#open(IndexCommit)}
+   *  it by calling {@link DirectoryReader#open(IndexCommit)}
    *  There must be at least one commit in
    *  the Directory, else this method throws {@link
    *  IndexNotFoundException}.  Note that if a commit is in
@@ -312,9 +307,8 @@ public abstract class DirectoryReader extends BaseCompositeReader
    * Returns true if an index exists at the specified directory.
    * @param  directory the directory to check for an index
    * @return true if an index exists; false otherwise
-   * @throws IOException if there is a problem with accessing the index
    */
-  public static boolean indexExists(Directory directory) throws IOException {
+  public static boolean indexExists(Directory directory) {
     try {
       new SegmentInfos().read(directory);
       return true;
@@ -332,7 +326,7 @@ public abstract class DirectoryReader extends BaseCompositeReader
    * Subclasses of {@code DirectoryReader} should take care to not allow
    * modification of this internal array, e.g. {@link #doOpenIfChanged()}.
    */
-  protected DirectoryReader(Directory directory, AtomicReader[] segmentReaders) throws CorruptIndexException, IOException {
+  protected DirectoryReader(Directory directory, AtomicReader[] segmentReaders) {
     super(segmentReaders);
     this.directory = directory;
   }
@@ -349,32 +343,29 @@ public abstract class DirectoryReader extends BaseCompositeReader
    * If this reader does not support reopen, return {@code null}, so
    * client code is happy. This should be consistent with {@link #isCurrent}
    * (should always return {@code true}) if reopen is not supported.
-   * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
    * @return null if there are no changes; else, a new
    * DirectoryReader instance.
    */
-  protected abstract DirectoryReader doOpenIfChanged() throws CorruptIndexException, IOException;
+  protected abstract DirectoryReader doOpenIfChanged() throws IOException;
 
   /** Implement this method to support {@link #openIfChanged(DirectoryReader,IndexCommit)}.
    * If this reader does not support reopen from a specific {@link IndexCommit},
    * throw {@link UnsupportedOperationException}.
-   * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
    * @return null if there are no changes; else, a new
    * DirectoryReader instance.
    */
-  protected abstract DirectoryReader doOpenIfChanged(final IndexCommit commit) throws CorruptIndexException, IOException;
+  protected abstract DirectoryReader doOpenIfChanged(final IndexCommit commit) throws IOException;
 
   /** Implement this method to support {@link #openIfChanged(DirectoryReader,IndexWriter,boolean)}.
    * If this reader does not support reopen from {@link IndexWriter},
    * throw {@link UnsupportedOperationException}.
-   * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
    * @return null if there are no changes; else, a new
    * DirectoryReader instance.
    */
-  protected abstract DirectoryReader doOpenIfChanged(IndexWriter writer, boolean applyAllDeletes) throws CorruptIndexException, IOException;
+  protected abstract DirectoryReader doOpenIfChanged(IndexWriter writer, boolean applyAllDeletes) throws IOException;
 
   /**
    * Version number when this IndexReader was opened.
@@ -397,9 +388,9 @@ public abstract class DirectoryReader extends BaseCompositeReader
    *
    * 
      If instead this reader is a near real-time reader
    * (ie, obtained by a call to {@link
-   * IndexReader#open(IndexWriter,boolean)}, or by calling {@link #openIfChanged}
+   * DirectoryReader#open(IndexWriter,boolean)}, or by calling {@link #openIfChanged}
    * on a near real-time reader), then this method checks if
-   * either a new commmit has occurred, or any new
+   * either a new commit has occurred, or any new
    * uncommitted changes have taken place via the writer.
    * Note that even if the writer has only performed
    * merging, this method will still return false.
      
@@ -408,16 +399,15 @@ public abstract class DirectoryReader extends BaseCompositeReader
    * {@link #openIfChanged} to get a new reader that sees the
    * changes.
      
    *
-   * @throws CorruptIndexException if the index is corrupt
    * @throws IOException           if there is a low-level IO error
    */
-  public abstract boolean isCurrent() throws CorruptIndexException, IOException;
+  public abstract boolean isCurrent() throws IOException;
 
   /**
    * Expert: return the IndexCommit that this reader has opened.
    * 
      
    * @lucene.experimental
    */
-  public abstract IndexCommit getIndexCommit() throws CorruptIndexException, IOException;
+  public abstract IndexCommit getIndexCommit() throws IOException;
 
 }
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java b/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java
index 87b732f..30e3b5b 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumer.java b/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumer.java
index 58d0a14..efa8638 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumer.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java b/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java
index 38be3f8..2bfd420 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java b/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java
index 4f4ae70..0929970 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -29,6 +29,7 @@ import org.apache.lucene.codecs.DocValuesConsumer;
 import org.apache.lucene.codecs.FieldInfosWriter;
 import org.apache.lucene.codecs.PerDocConsumer;
 import org.apache.lucene.index.DocumentsWriterPerThread.DocState;
+import org.apache.lucene.index.TypePromoter.TypeCompatibility;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.IOUtils;
@@ -80,7 +81,7 @@ final class DocFieldProcessor extends DocConsumer {
     fieldsWriter.flush(state);
     consumer.flush(childFields, state);
 
-    for (DocValuesConsumerAndDocID consumer : docValues.values()) {
+    for (DocValuesConsumerHolder consumer : docValues.values()) {
       consumer.docValuesConsumer.finish(state.segmentInfo.getDocCount());
     }
     
@@ -271,7 +272,26 @@ final class DocFieldProcessor extends DocConsumer {
       }
       final DocValues.Type dvType = field.fieldType().docValueType();
       if (dvType != null) {
-        docValuesConsumer(dvType, docState, fp.fieldInfo).add(docState.docID, field);
+        DocValuesConsumerHolder docValuesConsumer = docValuesConsumer(dvType,
+            docState, fp.fieldInfo);
+        DocValuesConsumer consumer = docValuesConsumer.docValuesConsumer;
+        if (docValuesConsumer.compatibility == null) {
+          consumer.add(docState.docID, field);
+          docValuesConsumer.compatibility = new TypeCompatibility(dvType,
+              consumer.getValueSize());
+        } else if (docValuesConsumer.compatibility.isCompatible(dvType,
+            TypePromoter.getValueSize(dvType, field.binaryValue()))) {
+          consumer.add(docState.docID, field);
+        } else {
+          docValuesConsumer.compatibility.isCompatible(dvType,
+              TypePromoter.getValueSize(dvType, field.binaryValue()));
+          TypeCompatibility compatibility = docValuesConsumer.compatibility;
+          throw new IllegalArgumentException("Incompatible DocValues type: "
+              + dvType.name() + " size: "
+              + TypePromoter.getValueSize(dvType, field.binaryValue())
+              + " expected: " + " type: " + compatibility.getBaseType()
+              + " size: " + compatibility.getBaseSize());
+        }
       }
     }
 
@@ -308,30 +328,31 @@ final class DocFieldProcessor extends DocConsumer {
     }
   }
 
-  private static class DocValuesConsumerAndDocID {
+  private static class DocValuesConsumerHolder {
     // Only used to enforce that same DV field name is never
     // added more than once per doc:
-    public int docID;
+    int docID;
     final DocValuesConsumer docValuesConsumer;
+    TypeCompatibility compatibility;
 
-    public DocValuesConsumerAndDocID(DocValuesConsumer docValuesConsumer) {
+    public DocValuesConsumerHolder(DocValuesConsumer docValuesConsumer) {
       this.docValuesConsumer = docValuesConsumer;
     }
   }
 
-  final private Map docValues = new HashMap();
+  final private Map docValues = new HashMap();
   private PerDocConsumer perDocConsumer;
 
-  DocValuesConsumer docValuesConsumer(DocValues.Type valueType, DocState docState, FieldInfo fieldInfo) 
+  DocValuesConsumerHolder docValuesConsumer(DocValues.Type valueType, DocState docState, FieldInfo fieldInfo) 
       throws IOException {
-    DocValuesConsumerAndDocID docValuesConsumerAndDocID = docValues.get(fieldInfo.name);
+    DocValuesConsumerHolder docValuesConsumerAndDocID = docValues.get(fieldInfo.name);
     if (docValuesConsumerAndDocID != null) {
       if (docState.docID == docValuesConsumerAndDocID.docID) {
         throw new IllegalArgumentException("DocValuesField \"" + fieldInfo.name + "\" appears more than once in this document (only one value is allowed, per field)");
       }
       assert docValuesConsumerAndDocID.docID < docState.docID;
       docValuesConsumerAndDocID.docID = docState.docID;
-      return docValuesConsumerAndDocID.docValuesConsumer;
+      return docValuesConsumerAndDocID;
     }
 
     if (perDocConsumer == null) {
@@ -345,9 +366,10 @@ final class DocFieldProcessor extends DocConsumer {
     assert fieldInfo.getDocValuesType() == null || fieldInfo.getDocValuesType() == valueType;
     fieldInfo.setDocValuesType(valueType);
 
-    docValuesConsumerAndDocID = new DocValuesConsumerAndDocID(docValuesConsumer);
+    docValuesConsumerAndDocID = new DocValuesConsumerHolder(docValuesConsumer);
     docValuesConsumerAndDocID.docID = docState.docID;
     docValues.put(fieldInfo.name, docValuesConsumerAndDocID);
-    return docValuesConsumer;
+    return docValuesConsumerAndDocID;
   }
+  
 }
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java b/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java
index f0d3e30..32fad15 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocInverter.java b/lucene/core/src/java/org/apache/lucene/index/DocInverter.java
index 91a9b3a..6ea1eb5 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocInverter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocInverter.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java b/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java
index d8a3acd..0ef2a6f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -18,9 +18,12 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
+
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.util.IOUtils;
 
 /**
  * Holds state for inverting all occurrences of a single
@@ -78,6 +81,11 @@ final class DocInverterPerField extends DocFieldConsumerPerField {
         if (fieldType.omitNorms() && field.boost() != 1.0f) {
           throw new UnsupportedOperationException("You cannot set an index-time boost: norms are omitted for field '" + field.name() + "'");
         }
+        
+        // only bother checking offsets if something will consume them.
+        // TODO: after we fix analyzers, also check if termVectorOffsets will be indexed.
+        final boolean checkOffsets = fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
+        int lastStartOffset = 0;
 
         if (i > 0) {
           fieldState.position += docState.analyzer == null ? 0 : docState.analyzer.getPositionIncrementGap(fieldInfo.name);
@@ -87,6 +95,8 @@ final class DocInverterPerField extends DocFieldConsumerPerField {
         // reset the TokenStream to the first token
         stream.reset();
 
+        boolean success2 = false;
+
         try {
           boolean hasMoreTokens = stream.incrementToken();
 
@@ -109,8 +119,16 @@ final class DocInverterPerField extends DocFieldConsumerPerField {
             if (!hasMoreTokens) break;
 
             final int posIncr = posIncrAttribute.getPositionIncrement();
+            if (posIncr < 0) {
+              throw new IllegalArgumentException("position increment must be >=0 (got " + posIncr + ")");
+            }
+            if (fieldState.position == 0 && posIncr == 0) {
+              throw new IllegalArgumentException("first position increment must be > 0 (got 0)");
+            }
             int position = fieldState.position + posIncr;
             if (position > 0) {
+              // NOTE: confusing: this "mirrors" the
+              // position++ we do below
               position--;
             } else if (position < 0) {
               throw new IllegalArgumentException("position overflow for field '" + field.name() + "'");
@@ -122,6 +140,20 @@ final class DocInverterPerField extends DocFieldConsumerPerField {
 
             if (posIncr == 0)
               fieldState.numOverlap++;
+            
+            if (checkOffsets) {
+              int startOffset = fieldState.offset + offsetAttribute.startOffset();
+              int endOffset = fieldState.offset + offsetAttribute.endOffset();
+              if (startOffset < 0 || endOffset < startOffset) {
+                throw new IllegalArgumentException("startOffset must be non-negative, and endOffset must be >= startOffset, "
+                    + "startOffset=" + startOffset + ",endOffset=" + endOffset);
+              }
+              if (startOffset < lastStartOffset) {
+                throw new IllegalArgumentException("offsets must not go backwards startOffset=" 
+                     + startOffset + " is < lastStartOffset=" + lastStartOffset);
+              }
+              lastStartOffset = startOffset;
+            }
 
             boolean success = false;
             try {
@@ -147,8 +179,13 @@ final class DocInverterPerField extends DocFieldConsumerPerField {
           stream.end();
 
           fieldState.offset += offsetAttribute.endOffset();
+          success2 = true;
         } finally {
-          stream.close();
+          if (!success2) {
+            IOUtils.closeWhileHandlingException(stream);
+          } else {
+            stream.close();
+          }
         }
 
         fieldState.offset += docState.analyzer == null ? 0 : docState.analyzer.getOffsetGap(field);
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java b/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java
index 4b120b6..b84a95f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -175,7 +175,7 @@ public class DocTermOrds {
 
   /** Subclass inits w/ this, but be sure you then call
    *  uninvert, only once */
-  protected DocTermOrds(String field, int maxTermDocFreq, int indexIntervalBits) throws IOException {
+  protected DocTermOrds(String field, int maxTermDocFreq, int indexIntervalBits) {
     //System.out.println("DTO init field=" + field + " maxTDFreq=" + maxTermDocFreq);
     this.field = field;
     this.maxTermDocFreq = maxTermDocFreq;
@@ -220,6 +220,13 @@ public class DocTermOrds {
     return numTermsInField;
   }
 
+  /**
+   * @return Whether this DocTermOrds instance is empty.
+   */
+  public boolean isEmpty() {
+    return index == null;
+  }
+
   /** Subclass can override this */
   protected void visitTerm(TermsEnum te, int termNum) throws IOException {
   }
@@ -696,7 +703,7 @@ public class DocTermOrds {
     }
 
     @Override
-    public long ord() throws IOException {
+    public long ord() {
       return ordBase + ord;
     }
 
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValues.java b/lucene/core/src/java/org/apache/lucene/index/DocValues.java
index d0af613..8694c24 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocValues.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
  */
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Comparator;
 
 import org.apache.lucene.codecs.DocValuesFormat;
@@ -31,6 +32,7 @@ import org.apache.lucene.document.PackedLongDocValuesField; // javadocs
 import org.apache.lucene.document.ShortDocValuesField; // javadocs
 import org.apache.lucene.document.SortedBytesDocValuesField; // javadocs
 import org.apache.lucene.document.StraightBytesDocValuesField; // javadocs
+import org.apache.lucene.store.DataOutput;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.packed.PackedInts;
 
@@ -47,6 +49,17 @@ import org.apache.lucene.util.packed.PackedInts;
  * IndexReader.
  * 
      
  * {@link DocValues} are fully integrated into the {@link DocValuesFormat} API.
+ * 
      
+ * NOTE: DocValues is a strongly typed per-field API. Type changes within an
+ * indexing session can result in exceptions if the type has changed in a way that
+ * the previously give type for a field can't promote the value without losing
+ * information. For instance a field initially indexed with {@link Type#FIXED_INTS_32}
+ * can promote a value with {@link Type#FIXED_INTS_8} but can't promote
+ * {@link Type#FIXED_INTS_64}. During segment merging type-promotion exceptions are suppressed. 
+ * Fields will be promoted to their common denominator or automatically transformed
+ * into a 3rd type like {@link Type#BYTES_VAR_STRAIGHT} to prevent data loss and merge exceptions.
+ * This behavior is considered best-effort might change in future releases.
+ * 
      
  * 
  * @see Type for limitations and default implementation documentation
  * @see ByteDocValuesField for adding byte values to the index
@@ -195,7 +208,6 @@ public abstract class DocValues implements Closeable {
      * Returns a {@link BytesRef} for the given document id or throws an
      * {@link UnsupportedOperationException} if this source doesn't support
      * byte[] values.
-     * @throws IOException 
      * 
      * @throws UnsupportedOperationException
      *           if this source doesn't support byte[] values.
@@ -392,6 +404,18 @@ public abstract class DocValues implements Closeable {
       public Object getArray() {
         return null;
       }
+
+      @Override
+      public int get(int index, long[] arr, int off, int len) {
+        len = Math.min(len, size() - index);
+        Arrays.fill(arr, off, off+len, 0);
+        return len;
+      }
+
+      @Override
+      public long ramBytesUsed() {
+        return 0;
+      }
     };
 
     return new SortedSource(type, BytesRef.getUTF8SortedAsUnicodeComparator()) {
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java b/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java
index f001eae..b7d8f2b 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java b/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java
index 7b48dd0..bc239f4 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
index a933813..b66e088 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -133,8 +133,8 @@ final class DocumentsWriter {
   final DocumentsWriterFlushControl flushControl;
   
   final Codec codec;
-  DocumentsWriter(Codec codec, IndexWriterConfig config, Directory directory, IndexWriter writer, FieldNumbers globalFieldNumbers,
-      BufferedDeletesStream bufferedDeletesStream) throws IOException {
+  DocumentsWriter(Codec codec, LiveIndexWriterConfig config, Directory directory, IndexWriter writer, FieldNumbers globalFieldNumbers,
+      BufferedDeletesStream bufferedDeletesStream) {
     this.codec = codec;
     this.directory = directory;
     this.indexWriter = writer;
@@ -200,9 +200,8 @@ final class DocumentsWriter {
    *  updating the index files) and must discard all
    *  currently buffered docs.  This resets our state,
    *  discarding any docs added since last flush. */
-  synchronized void abort() throws IOException {
+  synchronized void abort() {
     boolean success = false;
-
     synchronized (this) {
       deleteQueue.clear();
     }
@@ -220,8 +219,6 @@ final class DocumentsWriter {
           if (perThread.isActive()) { // we might be closed
             try {
               perThread.dwpt.abort();
-            } catch (IOException ex) {
-              // continue
             } finally {
               perThread.dwpt.checkAndResetHasAborted();
               flushControl.doOnAbort(perThread);
@@ -233,6 +230,7 @@ final class DocumentsWriter {
           perThread.unlock();
         }
       }
+      flushControl.waitForFlush();
       success = true;
     } finally {
       if (infoStream.isEnabled("DW")) {
@@ -276,7 +274,7 @@ final class DocumentsWriter {
     flushControl.setClosed();
   }
 
-  private boolean preUpdate() throws CorruptIndexException, IOException {
+  private boolean preUpdate() throws IOException {
     ensureOpen();
     boolean maybeMerge = false;
     if (flushControl.anyStalledThreads() || flushControl.numQueuedFlushes() > 0) {
@@ -325,7 +323,7 @@ final class DocumentsWriter {
   }
 
   boolean updateDocuments(final Iterable> docs, final Analyzer analyzer,
-                          final Term delTerm) throws CorruptIndexException, IOException {
+                          final Term delTerm) throws IOException {
     boolean maybeMerge = preUpdate();
 
     final ThreadState perThread = flushControl.obtainAndLock();
@@ -356,7 +354,7 @@ final class DocumentsWriter {
   }
 
   boolean updateDocument(final Iterable doc, final Analyzer analyzer,
-      final Term delTerm) throws CorruptIndexException, IOException {
+      final Term delTerm) throws IOException {
 
     boolean maybeMerge = preUpdate();
 
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java
index 29b0655..7ca5923 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements. See the NOTICE file distributed with this
  * work for additional information regarding copyright ownership. The ASF
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
index 39d2359..0005340 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements. See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -16,7 +16,7 @@ package org.apache.lucene.index;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import java.io.IOException;
+
 import java.util.ArrayList;
 import java.util.IdentityHashMap;
 import java.util.List;
@@ -26,7 +26,6 @@ import java.util.Queue;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState;
-import org.apache.lucene.index.DocumentsWriterStallControl.MemoryController;
 import org.apache.lucene.util.ThreadInterruptedException;
 
 /**
@@ -41,7 +40,7 @@ import org.apache.lucene.util.ThreadInterruptedException;
  * {@link IndexWriterConfig#getRAMPerThreadHardLimitMB()} to prevent address
  * space exhaustion.
  */
-final class DocumentsWriterFlushControl implements MemoryController {
+final class DocumentsWriterFlushControl  {
 
   private final long hardMaxBytesPerDWPT;
   private long activeBytes = 0;
@@ -65,10 +64,9 @@ final class DocumentsWriterFlushControl implements MemoryController {
   private final FlushPolicy flushPolicy;
   private boolean closed = false;
   private final DocumentsWriter documentsWriter;
-  private final IndexWriterConfig config;
+  private final LiveIndexWriterConfig config;
 
-  DocumentsWriterFlushControl(DocumentsWriter documentsWriter,
-      IndexWriterConfig config) {
+  DocumentsWriterFlushControl(DocumentsWriter documentsWriter, LiveIndexWriterConfig config) {
     this.stallControl = new DocumentsWriterStallControl();
     this.perThreadPool = documentsWriter.perThreadPool;
     this.flushPolicy = documentsWriter.flushPolicy;
@@ -89,7 +87,7 @@ final class DocumentsWriterFlushControl implements MemoryController {
     return flushBytes + activeBytes;
   }
   
-  public long stallLimitBytes() {
+  private long stallLimitBytes() {
     final double maxRamMB = config.getRAMBufferSizeMB();
     return maxRamMB != IndexWriterConfig.DISABLE_AUTO_FLUSH ? (long)(2 * (maxRamMB * 1024 * 1024)) : Long.MAX_VALUE;
   }
@@ -179,7 +177,7 @@ final class DocumentsWriterFlushControl implements MemoryController {
       }
       return flushingDWPT;
     } finally {
-      stallControl.updateStalled(this);
+      updateStallState();
       assert assertMemory();
     }
   }
@@ -193,13 +191,30 @@ final class DocumentsWriterFlushControl implements MemoryController {
       assert assertMemory();
     } finally {
       try {
-        stallControl.updateStalled(this);
+       updateStallState();
       } finally {
         notifyAll();
       }
     }
   }
   
+  private final void updateStallState() {
+    
+    assert Thread.holdsLock(this);
+    final long limit = stallLimitBytes();
+    /*
+     * we block indexing threads if net byte grows due to slow flushes
+     * yet, for small ram buffers and large documents we can easily
+     * reach the limit without any ongoing flushes. we need to ensure
+     * that we don't stall/block if an ongoing or pending flush can
+     * not free up enough memory to release the stall lock.
+     */
+    final boolean stall = ((activeBytes + flushBytes) > limit)  &&
+                          (activeBytes < limit) &&
+                          !closed;
+    stallControl.updateStalled(stall);
+  }
+  
   public synchronized void waitForFlush() {
     while (flushingWriters.size() != 0) {
       try {
@@ -239,7 +254,7 @@ final class DocumentsWriterFlushControl implements MemoryController {
       // Take it out of the loop this DWPT is stale
       perThreadPool.replaceForFlush(state, closed);
     } finally {
-      stallControl.updateStalled(this);
+      updateStallState();
     }
   }
 
@@ -289,7 +304,7 @@ final class DocumentsWriterFlushControl implements MemoryController {
       }
       return null;
     } finally {
-      stallControl.updateStalled(this);
+      updateStallState();
     }
   }
 
@@ -305,7 +320,7 @@ final class DocumentsWriterFlushControl implements MemoryController {
     synchronized (this) {
       final DocumentsWriterPerThread poll;
       if ((poll = flushQueue.poll()) != null) {
-        stallControl.updateStalled(this);
+        updateStallState();
         return poll;
       }
       fullFlush = this.fullFlush;
@@ -328,8 +343,10 @@ final class DocumentsWriterFlushControl implements MemoryController {
 
   synchronized void setClosed() {
     // set by DW to signal that we should not release new DWPT after close
-    this.closed = true;
-    perThreadPool.deactivateUnreleasedStates();
+    if (!closed) {
+      this.closed = true;
+      perThreadPool.deactivateUnreleasedStates();
+    }
   }
 
   /**
@@ -459,7 +476,7 @@ final class DocumentsWriterFlushControl implements MemoryController {
       assert assertBlockedFlushes(documentsWriter.deleteQueue);
       flushQueue.addAll(fullFlushBuffer);
       fullFlushBuffer.clear();
-      stallControl.updateStalled(this);
+      updateStallState();
     }
     assert assertActiveDeleteQueue(documentsWriter.deleteQueue);
   }
@@ -482,7 +499,7 @@ final class DocumentsWriterFlushControl implements MemoryController {
 
   void addFlushableState(ThreadState perThread) {
     if (documentsWriter.infoStream.isEnabled("DWFC")) {
-      documentsWriter.infoStream.message("DWFC", Thread.currentThread().getName() + ": addFlushableState " + perThread.dwpt);
+      documentsWriter.infoStream.message("DWFC", "addFlushableState " + perThread.dwpt);
     }
     final DocumentsWriterPerThread dwpt = perThread.dwpt;
     assert perThread.isHeldByCurrentThread();
@@ -538,7 +555,7 @@ final class DocumentsWriterFlushControl implements MemoryController {
       }
     } finally {
       fullFlush = false;
-      stallControl.updateStalled(this);
+      updateStallState();
     }
   }
   
@@ -553,27 +570,19 @@ final class DocumentsWriterFlushControl implements MemoryController {
     try {
       for (DocumentsWriterPerThread dwpt : flushQueue) {
         doAfterFlush(dwpt);
-        try {
-          dwpt.abort();
-        } catch (IOException ex) {
-          // continue
-        }
+        dwpt.abort();
       }
       for (BlockedFlush blockedFlush : blockedFlushes) {
         flushingWriters
             .put(blockedFlush.dwpt, Long.valueOf(blockedFlush.bytes));
         doAfterFlush(blockedFlush.dwpt);
-        try {
-          blockedFlush.dwpt.abort();
-        } catch (IOException ex) {
-          // continue
-        }
+        blockedFlush.dwpt.abort();
       }
     } finally {
       fullFlush = false;
       flushQueue.clear();
       blockedFlushes.clear();
-      stallControl.updateStalled(this);
+      updateStallState();
     }
   }
   
@@ -616,7 +625,13 @@ final class DocumentsWriterFlushControl implements MemoryController {
    * checked out DWPT are available
    */
   void waitIfStalled() {
-      stallControl.waitIfStalled();
+    if (documentsWriter.infoStream.isEnabled("DWFC")) {
+      documentsWriter.infoStream.message("DWFC",
+          "waitIfStalled: numFlushesPending: " + flushQueue.size()
+              + " netBytes: " + netBytes() + " flushBytes: " + flushBytes()
+              + " fullFlush: " + fullFlush);
+    }
+    stallControl.waitIfStalled();
   }
 
   /**
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java
index 312bf64..df4cc21 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java
@@ -1,5 +1,5 @@
 package org.apache.lucene.index;
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements. See the NOTICE file distributed with this
  * work for additional information regarding copyright ownership. The ASF
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
index 51c5c14..ec39336 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements. See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -135,7 +135,7 @@ class DocumentsWriterPerThread {
    *  updating the index files) and must discard all
    *  currently buffered docs.  This resets our state,
    *  discarding any docs added since last flush. */
-  void abort() throws IOException {
+  void abort() {
     hasAborted = aborting = true;
     try {
       if (infoStream.isEnabled("DWPT")) {
@@ -352,7 +352,7 @@ class DocumentsWriterPerThread {
     return docCount;
   }
   
-  private void finishDocument(Term delTerm) throws IOException {
+  private void finishDocument(Term delTerm) {
     /*
      * here we actually finish the document in two steps 1. push the delete into
      * the queue and update our slice. 2. increment the DWPT private document
@@ -412,7 +412,7 @@ class DocumentsWriterPerThread {
   }
 
   /** Reset after a flush */
-  private void doAfterFlush() throws IOException {
+  private void doAfterFlush() {
     segmentInfo = null;
     consumer.doAfterFlush();
     directory.getCreatedFiles().clear();
@@ -463,10 +463,6 @@ class DocumentsWriterPerThread {
       pendingDeletes.docIDs.clear();
     }
 
-    if (infoStream.isEnabled("DWPT")) {
-      infoStream.message("DWPT", "flush postings as segment " + flushState.segmentInfo.name + " numDocs=" + numDocsInRAM);
-    }
-
     if (aborting) {
       if (infoStream.isEnabled("DWPT")) {
         infoStream.message("DWPT", "flush: skip because aborting is set");
@@ -474,6 +470,10 @@ class DocumentsWriterPerThread {
       return null;
     }
 
+    if (infoStream.isEnabled("DWPT")) {
+      infoStream.message("DWPT", "flush postings as segment " + flushState.segmentInfo.name + " numDocs=" + numDocsInRAM);
+    }
+
     boolean success = false;
 
     try {
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java
index 1f18df8..a7a208f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java
@@ -1,5 +1,5 @@
 package org.apache.lucene.index;
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements. See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -36,7 +36,7 @@ import org.apache.lucene.util.SetOnce;
  * new {@link DocumentsWriterPerThread} instance.
  * 
      
  */
-abstract class DocumentsWriterPerThreadPool {
+abstract class DocumentsWriterPerThreadPool implements Cloneable {
   
   /**
    * {@link ThreadState} references and guards a
@@ -119,10 +119,10 @@ abstract class DocumentsWriterPerThreadPool {
     }
   }
 
-  private final ThreadState[] threadStates;
+  private ThreadState[] threadStates;
   private volatile int numThreadStatesActive;
-  private final SetOnce globalFieldMap = new SetOnce();
-  private final SetOnce documentsWriter = new SetOnce();
+  private SetOnce globalFieldMap = new SetOnce();
+  private SetOnce documentsWriter = new SetOnce();
   
   /**
    * Creates a new {@link DocumentsWriterPerThreadPool} with a given maximum of {@link ThreadState}s.
@@ -135,7 +135,7 @@ abstract class DocumentsWriterPerThreadPool {
     numThreadStatesActive = 0;
   }
 
-  void initialize(DocumentsWriter documentsWriter, FieldNumbers globalFieldMap, IndexWriterConfig config) {
+  void initialize(DocumentsWriter documentsWriter, FieldNumbers globalFieldMap, LiveIndexWriterConfig config) {
     this.documentsWriter.set(documentsWriter); // thread pool is bound to DW
     this.globalFieldMap.set(globalFieldMap);
     for (int i = 0; i < threadStates.length; i++) {
@@ -143,6 +143,23 @@ abstract class DocumentsWriterPerThreadPool {
       threadStates[i] = new ThreadState(new DocumentsWriterPerThread(documentsWriter.directory, documentsWriter, infos, documentsWriter.chain));
     }
   }
+
+  @Override
+  public DocumentsWriterPerThreadPool clone() {
+    // We should only be cloned before being used:
+    assert numThreadStatesActive == 0;
+    DocumentsWriterPerThreadPool clone;
+    try {
+      clone = (DocumentsWriterPerThreadPool) super.clone();
+    } catch (CloneNotSupportedException e) {
+      // should not happen
+      throw new RuntimeException(e);
+    }
+    clone.documentsWriter = new SetOnce();
+    clone.globalFieldMap = new SetOnce();
+    clone.threadStates = new ThreadState[threadStates.length];
+    return clone;
+  }
   
   /**
    * Returns the max number of {@link ThreadState} instances available in this
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterStallControl.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterStallControl.java
index 93b22e4..cfdb218 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterStallControl.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterStallControl.java
@@ -16,7 +16,8 @@ package org.apache.lucene.index;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import java.util.concurrent.locks.AbstractQueuedSynchronizer;
+import java.util.IdentityHashMap;
+import java.util.Map;
 
 import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState;
 import org.apache.lucene.util.ThreadInterruptedException;
@@ -37,105 +38,80 @@ import org.apache.lucene.util.ThreadInterruptedException;
  * continue indexing.
  */
 final class DocumentsWriterStallControl {
-  @SuppressWarnings("serial")
-  private static final class Sync extends AbstractQueuedSynchronizer {
-    volatile boolean hasBlockedThreads = false; // only with assert
-
-    Sync() {
-      setState(0);
-    }
-
-    boolean isHealthy() {
-      return getState() == 0;
-    }
-
-    boolean trySetStalled() {
-      int state = getState();
-      return compareAndSetState(state, state + 1);
-    }
-
-    boolean tryReset() {
-      final int oldState = getState();
-      if (oldState == 0) {
-        return true;
-      }
-      if (compareAndSetState(oldState, 0)) {
-        return releaseShared(0);
-      }
-      return false;
-    }
-
-    @Override
-    public int tryAcquireShared(int acquires) {
-      assert maybeSetHasBlocked(getState());
-      return getState() == 0 ? 1 : -1;
-    }
-
-    // only used for testing
-    private boolean maybeSetHasBlocked(int state) {
-      hasBlockedThreads |= getState() != 0;
-      return true;
-    }
-
-    @Override
-    public boolean tryReleaseShared(int newState) {
-      return (getState() == 0);
-    }
-  }
-
-  private final Sync sync = new Sync();
-  volatile boolean wasStalled = false; // only with asserts
-
-  boolean anyStalledThreads() {
-    return !sync.isHealthy();
-  }
-
+  
+  private volatile boolean stalled;
+  private int numWaiting; // only with assert
+  private boolean wasStalled; // only with assert
+  private final Map waiting = new IdentityHashMap(); // only with assert
+  
   /**
    * Update the stalled flag status. This method will set the stalled flag to
    * true iff the number of flushing
    * {@link DocumentsWriterPerThread} is greater than the number of active
    * {@link DocumentsWriterPerThread}. Otherwise it will reset the
-   * {@link DocumentsWriterStallControl} to healthy and release all threads waiting on
-   * {@link #waitIfStalled()}
+   * {@link DocumentsWriterStallControl} to healthy and release all threads
+   * waiting on {@link #waitIfStalled()}
    */
-  void updateStalled(MemoryController controller) {
-    do {
-      final long netBytes = controller.netBytes();
-      final long flushBytes = controller.flushBytes();
-      final long limit = controller.stallLimitBytes();
-      assert netBytes >= flushBytes;
-      assert limit > 0;
-      /*
-       * we block indexing threads if net byte grows due to slow flushes
-       * yet, for small ram buffers and large documents we can easily
-       * reach the limit without any ongoing flushes. we need to ensure
-       * that we don't stall/block if an ongoing or pending flush can 
-       * not free up enough memory to release the stall lock.
-       */
-      while (netBytes > limit && (netBytes - flushBytes) < limit) {
-        if (sync.trySetStalled()) {
-          assert wasStalled = true;
-          return;
-        }
-      }
-    } while (!sync.tryReset());
+  synchronized void updateStalled(boolean stalled) {
+    this.stalled = stalled;
+    if (stalled) {
+      wasStalled = true;
+    }
+    notifyAll();
   }
-
+  
+  /**
+   * Blocks if documents writing is currently in a stalled state. 
+   * 
+   */
   void waitIfStalled() {
-    try {
-      sync.acquireSharedInterruptibly(0);
-    } catch (InterruptedException e) {
-      throw new ThreadInterruptedException(e);
+    if (stalled) {
+      synchronized (this) {
+        if (stalled) { // react on the first wakeup call!
+          // don't loop here, higher level logic will re-stall!
+          try {
+            assert incWaiters();
+            wait();
+            assert  decrWaiters();
+          } catch (InterruptedException e) {
+            throw new ThreadInterruptedException(e);
+          }
+        }
+      }
     }
   }
   
-  boolean hasBlocked() { // for tests
-    return sync.hasBlockedThreads;
+  boolean anyStalledThreads() {
+    return stalled;
+  }
+  
+  
+  private boolean incWaiters() {
+    numWaiting++;
+    assert waiting.put(Thread.currentThread(), Boolean.TRUE) == null;
+    
+    return numWaiting > 0;
+  }
+  
+  private boolean decrWaiters() {
+    numWaiting--;
+    assert waiting.remove(Thread.currentThread()) != null;
+    return numWaiting >= 0;
   }
   
-  static interface MemoryController {
-    long netBytes();
-    long flushBytes();
-    long stallLimitBytes();
+  synchronized boolean hasBlocked() { // for tests
+    return numWaiting > 0;
+  }
+  
+  boolean isHealthy() { // for tests
+    return !stalled; // volatile read!
+  }
+  
+  synchronized boolean isThreadQueued(Thread t) { // for tests
+    return waiting.containsKey(t);
+  }
+
+  synchronized boolean wasStalled() { // for tests
+    return wasStalled;
   }
 }
diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
index 678f4ff..3ab2706 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -85,12 +85,11 @@ public final class FieldInfo {
       this.storeTermVector = false;
       this.storePayloads = false;
       this.omitNorms = false;
-      this.indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
+      this.indexOptions = null;
       this.normType = null;
     }
     this.attributes = attributes;
     assert checkConsistency();
-    assert indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !storePayloads;
   }
 
   private boolean checkConsistency() {
@@ -99,17 +98,16 @@ public final class FieldInfo {
       assert !storePayloads;
       assert !omitNorms;
       assert normType == null;
-      assert indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
+      assert indexOptions == null;
     } else {
       assert indexOptions != null;
       if (omitNorms) {
         assert normType == null;
       }
+      // Cannot store payloads unless positions are indexed:
+      assert indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !this.storePayloads;
     }
 
-    // Cannot store payloads unless positions are indexed:
-    assert indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !this.storePayloads;
-
     return true;
   }
 
@@ -131,15 +129,18 @@ public final class FieldInfo {
         this.normType = null;
       }
       if (this.indexOptions != indexOptions) {
-        // downgrade
-        this.indexOptions = this.indexOptions.compareTo(indexOptions) < 0 ? this.indexOptions : indexOptions;
+        if (this.indexOptions == null) {
+          this.indexOptions = indexOptions;
+        } else {
+          // downgrade
+          this.indexOptions = this.indexOptions.compareTo(indexOptions) < 0 ? this.indexOptions : indexOptions;
+        }
         if (this.indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
           // cannot store payloads if we don't store positions:
           this.storePayloads = false;
         }
       }
     }
-    assert this.indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !this.storePayloads;
     assert checkConsistency();
   }
 
@@ -148,7 +149,7 @@ public final class FieldInfo {
     assert checkConsistency();
   }
   
-  /** @return IndexOptions for the field */
+  /** @return IndexOptions for the field, or null if the field is not indexed */
   public IndexOptions getIndexOptions() {
     return indexOptions;
   }
diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
index 24ea30a..f0ff871 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java b/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java
index 717ba22..c302ab8 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/Fields.java b/lucene/core/src/java/org/apache/lucene/index/Fields.java
index 4cfe57a..71edf67 100644
--- a/lucene/core/src/java/org/apache/lucene/index/Fields.java
+++ b/lucene/core/src/java/org/apache/lucene/index/Fields.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldsEnum.java b/lucene/core/src/java/org/apache/lucene/index/FieldsEnum.java
index a0afada..17528c0 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FieldsEnum.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FieldsEnum.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java b/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java
index e0c4b58..866a007 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -348,7 +348,7 @@ public class FilterAtomicReader extends AtomicReader {
   }
 
   @Override
-  public void document(int docID, StoredFieldVisitor visitor) throws CorruptIndexException, IOException {
+  public void document(int docID, StoredFieldVisitor visitor) throws IOException {
     ensureOpen();
     in.document(docID, visitor);
   }
diff --git a/lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java b/lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java
index 0995ac1..13532bc 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -82,7 +82,7 @@ public abstract class FilteredTermsEnum extends TermsEnum {
    * 
      You can only use this method, if you keep the default
    * implementation of {@link #nextSeekTerm}.
    */
-  protected final void setInitialSeekTerm(BytesRef term) throws IOException {
+  protected final void setInitialSeekTerm(BytesRef term) {
     this.initialSeekTerm = term;
   }
   
diff --git a/lucene/core/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java b/lucene/core/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java
index 8da969d..fbb87d8 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements. See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/FlushPolicy.java b/lucene/core/src/java/org/apache/lucene/index/FlushPolicy.java
index 08bf838..9645479 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FlushPolicy.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FlushPolicy.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements. See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -50,9 +50,9 @@ import org.apache.lucene.util.SetOnce;
  * @see DocumentsWriterPerThread
  * @see IndexWriterConfig#setFlushPolicy(FlushPolicy)
  */
-abstract class FlushPolicy {
-  protected final SetOnce writer = new SetOnce();
-  protected IndexWriterConfig indexWriterConfig;
+abstract class FlushPolicy implements Cloneable {
+  protected SetOnce writer = new SetOnce();
+  protected LiveIndexWriterConfig indexWriterConfig;
 
   /**
    * Called for each delete term. If this is a delete triggered due to an update
@@ -132,4 +132,17 @@ abstract class FlushPolicy {
     return true;
   }
 
+  @Override
+  public FlushPolicy clone() {
+    FlushPolicy clone;
+    try {
+      clone = (FlushPolicy) super.clone();
+    } catch (CloneNotSupportedException e) {
+      // should not happen
+      throw new RuntimeException(e);
+    }
+    clone.writer = new SetOnce();
+    clone.indexWriterConfig = null;
+    return clone;
+  }
 }
diff --git a/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java b/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
index b678ac1..75eeaed 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -114,10 +114,10 @@ final class FreqProxTermsWriter extends TermsHashConsumer {
   }
 
   @Override
-  void finishDocument(TermsHash termsHash) throws IOException {
+  void finishDocument(TermsHash termsHash) {
   }
 
   @Override
-  void startDocument() throws IOException {
+  void startDocument() {
   }
 }
diff --git a/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java b/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
index 3cbf88a..425f158 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -42,10 +42,9 @@ final class FreqProxTermsWriterPerField extends TermsHashConsumerPerField implem
   final FieldInfo fieldInfo;
   final DocumentsWriterPerThread.DocState docState;
   final FieldInvertState fieldState;
-  IndexOptions indexOptions;
-  private boolean writeFreq;
-  private boolean writeProx;
-  private boolean writeOffsets;
+  private boolean hasFreq;
+  private boolean hasProx;
+  private boolean hasOffsets;
   PayloadAttribute payloadAttribute;
   OffsetAttribute offsetAttribute;
 
@@ -60,7 +59,7 @@ final class FreqProxTermsWriterPerField extends TermsHashConsumerPerField implem
 
   @Override
   int getStreamCount() {
-    if (!writeProx) {
+    if (!hasProx) {
       return 1;
     } else {
       return 2;
@@ -77,7 +76,7 @@ final class FreqProxTermsWriterPerField extends TermsHashConsumerPerField implem
   boolean hasPayloads;
 
   @Override
-  void skippingLongTerm() throws IOException {}
+  void skippingLongTerm() {}
 
   public int compareTo(FreqProxTermsWriterPerField other) {
     return fieldInfo.name.compareTo(other.fieldInfo.name);
@@ -92,10 +91,14 @@ final class FreqProxTermsWriterPerField extends TermsHashConsumerPerField implem
   }
 
   private void setIndexOptions(IndexOptions indexOptions) {
-    this.indexOptions = indexOptions;
-    writeFreq = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
-    writeProx = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
-    writeOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+    if (indexOptions == null) {
+      // field could later be updated with indexed=true, so set everything on
+      hasFreq = hasProx = hasOffsets = true;
+    } else {
+      hasFreq = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+      hasProx = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+      hasOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+    }
   }
 
   @Override
@@ -115,7 +118,7 @@ final class FreqProxTermsWriterPerField extends TermsHashConsumerPerField implem
     } else {
       payloadAttribute = null;
     }
-    if (writeOffsets) {
+    if (hasOffsets) {
       offsetAttribute = fieldState.attributeSource.addAttribute(OffsetAttribute.class);
     } else {
       offsetAttribute = null;
@@ -124,8 +127,8 @@ final class FreqProxTermsWriterPerField extends TermsHashConsumerPerField implem
 
   void writeProx(final int termID, int proxCode) {
     //System.out.println("writeProx termID=" + termID + " proxCode=" + proxCode);
-    assert writeProx;
-    final Payload payload;
+    assert hasProx;
+    final BytesRef payload;
     if (payloadAttribute == null) {
       payload = null;
     } else {
@@ -135,7 +138,7 @@ final class FreqProxTermsWriterPerField extends TermsHashConsumerPerField implem
     if (payload != null && payload.length > 0) {
       termsHashPerField.writeVInt(1, (proxCode<<1)|1);
       termsHashPerField.writeVInt(1, payload.length);
-      termsHashPerField.writeBytes(1, payload.data, payload.offset, payload.length);
+      termsHashPerField.writeBytes(1, payload.bytes, payload.offset, payload.length);
       hasPayloads = true;
     } else {
       termsHashPerField.writeVInt(1, proxCode<<1);
@@ -145,15 +148,16 @@ final class FreqProxTermsWriterPerField extends TermsHashConsumerPerField implem
     postings.lastPositions[termID] = fieldState.position;
   }
 
-  void writeOffsets(final int termID, int prevOffset) {
-    assert writeOffsets;
-    final int startOffset = offsetAttribute.startOffset();
-    final int endOffset = offsetAttribute.endOffset();
+  void writeOffsets(final int termID, int offsetAccum) {
+    assert hasOffsets;
+    final int startOffset = offsetAccum + offsetAttribute.startOffset();
+    final int endOffset = offsetAccum + offsetAttribute.endOffset();
     //System.out.println("writeOffsets termID=" + termID + " prevOffset=" + prevOffset + " startOff=" + startOffset + " endOff=" + endOffset);
-    termsHashPerField.writeVInt(1, startOffset - prevOffset);
+    FreqProxPostingsArray postings = (FreqProxPostingsArray) termsHashPerField.postingsArray;
+    assert startOffset - postings.lastOffsets[termID] >= 0;
+    termsHashPerField.writeVInt(1, startOffset - postings.lastOffsets[termID]);
     termsHashPerField.writeVInt(1, endOffset - startOffset);
 
-    FreqProxPostingsArray postings = (FreqProxPostingsArray) termsHashPerField.postingsArray;
     postings.lastOffsets[termID] = startOffset;
   }
 
@@ -165,18 +169,18 @@ final class FreqProxTermsWriterPerField extends TermsHashConsumerPerField implem
 
     FreqProxPostingsArray postings = (FreqProxPostingsArray) termsHashPerField.postingsArray;
     postings.lastDocIDs[termID] = docState.docID;
-    if (!writeFreq) {
+    if (!hasFreq) {
       postings.lastDocCodes[termID] = docState.docID;
     } else {
       postings.lastDocCodes[termID] = docState.docID << 1;
       postings.docFreqs[termID] = 1;
-      if (writeProx) {
+      if (hasProx) {
         writeProx(termID, fieldState.position);
-        if (writeOffsets) {
+        if (hasOffsets) {
           writeOffsets(termID, fieldState.offset);
         }
       } else {
-        assert !writeOffsets;
+        assert !hasOffsets;
       }
     }
     fieldState.maxTermFrequency = Math.max(1, fieldState.maxTermFrequency);
@@ -190,9 +194,9 @@ final class FreqProxTermsWriterPerField extends TermsHashConsumerPerField implem
 
     FreqProxPostingsArray postings = (FreqProxPostingsArray) termsHashPerField.postingsArray;
 
-    assert !writeFreq || postings.docFreqs[termID] > 0;
+    assert !hasFreq || postings.docFreqs[termID] > 0;
 
-    if (!writeFreq) {
+    if (!hasFreq) {
       assert postings.docFreqs == null;
       if (docState.docID != postings.lastDocIDs[termID]) {
         assert docState.docID > postings.lastDocIDs[termID];
@@ -218,29 +222,30 @@ final class FreqProxTermsWriterPerField extends TermsHashConsumerPerField implem
       fieldState.maxTermFrequency = Math.max(1, fieldState.maxTermFrequency);
       postings.lastDocCodes[termID] = (docState.docID - postings.lastDocIDs[termID]) << 1;
       postings.lastDocIDs[termID] = docState.docID;
-      if (writeProx) {
+      if (hasProx) {
         writeProx(termID, fieldState.position);
-        if (writeOffsets) {
+        if (hasOffsets) {
+          postings.lastOffsets[termID] = 0;
           writeOffsets(termID, fieldState.offset);
         }
       } else {
-        assert !writeOffsets;
+        assert !hasOffsets;
       }
       fieldState.uniqueTermCount++;
     } else {
       fieldState.maxTermFrequency = Math.max(fieldState.maxTermFrequency, ++postings.docFreqs[termID]);
-      if (writeProx) {
+      if (hasProx) {
         writeProx(termID, fieldState.position-postings.lastPositions[termID]);
       }
-      if (writeOffsets) {
-        writeOffsets(termID, postings.lastOffsets[termID]);
+      if (hasOffsets) {
+        writeOffsets(termID, fieldState.offset);
       }
     }
   }
 
   @Override
   ParallelPostingsArray createPostingsArray(int size) {
-    return new FreqProxPostingsArray(size, writeFreq, writeProx, writeOffsets);
+    return new FreqProxPostingsArray(size, hasFreq, hasProx, hasOffsets);
   }
 
   static final class FreqProxPostingsArray extends ParallelPostingsArray {
@@ -321,8 +326,12 @@ final class FreqProxTermsWriterPerField extends TermsHashConsumerPerField implem
    * instances) found in this field and serialize them
    * into a single RAM segment. */
   void flush(String fieldName, FieldsConsumer consumer,  final SegmentWriteState state)
-    throws CorruptIndexException, IOException {
+    throws IOException {
 
+    if (!fieldInfo.isIndexed()) {
+      return; // nothing to flush, don't bother the codec with the unindexed field
+    }
+    
     final TermsConsumer termsConsumer = consumer.addField(fieldInfo);
     final Comparator termComp = termsConsumer.getComparator();
 
@@ -335,14 +344,15 @@ final class FreqProxTermsWriterPerField extends TermsHashConsumerPerField implem
     // new segment to the directory according to
     // currentFieldIndexOptions:
     final IndexOptions currentFieldIndexOptions = fieldInfo.getIndexOptions();
+    assert currentFieldIndexOptions != null;
 
     final boolean writeTermFreq = currentFieldIndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
     final boolean writePositions = currentFieldIndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
     final boolean writeOffsets = currentFieldIndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
 
-    final boolean readTermFreq = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
-    final boolean readPositions = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
-    final boolean readOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+    final boolean readTermFreq = this.hasFreq;
+    final boolean readPositions = this.hasProx;
+    final boolean readOffsets = this.hasOffsets;
 
     //System.out.println("flush readTF=" + readTermFreq + " readPos=" + readPositions + " readOffs=" + readOffsets);
 
@@ -515,10 +525,15 @@ final class FreqProxTermsWriterPerField extends TermsHashConsumerPerField implem
               if (readOffsets) {
                 final int startOffset = offset + prox.readVInt();
                 final int endOffset = startOffset + prox.readVInt();
-                offset = startOffset;
                 if (writePositions) {
-                  postingsConsumer.addPosition(position, thisPayload, startOffset, endOffset);
+                  if (writeOffsets) {
+                    assert startOffset >=0 && endOffset >= startOffset : "startOffset=" + startOffset + ",endOffset=" + endOffset + ",offset=" + offset;
+                    postingsConsumer.addPosition(position, thisPayload, startOffset, endOffset);
+                  } else {
+                    postingsConsumer.addPosition(position, thisPayload, -1, -1);
+                  }
                 }
+                offset = startOffset;
               } else if (writePositions) {
                 postingsConsumer.addPosition(position, thisPayload, -1, -1);
               }
diff --git a/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedDeletes.java b/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedDeletes.java
index 2cea0d6..916a53f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedDeletes.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedDeletes.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexCommit.java b/lucene/core/src/java/org/apache/lucene/index/IndexCommit.java
index f36d2e1..069f706 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexCommit.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexCommit.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexDeletionPolicy.java b/lucene/core/src/java/org/apache/lucene/index/IndexDeletionPolicy.java
index 3005bae..9b4241c 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexDeletionPolicy.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexDeletionPolicy.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java b/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
index 6aa56a4..4447472 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -118,11 +118,10 @@ final class IndexFileDeleter {
    * the Directory, incref the files they reference, call
    * the policy to let it delete commits.  This will remove
    * any files not referenced by any of the commits.
-   * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
    */
   public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos,
-                          InfoStream infoStream, IndexWriter writer) throws CorruptIndexException, IOException {
+                          InfoStream infoStream, IndexWriter writer) throws IOException {
     this.infoStream = infoStream;
     this.writer = writer;
 
@@ -469,14 +468,14 @@ final class IndexFileDeleter {
     }
   }
 
-  void incRef(Collection files) throws IOException {
+  void incRef(Collection files) {
     assert locked();
     for(final String file : files) {
       incRef(file);
     }
   }
 
-  void incRef(String fileName) throws IOException {
+  void incRef(String fileName) {
     assert locked();
     RefCount rc = getRefCount(fileName);
     if (infoStream.isEnabled("IFD")) {
@@ -661,7 +660,7 @@ final class IndexFileDeleter {
     }
 
     @Override
-    public Collection getFileNames() throws IOException {
+    public Collection getFileNames() {
       return files;
     }
 
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java b/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java
index 98452d3..e853974 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -140,6 +140,17 @@ public final class IndexFileNames {
     return filename.endsWith("." + ext);
   }
 
+  /** locates the boundary of the segment name, or -1 */
+  private static int indexOfSegmentName(String filename) {
+    // If it is a .del file, there's an '_' after the first character
+    int idx = filename.indexOf('_', 1);
+    if (idx == -1) {
+      // If it's not, strip everything that's before the '.'
+      idx = filename.indexOf('.');
+    }
+    return idx;
+  }
+  
   /**
    * Strips the segment name out of the given file name. If you used
    * {@link #segmentFileName} or {@link #fileNameFromGeneration} to create your
@@ -150,18 +161,27 @@ public final class IndexFileNames {
    *         if it does not contain a '.' and '_'.
    */
   public static String stripSegmentName(String filename) {
-    // If it is a .del file, there's an '_' after the first character
-    int idx = filename.indexOf('_', 1);
-    if (idx == -1) {
-      // If it's not, strip everything that's before the '.'
-      idx = filename.indexOf('.');
-    }
+    int idx = indexOfSegmentName(filename);
     if (idx != -1) {
       filename = filename.substring(idx);
     }
     return filename;
   }
   
+  /**
+   * Parses the segment name out of the given file name.
+   * 
+   * @return the segment name only, or filename
+   *         if it does not contain a '.' and '_'.
+   */
+  public static String parseSegmentName(String filename) {
+    int idx = indexOfSegmentName(filename);
+    if (idx != -1) {
+      filename = filename.substring(0, idx);
+    }
+    return filename;
+  }
+  
   public static String stripExtension(String filename) {
     int idx = filename.indexOf('.');
     if (idx != -1) {
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexFormatTooNewException.java b/lucene/core/src/java/org/apache/lucene/index/IndexFormatTooNewException.java
index f5a7149..1531977 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexFormatTooNewException.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexFormatTooNewException.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexFormatTooOldException.java b/lucene/core/src/java/org/apache/lucene/index/IndexFormatTooOldException.java
index 5c2920e..f7951cd 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexFormatTooOldException.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexFormatTooOldException.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -28,7 +28,7 @@ public class IndexFormatTooOldException extends CorruptIndexException {
   /** @lucene.internal */
   public IndexFormatTooOldException(String resourceDesc, String version) {
     super("Format version is not supported (resource: " + resourceDesc + "): " +
-        version + ". This version of Lucene only supports indexes created with release 3.0 and later.");
+        version + ". This version of Lucene only supports indexes created with release 4.0 and later.");
     assert resourceDesc != null;
   }
 
@@ -41,7 +41,7 @@ public class IndexFormatTooOldException extends CorruptIndexException {
   public IndexFormatTooOldException(String resourceDesc, int version, int minVersion, int maxVersion) {
     super("Format version is not supported (resource: " + resourceDesc + "): " +
         version + " (needs to be between " + minVersion + " and " + maxVersion +
-    "). This version of Lucene only supports indexes created with release 3.0 and later.");
+    "). This version of Lucene only supports indexes created with release 4.0 and later.");
     assert resourceDesc != null;
   }
 
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexNotFoundException.java b/lucene/core/src/java/org/apache/lucene/index/IndexNotFoundException.java
index dc0a6fa..81b5ff3 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexNotFoundException.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexNotFoundException.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexReader.java b/lucene/core/src/java/org/apache/lucene/index/IndexReader.java
index de11662..59b6e91 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexReader.java
@@ -1,6 +1,6 @@
 package org.apache.lucene.index;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -28,10 +28,9 @@ import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.DocumentStoredFieldVisitor;
 import org.apache.lucene.search.SearcherManager; // javadocs
-import org.apache.lucene.store.*;
+import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.ReaderUtil;         // for javadocs
 
 /** IndexReader is an abstract class, providing an interface for accessing an
  index.  Search of an index is done entirely through this abstract interface,
@@ -52,8 +51,8 @@ import org.apache.lucene.util.ReaderUtil;         // for javadocs

    IndexReader instances for indexes on disk are usually constructed - with a call to one of the static DirectoryReader,open() methods, - e.g. {@link DirectoryReader#open(Directory)}. {@link DirectoryReader} implements + with a call to one of the static DirectoryReader.open() methods, + e.g. {@link DirectoryReader#open(org.apache.lucene.store.Directory)}. {@link DirectoryReader} implements the {@link CompositeReader} interface, it is not possible to directly get postings.

    For efficiency, in this API documents are often referred to via @@ -277,100 +276,6 @@ public abstract class IndexReader implements Closeable { public final int hashCode() { return System.identityHashCode(this); } - - /** Returns a IndexReader reading the index in the given - * Directory - * @param directory the index directory - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - * @deprecated Use {@link DirectoryReader#open(Directory)} - */ - @Deprecated - public static DirectoryReader open(final Directory directory) throws CorruptIndexException, IOException { - return DirectoryReader.open(directory); - } - - /** Expert: Returns a IndexReader reading the index in the given - * Directory with the given termInfosIndexDivisor. - * @param directory the index directory - * @param termInfosIndexDivisor Subsamples which indexed - * terms are loaded into RAM. This has the same effect as {@link - * IndexWriterConfig#setTermIndexInterval} except that setting - * must be done at indexing time while this setting can be - * set per reader. When set to N, then one in every - * N*termIndexInterval terms in the index is loaded into - * memory. By setting this to a value > 1 you can reduce - * memory usage, at the expense of higher latency when - * loading a TermInfo. The default value is 1. Set this - * to -1 to skip loading the terms index entirely. - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - * @deprecated Use {@link DirectoryReader#open(Directory,int)} - */ - @Deprecated - public static DirectoryReader open(final Directory directory, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return DirectoryReader.open(directory, termInfosIndexDivisor); - } - - /** - * Open a near real time IndexReader from the {@link org.apache.lucene.index.IndexWriter}. - * - * @param writer The IndexWriter to open from - * @param applyAllDeletes If true, all buffered deletes will - * be applied (made visible) in the returned reader. If - * false, the deletes are not applied but remain buffered - * (in IndexWriter) so that they will be applied in the - * future. Applying deletes can be costly, so if your app - * can tolerate deleted documents being returned you might - * gain some performance by passing false. - * @return The new IndexReader - * @throws CorruptIndexException - * @throws IOException if there is a low-level IO error - * - * @see DirectoryReader#openIfChanged(DirectoryReader,IndexWriter,boolean) - * - * @lucene.experimental - * @deprecated Use {@link DirectoryReader#open(IndexWriter,boolean)} - */ - @Deprecated - public static DirectoryReader open(final IndexWriter writer, boolean applyAllDeletes) throws CorruptIndexException, IOException { - return DirectoryReader.open(writer, applyAllDeletes); - } - - /** Expert: returns an IndexReader reading the index in the given - * {@link IndexCommit}. - * @param commit the commit point to open - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - * @deprecated Use {@link DirectoryReader#open(IndexCommit)} - */ - @Deprecated - public static DirectoryReader open(final IndexCommit commit) throws CorruptIndexException, IOException { - return DirectoryReader.open(commit); - } - - - /** Expert: returns an IndexReader reading the index in the given - * {@link IndexCommit} and termInfosIndexDivisor. - * @param commit the commit point to open - * @param termInfosIndexDivisor Subsamples which indexed - * terms are loaded into RAM. This has the same effect as {@link - * IndexWriterConfig#setTermIndexInterval} except that setting - * must be done at indexing time while this setting can be - * set per reader. When set to N, then one in every - * N*termIndexInterval terms in the index is loaded into - * memory. By setting this to a value > 1 you can reduce - * memory usage, at the expense of higher latency when - * loading a TermInfo. The default value is 1. Set this - * to -1 to skip loading the terms index entirely. - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - * @deprecated Use {@link DirectoryReader#open(IndexCommit,int)} - */ - @Deprecated - public static DirectoryReader open(final IndexCommit commit, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return DirectoryReader.open(commit, termInfosIndexDivisor); - } /** Retrieve term vectors for this document, or null if * term vectors were not indexed. The returned Fields @@ -411,7 +316,7 @@ public abstract class IndexReader implements Closeable { * simply want to load all fields, use {@link * #document(int)}. If you want to load a subset, use * {@link DocumentStoredFieldVisitor}. */ - public abstract void document(int docID, StoredFieldVisitor visitor) throws CorruptIndexException, IOException; + public abstract void document(int docID, StoredFieldVisitor visitor) throws IOException; /** * Returns the stored fields of the nth @@ -435,7 +340,7 @@ public abstract class IndexReader implements Closeable { // TODO: we need a separate StoredField, so that the // Document returned here contains that class not // IndexableField - public final Document document(int docID) throws CorruptIndexException, IOException { + public final Document document(int docID) throws IOException { final DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor(); document(docID, visitor); return visitor.getDocument(); @@ -446,7 +351,7 @@ public abstract class IndexReader implements Closeable { * fields. Note that this is simply sugar for {@link * DocumentStoredFieldVisitor#DocumentStoredFieldVisitor(Set)}. */ - public final Document document(int docID, Set fieldsToLoad) throws CorruptIndexException, IOException { + public final Document document(int docID, Set fieldsToLoad) throws IOException { final DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor(fieldsToLoad); document(docID, visitor); return visitor.getDocument(); diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java b/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java index 8c33c19..e7bd2cc 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java @@ -1,6 +1,8 @@ package org.apache.lucene.index; -/** +import java.util.List; + +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -45,23 +47,19 @@ public abstract class IndexReaderContext { public abstract IndexReader reader(); /** - * Returns the context's leaves if this context is a top-level context - * otherwise null. For convenience, if this is an - * {@link AtomicReaderContext} this returns itsself as the only leaf. + * Returns the context's leaves if this context is a top-level context. + * For convenience, if this is an {@link AtomicReaderContext} this + * returns itself as the only leaf. *

    Note: this is convenience method since leaves can always be obtained by - * walking the context tree. - *

    Warning: Don't modify the returned array! - * Doing so will corrupt the internal structure of this - * {@code IndexReaderContext}. + * walking the context tree using {@link #children()}. + * @throws UnsupportedOperationException if this is not a top-level context. + * @see #children() */ - public abstract AtomicReaderContext[] leaves(); + public abstract List leaves() throws UnsupportedOperationException; /** * Returns the context's children iff this context is a composite context * otherwise null. - *

    Warning: Don't modify the returned array! - * Doing so will corrupt the internal structure of this - * {@code IndexReaderContext}. */ - public abstract IndexReaderContext[] children(); + public abstract List children(); } \ No newline at end of file diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexUpgrader.java b/lucene/core/src/java/org/apache/lucene/index/IndexUpgrader.java index 335a052..62472a3 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexUpgrader.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexUpgrader.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java index b5e9e77..498c5be 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -33,8 +33,6 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene3x.Lucene3xCodec; -import org.apache.lucene.codecs.lucene3x.Lucene3xSegmentInfoFormat; import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment; import org.apache.lucene.index.FieldInfos.FieldNumbers; import org.apache.lucene.index.IndexWriterConfig.OpenMode; @@ -55,7 +53,6 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.MutableBits; import org.apache.lucene.util.ThreadInterruptedException; -import org.apache.lucene.util.TwoPhaseCommit; /** An IndexWriter creates and maintains an index. @@ -260,7 +257,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // The instance that was passed to the constructor. It is saved only in order // to allow users to query an IndexWriter settings. - private final IndexWriterConfig config; + private final LiveIndexWriterConfig config; // The PayloadProcessorProvider to use when segments are merged private PayloadProcessorProvider payloadProcessorProvider; @@ -278,8 +275,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * the writer nor calling {@link #commit}. * *

    Note that this is functionally equivalent to calling - * {#flush} and then using {@link IndexReader#open} to - * open a new reader. But the turnaround time of this + * {#flush} and then opening a new reader. But the turnaround time of this * method should be faster since it avoids the potentially * costly {@link #commit}.

    * @@ -501,7 +497,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * Obtain a ReadersAndLiveDocs instance from the * readerPool. If create is true, you must later call * {@link #release(ReadersAndLiveDocs)}. - * @throws IOException */ public synchronized ReadersAndLiveDocs get(SegmentInfoPerCommit info, boolean create) { @@ -533,7 +528,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * If the reader isn't being pooled, the segmentInfo's * delCount is returned. */ - public int numDeletedDocs(SegmentInfoPerCommit info) throws IOException { + public int numDeletedDocs(SegmentInfoPerCommit info) { ensureOpen(false); int delCount = info.getDelCount(); @@ -575,39 +570,33 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * @param conf * the configuration settings according to which IndexWriter should * be initialized. - * @throws CorruptIndexException - * if the index is corrupt - * @throws LockObtainFailedException - * if another writer has this index open (write.lock - * could not be obtained) * @throws IOException * if the directory cannot be read/written to, or if it does not * exist and conf.getOpenMode() is * OpenMode.APPEND or if there is any other low-level * IO error */ - public IndexWriter(Directory d, IndexWriterConfig conf) - throws CorruptIndexException, LockObtainFailedException, IOException { - config = conf.clone(); + public IndexWriter(Directory d, IndexWriterConfig conf) throws IOException { + config = new LiveIndexWriterConfig(conf.clone()); directory = d; - analyzer = conf.getAnalyzer(); - infoStream = conf.getInfoStream(); - mergePolicy = conf.getMergePolicy(); + analyzer = config.getAnalyzer(); + infoStream = config.getInfoStream(); + mergePolicy = config.getMergePolicy(); mergePolicy.setIndexWriter(this); - mergeScheduler = conf.getMergeScheduler(); - codec = conf.getCodec(); + mergeScheduler = config.getMergeScheduler(); + codec = config.getCodec(); bufferedDeletesStream = new BufferedDeletesStream(infoStream); - poolReaders = conf.getReaderPooling(); + poolReaders = config.getReaderPooling(); writeLock = directory.makeLock(WRITE_LOCK_NAME); - if (!writeLock.obtain(conf.getWriteLockTimeout())) // obtain write lock + if (!writeLock.obtain(config.getWriteLockTimeout())) // obtain write lock throw new LockObtainFailedException("Index locked for write: " + writeLock); boolean success = false; try { - OpenMode mode = conf.getOpenMode(); + OpenMode mode = config.getOpenMode(); boolean create; if (mode == OpenMode.CREATE) { create = true; @@ -641,7 +630,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { } else { segmentInfos.read(directory); - IndexCommit commit = conf.getIndexCommit(); + IndexCommit commit = config.getIndexCommit(); if (commit != null) { // Swap out all segments, but, keep metadata in // SegmentInfos, like version & generation, to @@ -671,7 +660,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // KeepOnlyLastCommitDeleter: synchronized(this) { deleter = new IndexFileDeleter(directory, - conf.getIndexDeletionPolicy(), + config.getIndexDeletionPolicy(), segmentInfos, infoStream, this); } @@ -755,22 +744,15 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { } /** - * Returns the private {@link IndexWriterConfig}, cloned - * from the {@link IndexWriterConfig} passed to - * {@link #IndexWriter(Directory, IndexWriterConfig)}. - *

    - * NOTE: some settings may be changed on the - * returned {@link IndexWriterConfig}, and will take - * effect in the current IndexWriter instance. See the - * javadocs for the specific setters in {@link - * IndexWriterConfig} for details. + * Returns a {@link LiveIndexWriterConfig}, which can be used to query the IndexWriter + * current settings, as well as modify "live" ones. */ - public IndexWriterConfig getConfig() { + public LiveIndexWriterConfig getConfig() { ensureOpen(false); return config; } - private void messageState() throws IOException { + private void messageState() { if (infoStream.isEnabled("IW")) { infoStream.message("IW", "\ndir=" + directory + "\n" + "index=" + segString() + "\n" + @@ -818,10 +800,9 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * you should immediately close the writer, again. See above for details.

    * - * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public void close() throws CorruptIndexException, IOException { + public void close() throws IOException { close(true); } @@ -848,17 +829,21 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * finished (which should be at most a few seconds), and * then return. */ - public void close(boolean waitForMerges) throws CorruptIndexException, IOException { - - // Ensure that only one thread actually gets to do the closing: - if (shouldClose()) { - // If any methods have hit OutOfMemoryError, then abort - // on close, in case the internal state of IndexWriter - // or DocumentsWriter is corrupt - if (hitOOM) - rollbackInternal(); - else - closeInternal(waitForMerges); + public void close(boolean waitForMerges) throws IOException { + + // Ensure that only one thread actually gets to do the + // closing, and make sure no commit is also in progress: + synchronized(commitLock) { + if (shouldClose()) { + // If any methods have hit OutOfMemoryError, then abort + // on close, in case the internal state of IndexWriter + // or DocumentsWriter is corrupt + if (hitOOM) { + rollbackInternal(); + } else { + closeInternal(waitForMerges, !hitOOM); + } + } } } @@ -877,12 +862,13 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // successfully) or another (fails to close) doWait(); } - } else + } else { return false; + } } } - private void closeInternal(boolean waitForMerges) throws CorruptIndexException, IOException { + private void closeInternal(boolean waitForMerges, boolean doFlush) throws IOException { try { @@ -898,8 +884,10 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // Only allow a new merge to be triggered if we are // going to wait for merges: - if (!hitOOM) { + if (doFlush) { flush(waitForMerges, true); + } else { + docWriter.abort(); // already closed } if (waitForMerges) @@ -919,7 +907,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { infoStream.message("IW", "now call final commit()"); } - if (!hitOOM) { + if (doFlush) { commitInternal(null); } @@ -990,7 +978,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * are not counted. If you really need these to be * counted you should call {@link #commit()} first. * @see #numDocs */ - public synchronized int numDocs() throws IOException { + public synchronized int numDocs() { ensureOpen(); int count; if (docWriter != null) @@ -1004,7 +992,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { return count; } - public synchronized boolean hasDeletions() throws IOException { + public synchronized boolean hasDeletions() { ensureOpen(); if (bufferedDeletesStream.any()) { return true; @@ -1062,7 +1050,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public void addDocument(Iterable doc) throws CorruptIndexException, IOException { + public void addDocument(Iterable doc) throws IOException { addDocument(doc, analyzer); } @@ -1081,7 +1069,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public void addDocument(Iterable doc, Analyzer analyzer) throws CorruptIndexException, IOException { + public void addDocument(Iterable doc, Analyzer analyzer) throws IOException { updateDocument(null, doc, analyzer); } @@ -1126,7 +1114,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * * @lucene.experimental */ - public void addDocuments(Iterable> docs) throws CorruptIndexException, IOException { + public void addDocuments(Iterable> docs) throws IOException { addDocuments(docs, analyzer); } @@ -1141,7 +1129,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * * @lucene.experimental */ - public void addDocuments(Iterable> docs, Analyzer analyzer) throws CorruptIndexException, IOException { + public void addDocuments(Iterable> docs, Analyzer analyzer) throws IOException { updateDocuments(null, docs, analyzer); } @@ -1158,7 +1146,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * * @lucene.experimental */ - public void updateDocuments(Term delTerm, Iterable> docs) throws CorruptIndexException, IOException { + public void updateDocuments(Term delTerm, Iterable> docs) throws IOException { updateDocuments(delTerm, docs, analyzer); } @@ -1176,7 +1164,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * * @lucene.experimental */ - public void updateDocuments(Term delTerm, Iterable> docs, Analyzer analyzer) throws CorruptIndexException, IOException { + public void updateDocuments(Term delTerm, Iterable> docs, Analyzer analyzer) throws IOException { ensureOpen(); try { boolean success = false; @@ -1210,7 +1198,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public void deleteDocuments(Term term) throws CorruptIndexException, IOException { + public void deleteDocuments(Term term) throws IOException { ensureOpen(); try { docWriter.deleteTerms(term); @@ -1233,7 +1221,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public void deleteDocuments(Term... terms) throws CorruptIndexException, IOException { + public void deleteDocuments(Term... terms) throws IOException { ensureOpen(); try { docWriter.deleteTerms(terms); @@ -1253,7 +1241,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public void deleteDocuments(Query query) throws CorruptIndexException, IOException { + public void deleteDocuments(Query query) throws IOException { ensureOpen(); try { docWriter.deleteQueries(query); @@ -1275,7 +1263,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public void deleteDocuments(Query... queries) throws CorruptIndexException, IOException { + public void deleteDocuments(Query... queries) throws IOException { ensureOpen(); try { docWriter.deleteQueries(queries); @@ -1301,7 +1289,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public void updateDocument(Term term, Iterable doc) throws CorruptIndexException, IOException { + public void updateDocument(Term term, Iterable doc) throws IOException { ensureOpen(); updateDocument(term, doc, getAnalyzer()); } @@ -1325,7 +1313,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * @throws IOException if there is a low-level IO error */ public void updateDocument(Term term, Iterable doc, Analyzer analyzer) - throws CorruptIndexException, IOException { + throws IOException { ensureOpen(); try { boolean success = false; @@ -1459,14 +1447,15 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * then any thread still running this method might hit a * {@link MergePolicy.MergeAbortedException}. * + * @param maxNumSegments maximum number of segments left + * in the index after merging finishes + * * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error * @see MergePolicy#findMerges * - * @param maxNumSegments maximum number of segments left - * in the index after merging finishes */ - public void forceMerge(int maxNumSegments) throws CorruptIndexException, IOException { + public void forceMerge(int maxNumSegments) throws IOException { forceMerge(maxNumSegments, true); } @@ -1480,7 +1469,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * you should immediately close the writer. See above for details.

    */ - public void forceMerge(int maxNumSegments, boolean doWait) throws CorruptIndexException, IOException { + public void forceMerge(int maxNumSegments, boolean doWait) throws IOException { ensureOpen(); if (maxNumSegments < 1) @@ -1591,7 +1580,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * {@link MergePolicy.MergeAbortedException}. */ public void forceMergeDeletes(boolean doWait) - throws CorruptIndexException, IOException { + throws IOException { ensureOpen(); flush(true, true); @@ -1676,7 +1665,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * you should immediately close the writer. See above for details.

    */ - public void forceMergeDeletes() throws CorruptIndexException, IOException { + public void forceMergeDeletes() throws IOException { forceMergeDeletes(true); } @@ -1694,18 +1683,18 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * you should immediately close the writer. See above for details.

    */ - public final void maybeMerge() throws CorruptIndexException, IOException { + public final void maybeMerge() throws IOException { maybeMerge(-1); } - private final void maybeMerge(int maxNumSegments) throws CorruptIndexException, IOException { + private final void maybeMerge(int maxNumSegments) throws IOException { ensureOpen(false); updatePendingMerges(maxNumSegments); mergeScheduler.merge(this); } private synchronized void updatePendingMerges(int maxNumSegments) - throws CorruptIndexException, IOException { + throws IOException { assert maxNumSegments == -1 || maxNumSegments > 0; if (stopMerges) { @@ -1783,9 +1772,13 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { public void rollback() throws IOException { ensureOpen(); - // Ensure that only one thread actually gets to do the closing: - if (shouldClose()) - rollbackInternal(); + // Ensure that only one thread actually gets to do the + // closing, and make sure no commit is also in progress: + synchronized(commitLock) { + if (shouldClose()) { + rollbackInternal(); + } + } } private void rollbackInternal() throws IOException { @@ -1795,6 +1788,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { if (infoStream.isEnabled("IW")) { infoStream.message("IW", "rollback"); } + try { synchronized(this) { @@ -1813,7 +1807,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { mergeScheduler.close(); bufferedDeletesStream.clear(); - + docWriter.close(); // mark it as closed first to prevent subsequent indexing actions/flushes + docWriter.abort(); synchronized(this) { if (pendingCommit != null) { @@ -1835,8 +1830,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { if (infoStream.isEnabled("IW") ) { infoStream.message("IW", "rollback: infos=" + segString(segmentInfos)); } - - docWriter.abort(); + assert testPoint("rollback before checkpoint"); @@ -1863,7 +1857,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { } } - closeInternal(false); + closeInternal(false, false); } /** @@ -1918,7 +1912,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { } } - private synchronized void finishMerges(boolean waitForMerges) throws IOException { + private synchronized void finishMerges(boolean waitForMerges) { if (!waitForMerges) { stopMerges = true; @@ -2008,11 +2002,11 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { /** * Prepares the {@link SegmentInfo} for the new flushed segment and persists * the deleted documents {@link MutableBits}. Use - * {@link #publishFlushedSegment(SegmentInfo, FrozenBufferedDeletes, FrozenBufferedDeletes)} to + * {@link #publishFlushedSegment(SegmentInfoPerCommit, FrozenBufferedDeletes, FrozenBufferedDeletes)} to * publish the returned {@link SegmentInfo} together with its segment private * delete packet. * - * @see #publishFlushedSegment(SegmentInfo, FrozenBufferedDeletes, FrozenBufferedDeletes) + * @see #publishFlushedSegment(SegmentInfoPerCommit, FrozenBufferedDeletes, FrozenBufferedDeletes) */ SegmentInfoPerCommit prepareFlushedSegment(FlushedSegment flushedSegment) throws IOException { assert flushedSegment != null; @@ -2040,7 +2034,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // creating CFS so that 1) .si isn't slurped into CFS, // and 2) .si reflects useCompoundFile=true change // above: - codec.segmentInfoFormat().getSegmentInfosWriter().write(directory, newSegment.info, flushedSegment.fieldInfos, context); + codec.segmentInfoFormat().getSegmentInfoWriter().write(directory, newSegment.info, flushedSegment.fieldInfos, context); // TODO: ideally we would freeze newSegment here!! // because any changes after writing the .si will be @@ -2084,7 +2078,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { return newSegment; } - synchronized void publishFrozenDeletes(FrozenBufferedDeletes packet) throws IOException { + synchronized void publishFrozenDeletes(FrozenBufferedDeletes packet) { assert packet != null && packet.any(); synchronized (bufferedDeletesStream) { bufferedDeletesStream.push(packet); @@ -2121,7 +2115,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { nextGen = bufferedDeletesStream.getNextGen(); } if (infoStream.isEnabled("IW")) { - infoStream.message("IW", "publish sets newSegment delGen=" + nextGen + " seg=" + newSegment); + infoStream.message("IW", "publish sets newSegment delGen=" + nextGen + " seg=" + segString(newSegment)); } newSegment.setBufferedDeletesGen(nextGen); segmentInfos.add(newSegment); @@ -2193,7 +2187,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public void addIndexes(Directory... dirs) throws CorruptIndexException, IOException { + public void addIndexes(Directory... dirs) throws IOException { ensureOpen(); noDupDirs(dirs); @@ -2219,7 +2213,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { assert !infos.contains(info): "dup info dir=" + info.info.dir + " name=" + info.info.name; String newSegName = newSegmentName(); - String dsName = Lucene3xSegmentInfoFormat.getDocStoreSegment(info.info); + String dsName = info.info.name; if (infoStream.isEnabled("IW")) { infoStream.message("IW", "addIndexes: process segment origName=" + info.info.name + " newName=" + newSegName + " dsName=" + dsName + " info=" + info); @@ -2241,31 +2235,43 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { handleOOM(oom, "addIndexes(Directory...)"); } } - - /** Merges the provided indexes into this index. - *

    The provided IndexReaders are not closed.

    - * - *

    NOTE: while this is running, any attempts to - * add or delete documents (with another thread) will be - * paused until this method completes. - * - *

    See {@link #addIndexes} for details on transactional - * semantics, temporary free space required in the Directory, - * and non-CFS segments on an Exception.

    - * - *

    NOTE: if this method hits an OutOfMemoryError - * you should immediately close the writer. See above for details.

    - * - *

    NOTE: if you call {@link #close(boolean)} - * with false, which aborts all running merges, - * then any thread still running this method might hit a - * {@link MergePolicy.MergeAbortedException}. - * - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error + + /** + * Merges the provided indexes into this index. + * + *

    + * The provided IndexReaders are not closed. + * + *

    + * See {@link #addIndexes} for details on transactional semantics, temporary + * free space required in the Directory, and non-CFS segments on an Exception. + * + *

    + * NOTE: if this method hits an OutOfMemoryError you should immediately + * close the writer. See above for details. + * + *

    + * NOTE: this method merges all given {@link IndexReader}s in one + * merge. If you intend to merge a large number of readers, it may be better + * to call this method multiple times, each time with a small set of readers. + * In principle, if you use a merge policy with a {@code mergeFactor} or + * {@code maxMergeAtOnce} parameter, you should pass that many readers in one + * call. Also, if the given readers are {@link DirectoryReader}s, they can be + * opened with {@code termIndexInterval=-1} to save RAM, since during merge + * the in-memory structure is not used. See + * {@link DirectoryReader#open(Directory, int)}. + * + *

    + * NOTE: if you call {@link #close(boolean)} with false, which + * aborts all running merges, then any thread still running this method might + * hit a {@link MergePolicy.MergeAbortedException}. + * + * @throws CorruptIndexException + * if the index is corrupt + * @throws IOException + * if there is a low-level IO error */ - public void addIndexes(IndexReader... readers) throws CorruptIndexException, IOException { + public void addIndexes(IndexReader... readers) throws IOException { ensureOpen(); int numDocs = 0; @@ -2332,7 +2338,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // creating CFS so that 1) .si isn't slurped into CFS, // and 2) .si reflects useCompoundFile=true change // above: - codec.segmentInfoFormat().getSegmentInfosWriter().write(trackingDir, info, mergeState.fieldInfos, context); + codec.segmentInfoFormat().getSegmentInfoWriter().write(trackingDir, info, mergeState.fieldInfos, context); info.addFiles(trackingDir.getCreatedFiles()); // Register the new segment @@ -2351,6 +2357,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { } /** Copies the segment files as-is into the IndexWriter's directory. */ + // TODO: this can be substantially simplified now that 3.x support/shared docstores is removed! private SegmentInfoPerCommit copySegmentAsIs(SegmentInfoPerCommit info, String segName, Map dsNames, Set dsFilesCopied, IOContext context, Set copiedFiles) @@ -2359,7 +2366,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // only relevant for segments that share doc store with others, // because the DS might have been copied already, in which case we // just want to update the DS name of this SegmentInfo. - final String dsName = Lucene3xSegmentInfoFormat.getDocStoreSegment(info.info); + final String dsName = info.info.name; assert dsName != null; final String newDsName; if (dsNames.containsKey(dsName)) { @@ -2373,8 +2380,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // so we don't pass a null value to the si writer FieldInfos fis = getFieldInfos(info.info); - Set docStoreFiles3xOnly = Lucene3xCodec.getDocStoreFiles(info.info); - final Map attributes; // copy the attributes map, we might modify it below. // also we need to ensure its read-write, since we will invoke the SIwriter (which might want to set something). @@ -2383,12 +2388,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { } else { attributes = new HashMap(info.info.attributes()); } - if (docStoreFiles3xOnly != null) { - // only violate the codec this way if it's preflex & - // shares doc stores - // change docStoreSegment to newDsName - attributes.put(Lucene3xSegmentInfoFormat.DS_NAME_KEY, newDsName); - } //System.out.println("copy seg=" + info.info.name + " version=" + info.info.getVersion()); // Same SI as before but we change directory, name and docStoreSegment: @@ -2403,11 +2402,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // before writing SegmentInfo: for (String file: info.files()) { final String newFileName; - if (docStoreFiles3xOnly != null && docStoreFiles3xOnly.contains(file)) { - newFileName = newDsName + IndexFileNames.stripSegmentName(file); - } else { - newFileName = segName + IndexFileNames.stripSegmentName(file); - } + newFileName = segName + IndexFileNames.stripSegmentName(file); segFiles.add(newFileName); } newInfo.setFiles(segFiles); @@ -2417,7 +2412,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // store segment name): TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(directory); try { - newInfo.getCodec().segmentInfoFormat().getSegmentInfosWriter().write(trackingDir, newInfo, fis, context); + newInfo.getCodec().segmentInfoFormat().getSegmentInfoWriter().write(trackingDir, newInfo, fis, context); } catch (UnsupportedOperationException uoe) { // OK: 3x codec cannot write a new SI file; // SegmentInfos will write this on commit @@ -2428,16 +2423,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // Copy the segment's files for (String file: info.files()) { - final String newFileName; - if (docStoreFiles3xOnly != null && docStoreFiles3xOnly.contains(file)) { - newFileName = newDsName + IndexFileNames.stripSegmentName(file); - if (dsFilesCopied.contains(newFileName)) { - continue; - } - dsFilesCopied.add(newFileName); - } else { - newFileName = segName + IndexFileNames.stripSegmentName(file); - } + final String newFileName = segName + IndexFileNames.stripSegmentName(file); if (siFiles.contains(newFileName)) { // We already rewrote this above @@ -2473,7 +2459,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * href="#OOME">above for details.

    * * @see #prepareCommit(Map) */ - public final void prepareCommit() throws CorruptIndexException, IOException { + public final void prepareCommit() throws IOException { ensureOpen(); prepareCommit(null); } @@ -2508,102 +2494,105 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * only "stick" if there are actually changes in the * index to commit. */ - public final void prepareCommit(Map commitUserData) throws CorruptIndexException, IOException { + public final void prepareCommit(Map commitUserData) throws IOException { ensureOpen(false); - if (infoStream.isEnabled("IW")) { - infoStream.message("IW", "prepareCommit: flush"); - infoStream.message("IW", " index before flush " + segString()); - } + synchronized(commitLock) { + if (infoStream.isEnabled("IW")) { + infoStream.message("IW", "prepareCommit: flush"); + infoStream.message("IW", " index before flush " + segString()); + } - if (hitOOM) { - throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot commit"); - } + if (hitOOM) { + throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot commit"); + } - if (pendingCommit != null) { - throw new IllegalStateException("prepareCommit was already called with no corresponding call to commit"); - } + if (pendingCommit != null) { + throw new IllegalStateException("prepareCommit was already called with no corresponding call to commit"); + } - doBeforeFlush(); - assert testPoint("startDoFlush"); - SegmentInfos toCommit = null; - boolean anySegmentsFlushed = false; + doBeforeFlush(); + assert testPoint("startDoFlush"); + SegmentInfos toCommit = null; + boolean anySegmentsFlushed = false; - // This is copied from doFlush, except it's modified to - // clone & incRef the flushed SegmentInfos inside the - // sync block: + // This is copied from doFlush, except it's modified to + // clone & incRef the flushed SegmentInfos inside the + // sync block: - try { + try { - synchronized (fullFlushLock) { - boolean flushSuccess = false; - boolean success = false; - try { - anySegmentsFlushed = docWriter.flushAllThreads(); - if (!anySegmentsFlushed) { - // prevent double increment since docWriter#doFlush increments the flushcount - // if we flushed anything. - flushCount.incrementAndGet(); - } - flushSuccess = true; + synchronized (fullFlushLock) { + boolean flushSuccess = false; + boolean success = false; + try { + anySegmentsFlushed = docWriter.flushAllThreads(); + if (!anySegmentsFlushed) { + // prevent double increment since docWriter#doFlush increments the flushcount + // if we flushed anything. + flushCount.incrementAndGet(); + } + flushSuccess = true; - synchronized(this) { - maybeApplyDeletes(true); - - readerPool.commit(segmentInfos); - - // Must clone the segmentInfos while we still - // hold fullFlushLock and while sync'd so that - // no partial changes (eg a delete w/o - // corresponding add from an updateDocument) can - // sneak into the commit point: - toCommit = segmentInfos.clone(); - - pendingCommitChangeCount = changeCount; - - // This protects the segmentInfos we are now going - // to commit. This is important in case, eg, while - // we are trying to sync all referenced files, a - // merge completes which would otherwise have - // removed the files we are now syncing. - filesToCommit = toCommit.files(directory, false); - deleter.incRef(filesToCommit); - } - success = true; - } finally { - if (!success) { - if (infoStream.isEnabled("IW")) { - infoStream.message("IW", "hit exception during prepareCommit"); + synchronized(this) { + maybeApplyDeletes(true); + + readerPool.commit(segmentInfos); + + // Must clone the segmentInfos while we still + // hold fullFlushLock and while sync'd so that + // no partial changes (eg a delete w/o + // corresponding add from an updateDocument) can + // sneak into the commit point: + toCommit = segmentInfos.clone(); + + pendingCommitChangeCount = changeCount; + + // This protects the segmentInfos we are now going + // to commit. This is important in case, eg, while + // we are trying to sync all referenced files, a + // merge completes which would otherwise have + // removed the files we are now syncing. + filesToCommit = toCommit.files(directory, false); + deleter.incRef(filesToCommit); + } + success = true; + } finally { + if (!success) { + if (infoStream.isEnabled("IW")) { + infoStream.message("IW", "hit exception during prepareCommit"); + } } + // Done: finish the full flush! + docWriter.finishFullFlush(flushSuccess); + doAfterFlush(); } - // Done: finish the full flush! - docWriter.finishFullFlush(flushSuccess); - doAfterFlush(); } + } catch (OutOfMemoryError oom) { + handleOOM(oom, "prepareCommit"); } - } catch (OutOfMemoryError oom) { - handleOOM(oom, "prepareCommit"); - } - boolean success = false; - try { - if (anySegmentsFlushed) { - maybeMerge(); - } - success = true; - } finally { - if (!success) { - synchronized (this) { - deleter.decRef(filesToCommit); - filesToCommit = null; + boolean success = false; + try { + if (anySegmentsFlushed) { + maybeMerge(); + } + success = true; + } finally { + if (!success) { + synchronized (this) { + deleter.decRef(filesToCommit); + filesToCommit = null; + } } } - } - startCommit(toCommit, commitUserData); + startCommit(toCommit, commitUserData); + } } - // Used only by commit, below; lock order is commitLock -> IW + // Used only by commit and prepareCommit, below; lock + // order is commitLock -> IW private final Object commitLock = new Object(); /** @@ -2636,7 +2625,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * @see #prepareCommit * @see #commit(Map) */ - public final void commit() throws CorruptIndexException, IOException { + public final void commit() throws IOException { commit(null); } @@ -2649,20 +2638,22 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * you should immediately close the writer. See above for details.

    */ - public final void commit(Map commitUserData) throws CorruptIndexException, IOException { + public final void commit(Map commitUserData) throws IOException { ensureOpen(); commitInternal(commitUserData); } - private final void commitInternal(Map commitUserData) throws CorruptIndexException, IOException { + private final void commitInternal(Map commitUserData) throws IOException { if (infoStream.isEnabled("IW")) { infoStream.message("IW", "commit: start"); } synchronized(commitLock) { + ensureOpen(false); + if (infoStream.isEnabled("IW")) { infoStream.message("IW", "commit: enter lock"); } @@ -2682,7 +2673,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { } } - private synchronized final void finishCommit() throws CorruptIndexException, IOException { + private synchronized final void finishCommit() throws IOException { if (pendingCommit != null) { try { @@ -2728,7 +2719,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * deletes or docs were flushed) if necessary * @param applyAllDeletes whether pending deletes should also */ - protected final void flush(boolean triggerMerge, boolean applyAllDeletes) throws CorruptIndexException, IOException { + protected final void flush(boolean triggerMerge, boolean applyAllDeletes) throws IOException { // NOTE: this method cannot be sync'd because // maybeMerge() in turn calls mergeScheduler.merge which @@ -2744,7 +2735,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { } } - private boolean doFlush(boolean applyAllDeletes) throws CorruptIndexException, IOException { + private boolean doFlush(boolean applyAllDeletes) throws IOException { if (hitOOM) { throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot flush"); } @@ -2851,7 +2842,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { return docWriter.getNumDocs(); } - private synchronized void ensureValidMerge(MergePolicy.OneMerge merge) throws IOException { + private synchronized void ensureValidMerge(MergePolicy.OneMerge merge) { for(SegmentInfoPerCommit info : merge.segments) { if (!segmentInfos.contains(info)) { throw new MergePolicy.MergeException("MergePolicy selected a segment (" + info.info.name + ") that is not in the current index " + segString(), directory); @@ -2890,7 +2881,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { SegmentInfoPerCommit info = sourceSegments.get(i); minGen = Math.min(info.getBufferedDeletesGen(), minGen); final int docCount = info.info.getDocCount(); - final Bits prevLiveDocs = merge.readerLiveDocs.get(i); + final Bits prevLiveDocs = merge.readers.get(i).getLiveDocs(); final Bits currentLiveDocs; final ReadersAndLiveDocs rld = readerPool.get(info, false); // We hold a ref so it should still be in the pool: @@ -3111,8 +3102,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * * @lucene.experimental */ - public void merge(MergePolicy.OneMerge merge) - throws CorruptIndexException, IOException { + public void merge(MergePolicy.OneMerge merge) throws IOException { boolean success = false; @@ -3174,7 +3164,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * are now participating in a merge, and true is * returned. Else (the merge conflicts) false is * returned. */ - final synchronized boolean registerMerge(MergePolicy.OneMerge merge) throws MergePolicy.MergeAbortedException, IOException { + final synchronized boolean registerMerge(MergePolicy.OneMerge merge) throws IOException { if (merge.registerDone) { return true; @@ -3231,7 +3221,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { } for(SegmentInfoPerCommit info : merge.segments) { if (infoStream.isEnabled("IW")) { - infoStream.message("IW", "registerMerge info=" + info); + infoStream.message("IW", "registerMerge info=" + segString(info)); } mergingSegments.add(info); } @@ -3306,20 +3296,21 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { checkpoint(); } - Map details = new HashMap(); - details.put("mergeMaxNumSegments", ""+merge.maxNumSegments); - details.put("mergeFactor", Integer.toString(merge.segments.size())); - // Bind a new segment name here so even with // ConcurrentMergePolicy we keep deterministic segment // names. final String mergeSegmentName = newSegmentName(); - SegmentInfo si = new SegmentInfo(directory, Constants.LUCENE_MAIN_VERSION, mergeSegmentName, -1, false, codec, details, null); + SegmentInfo si = new SegmentInfo(directory, Constants.LUCENE_MAIN_VERSION, mergeSegmentName, -1, false, codec, null, null); merge.info = new SegmentInfoPerCommit(si, 0, -1L); // Lock order: IW -> BD bufferedDeletesStream.prune(segmentInfos); + Map details = new HashMap(); + details.put("mergeMaxNumSegments", ""+merge.maxNumSegments); + details.put("mergeFactor", Integer.toString(merge.segments.size())); + setDiagnostics(si, "merge", details); + if (infoStream.isEnabled("IW")) { infoStream.message("IW", "merge seg=" + merge.info.info.name); } @@ -3356,7 +3347,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { /** Does fininishing for a merge, which is fast but holds * the synchronized lock on IndexWriter instance. */ - final synchronized void mergeFinish(MergePolicy.OneMerge merge) throws IOException { + final synchronized void mergeFinish(MergePolicy.OneMerge merge) { // forceMerge, addIndexes or finishMerges may be waiting // on merges to finish. @@ -3417,8 +3408,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { /** Does the actual (time-consuming) work of the merge, * but without holding synchronized lock on IndexWriter * instance */ - private int mergeMiddle(MergePolicy.OneMerge merge) - throws CorruptIndexException, IOException { + private int mergeMiddle(MergePolicy.OneMerge merge) throws IOException { merge.checkAborted(directory); @@ -3439,7 +3429,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { } merge.readers = new ArrayList(); - merge.readerLiveDocs = new ArrayList(); // This is try/finally to make sure merger's readers are // closed: @@ -3453,7 +3442,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // Hold onto the "live" reader; we will use this to // commit merged deletes final ReadersAndLiveDocs rld = readerPool.get(info, true); - final SegmentReader reader = rld.getMergeReader(context); + SegmentReader reader = rld.getMergeReader(context); assert reader != null; // Carefully pull the most recent live docs: @@ -3471,19 +3460,41 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { if (infoStream.isEnabled("IW")) { if (rld.getPendingDeleteCount() != 0) { - infoStream.message("IW", "seg=" + info + " delCount=" + info.getDelCount() + " pendingDelCount=" + rld.getPendingDeleteCount()); + infoStream.message("IW", "seg=" + segString(info) + " delCount=" + info.getDelCount() + " pendingDelCount=" + rld.getPendingDeleteCount()); } else if (info.getDelCount() != 0) { - infoStream.message("IW", "seg=" + info + " delCount=" + info.getDelCount()); + infoStream.message("IW", "seg=" + segString(info) + " delCount=" + info.getDelCount()); } else { - infoStream.message("IW", "seg=" + info + " no deletes"); + infoStream.message("IW", "seg=" + segString(info) + " no deletes"); + } + } + } + + // Deletes might have happened after we pulled the merge reader and + // before we got a read-only copy of the segment's actual live docs + // (taking pending deletes into account). In that case we need to + // make a new reader with updated live docs and del count. + if (reader.numDeletedDocs() != delCount) { + // fix the reader's live docs and del count + assert delCount > reader.numDeletedDocs(); // beware of zombies + + SegmentReader newReader = new SegmentReader(info, reader.core, liveDocs, info.info.getDocCount() - delCount); + boolean released = false; + try { + rld.release(reader); + released = true; + } finally { + if (!released) { + newReader.decRef(); } } + + reader = newReader; } - merge.readerLiveDocs.add(liveDocs); + merge.readers.add(reader); assert delCount <= info.info.getDocCount(): "delCount=" + delCount + " info.docCount=" + info.info.getDocCount() + " rld.pendingDeleteCount=" + rld.getPendingDeleteCount() + " info.getDelCount()=" + info.getDelCount(); if (delCount < info.info.getDocCount()) { - merger.add(reader, liveDocs); + merger.add(reader); } segUpto++; } @@ -3584,7 +3595,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // above: boolean success2 = false; try { - codec.segmentInfoFormat().getSegmentInfosWriter().write(directory, merge.info.info, mergeState.fieldInfos, context); + codec.segmentInfoFormat().getSegmentInfoWriter().write(directory, merge.info.info, mergeState.fieldInfos, context); success2 = true; } finally { if (!success2) { @@ -3660,12 +3671,12 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { } /** @lucene.internal */ - public synchronized String segString() throws IOException { + public synchronized String segString() { return segString(segmentInfos); } /** @lucene.internal */ - public synchronized String segString(Iterable infos) throws IOException { + public synchronized String segString(Iterable infos) { final StringBuilder buffer = new StringBuilder(); for(final SegmentInfoPerCommit info : infos) { if (buffer.length() > 0) { @@ -3677,7 +3688,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { } /** @lucene.internal */ - public synchronized String segString(SegmentInfoPerCommit info) throws IOException { + public synchronized String segString(SegmentInfoPerCommit info) { return info.toString(info.info.dir, numDeletedDocs(info) - info.getDelCount()); } @@ -3877,7 +3888,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { directory.makeLock(IndexWriter.WRITE_LOCK_NAME).release(); } - /** If {@link IndexReader#open(IndexWriter,boolean)} has + /** If {@link DirectoryReader#open(IndexWriter,boolean)} has * been called (ie, this writer is in near real-time * mode), then after a merge completes, this class can be * invoked to warm the reader on the newly merged @@ -4012,7 +4023,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { if (infoStream.isEnabled("IW")) { infoStream.message("IW", "create compound file " + fileName); } - assert Lucene3xSegmentInfoFormat.getDocStoreOffset(info) == -1; // Now merge all added files Collection files = info.files(); CompoundFileDirectory cfsDir = new CompoundFileDirectory(directory, fileName, context, true); diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java index ca1f1d0..2cec18c 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -30,13 +30,11 @@ import org.apache.lucene.util.PrintStreamInfoStream; import org.apache.lucene.util.Version; /** - * Holds all the configuration of {@link IndexWriter}. You - * should instantiate this class, call the setters to set - * your configuration, then pass it to {@link IndexWriter}. - * Note that {@link IndexWriter} makes a private clone; if - * you need to subsequently change settings use {@link - * IndexWriter#getConfig}. - * + * Holds all the configuration that is used to create an {@link IndexWriter}. + * Once {@link IndexWriter} has been created with this object, changes to this + * object will not affect the {@link IndexWriter} instance. For that, use + * {@link LiveIndexWriterConfig} that is returned from {@link IndexWriter#getConfig()}. + * *

    * All setter methods return {@link IndexWriterConfig} to allow chaining * settings conveniently, for example: @@ -45,10 +43,12 @@ import org.apache.lucene.util.Version; * IndexWriterConfig conf = new IndexWriterConfig(analyzer); * conf.setter1().setter2(); * - * + * + * @see IndexWriter#getConfig() + * * @since 3.1 */ -public final class IndexWriterConfig implements Cloneable { +public final class IndexWriterConfig extends LiveIndexWriterConfig implements Cloneable { /** * Specifies the open mode for {@link IndexWriter}. @@ -129,88 +129,39 @@ public final class IndexWriterConfig implements Cloneable { return WRITE_LOCK_TIMEOUT; } - private final Analyzer analyzer; - private volatile IndexDeletionPolicy delPolicy; - private volatile IndexCommit commit; - private volatile OpenMode openMode; - private volatile Similarity similarity; - private volatile int termIndexInterval; // TODO: this should be private to the codec, not settable here - private volatile MergeScheduler mergeScheduler; - private volatile long writeLockTimeout; - private volatile int maxBufferedDeleteTerms; - private volatile double ramBufferSizeMB; - private volatile int maxBufferedDocs; - private volatile IndexingChain indexingChain; - private volatile IndexReaderWarmer mergedSegmentWarmer; - private volatile Codec codec; - private volatile InfoStream infoStream; - private volatile MergePolicy mergePolicy; - private volatile DocumentsWriterPerThreadPool indexerThreadPool; - private volatile boolean readerPooling; - private volatile int readerTermsIndexDivisor; - private volatile FlushPolicy flushPolicy; - private volatile int perThreadHardLimitMB; - - private Version matchVersion; - /** * Creates a new config that with defaults that match the specified * {@link Version} as well as the default {@link - * Analyzer}. If matchVersion is >= {@link - * Version#LUCENE_32}, {@link TieredMergePolicy} is used - * for merging; else {@link LogByteSizeMergePolicy}. + * Analyzer}. By default, {@link TieredMergePolicy} is used + * for merging; * Note that {@link TieredMergePolicy} is free to select * non-contiguous merges, which means docIDs may not - * remain montonic over time. If this is a problem you + * remain monotonic over time. If this is a problem you * should switch to {@link LogByteSizeMergePolicy} or * {@link LogDocMergePolicy}. */ public IndexWriterConfig(Version matchVersion, Analyzer analyzer) { - this.matchVersion = matchVersion; - this.analyzer = analyzer; - delPolicy = new KeepOnlyLastCommitDeletionPolicy(); - commit = null; - openMode = OpenMode.CREATE_OR_APPEND; - similarity = IndexSearcher.getDefaultSimilarity(); - termIndexInterval = DEFAULT_TERM_INDEX_INTERVAL; // TODO: this should be private to the codec, not settable here - mergeScheduler = new ConcurrentMergeScheduler(); - writeLockTimeout = WRITE_LOCK_TIMEOUT; - maxBufferedDeleteTerms = DEFAULT_MAX_BUFFERED_DELETE_TERMS; - ramBufferSizeMB = DEFAULT_RAM_BUFFER_SIZE_MB; - maxBufferedDocs = DEFAULT_MAX_BUFFERED_DOCS; - indexingChain = DocumentsWriterPerThread.defaultIndexingChain; - mergedSegmentWarmer = null; - codec = Codec.getDefault(); - infoStream = InfoStream.getDefault(); - if (matchVersion.onOrAfter(Version.LUCENE_32)) { - mergePolicy = new TieredMergePolicy(); - } else { - mergePolicy = new LogByteSizeMergePolicy(); - } - flushPolicy = new FlushByRamOrCountsPolicy(); - readerPooling = DEFAULT_READER_POOLING; - indexerThreadPool = new ThreadAffinityDocumentsWriterThreadPool(DEFAULT_MAX_THREAD_STATES); - readerTermsIndexDivisor = DEFAULT_READER_TERMS_INDEX_DIVISOR; - perThreadHardLimitMB = DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB; + super(analyzer, matchVersion); } @Override public IndexWriterConfig clone() { - // Shallow clone is the only thing that's possible, since parameters like - // analyzer, index commit etc. do not implement Cloneable. try { - return (IndexWriterConfig)super.clone(); + IndexWriterConfig clone = (IndexWriterConfig) super.clone(); + + // Mostly shallow clone, but do a deepish clone of + // certain objects that have state that cannot be shared + // across IW instances: + clone.flushPolicy = flushPolicy.clone(); + clone.indexerThreadPool = indexerThreadPool.clone(); + clone.mergePolicy = mergePolicy.clone(); + + return clone; } catch (CloneNotSupportedException e) { - // should not happen throw new RuntimeException(e); } } - - /** Returns the default analyzer to use for indexing documents. */ - public Analyzer getAnalyzer() { - return analyzer; - } - + /** Specifies {@link OpenMode} of the index. * *

    Only takes effect when IndexWriter is first created. */ @@ -219,7 +170,7 @@ public final class IndexWriterConfig implements Cloneable { return this; } - /** Returns the {@link OpenMode} set by {@link #setOpenMode(OpenMode)}. */ + @Override public OpenMode getOpenMode() { return openMode; } @@ -246,11 +197,7 @@ public final class IndexWriterConfig implements Cloneable { return this; } - /** - * Returns the {@link IndexDeletionPolicy} specified in - * {@link #setIndexDeletionPolicy(IndexDeletionPolicy)} or the default - * {@link KeepOnlyLastCommitDeletionPolicy}/ - */ + @Override public IndexDeletionPolicy getIndexDeletionPolicy() { return delPolicy; } @@ -265,11 +212,7 @@ public final class IndexWriterConfig implements Cloneable { return this; } - /** - * Returns the {@link IndexCommit} as specified in - * {@link #setIndexCommit(IndexCommit)} or the default, null - * which specifies to open the latest index commit point. - */ + @Override public IndexCommit getIndexCommit() { return commit; } @@ -286,53 +229,12 @@ public final class IndexWriterConfig implements Cloneable { return this; } - /** - * Expert: returns the {@link Similarity} implementation used by this - * IndexWriter. - */ + @Override public Similarity getSimilarity() { return similarity; } /** - * Expert: set the interval between indexed terms. Large values cause less - * memory to be used by IndexReader, but slow random-access to terms. Small - * values cause more memory to be used by an IndexReader, and speed - * random-access to terms. - *

    - * This parameter determines the amount of computation required per query - * term, regardless of the number of documents that contain that term. In - * particular, it is the maximum number of other terms that must be scanned - * before a term is located and its frequency and position information may be - * processed. In a large index with user-entered query terms, query processing - * time is likely to be dominated not by term lookup but rather by the - * processing of frequency and positional data. In a small index or when many - * uncommon query terms are generated (e.g., by wildcard queries) term lookup - * may become a dominant cost. - *

    - * In particular, numUniqueTerms/interval terms are read into - * memory by an IndexReader, and, on average, interval/2 terms - * must be scanned for each random term access. - * - * @see #DEFAULT_TERM_INDEX_INTERVAL - * - *

    Takes effect immediately, but only applies to newly - * flushed/merged segments. */ - public IndexWriterConfig setTermIndexInterval(int interval) { // TODO: this should be private to the codec, not settable here - this.termIndexInterval = interval; - return this; - } - - /** - * Returns the interval between indexed terms. - * - * @see #setTermIndexInterval(int) - */ - public int getTermIndexInterval() { // TODO: this should be private to the codec, not settable here - return termIndexInterval; - } - - /** * Expert: sets the merge scheduler used by this writer. The default is * {@link ConcurrentMergeScheduler}. *

    @@ -345,10 +247,7 @@ public final class IndexWriterConfig implements Cloneable { return this; } - /** - * Returns the {@link MergeScheduler} that was set by - * {@link #setMergeScheduler(MergeScheduler)} - */ + @Override public MergeScheduler getMergeScheduler() { return mergeScheduler; } @@ -364,174 +263,12 @@ public final class IndexWriterConfig implements Cloneable { return this; } - /** - * Returns allowed timeout when acquiring the write lock. - * - * @see #setWriteLockTimeout(long) - */ + @Override public long getWriteLockTimeout() { return writeLockTimeout; } /** - * Determines the minimal number of delete terms required before the buffered - * in-memory delete terms and queries are applied and flushed. - *

    Disabled by default (writer flushes by RAM usage).

    - *

    - * NOTE: This setting won't trigger a segment flush. - *

    - * - * @throws IllegalArgumentException if maxBufferedDeleteTerms - * is enabled but smaller than 1 - * @see #setRAMBufferSizeMB - * @see #setFlushPolicy(FlushPolicy) - * - *

    Takes effect immediately, but only the next time a - * document is added, updated or deleted. - */ - public IndexWriterConfig setMaxBufferedDeleteTerms(int maxBufferedDeleteTerms) { - if (maxBufferedDeleteTerms != DISABLE_AUTO_FLUSH - && maxBufferedDeleteTerms < 1) - throw new IllegalArgumentException( - "maxBufferedDeleteTerms must at least be 1 when enabled"); - this.maxBufferedDeleteTerms = maxBufferedDeleteTerms; - return this; - } - - /** - * Returns the number of buffered deleted terms that will trigger a flush of all - * buffered deletes if enabled. - * - * @see #setMaxBufferedDeleteTerms(int) - */ - public int getMaxBufferedDeleteTerms() { - return maxBufferedDeleteTerms; - } - - /** - * Determines the amount of RAM that may be used for buffering added documents - * and deletions before they are flushed to the Directory. Generally for - * faster indexing performance it's best to flush by RAM usage instead of - * document count and use as large a RAM buffer as you can. - *

    - * When this is set, the writer will flush whenever buffered documents and - * deletions use this much RAM. Pass in {@link #DISABLE_AUTO_FLUSH} to prevent - * triggering a flush due to RAM usage. Note that if flushing by document - * count is also enabled, then the flush will be triggered by whichever comes - * first. - *

    - * The maximum RAM limit is inherently determined by the JVMs available memory. - * Yet, an {@link IndexWriter} session can consume a significantly larger amount - * of memory than the given RAM limit since this limit is just an indicator when - * to flush memory resident documents to the Directory. Flushes are likely happen - * concurrently while other threads adding documents to the writer. For application - * stability the available memory in the JVM should be significantly larger than - * the RAM buffer used for indexing. - *

    - * NOTE: the account of RAM usage for pending deletions is only - * approximate. Specifically, if you delete by Query, Lucene currently has no - * way to measure the RAM usage of individual Queries so the accounting will - * under-estimate and you should compensate by either calling commit() - * periodically yourself, or by using {@link #setMaxBufferedDeleteTerms(int)} - * to flush and apply buffered deletes by count instead of RAM usage - * (for each buffered delete Query a constant number of bytes is used to estimate - * RAM usage). Note that enabling {@link #setMaxBufferedDeleteTerms(int)} will - * not trigger any segment flushes. - *

    - * NOTE: It's not guaranteed that all memory resident documents are flushed - * once this limit is exceeded. Depending on the configured {@link FlushPolicy} only a - * subset of the buffered documents are flushed and therefore only parts of the RAM - * buffer is released. - *

    - * - * The default value is {@link #DEFAULT_RAM_BUFFER_SIZE_MB}. - * @see #setFlushPolicy(FlushPolicy) - * @see #setRAMPerThreadHardLimitMB(int) - * - *

    Takes effect immediately, but only the next time a - * document is added, updated or deleted. - * - * @throws IllegalArgumentException - * if ramBufferSize is enabled but non-positive, or it disables - * ramBufferSize when maxBufferedDocs is already disabled - * - */ - public IndexWriterConfig setRAMBufferSizeMB(double ramBufferSizeMB) { - if (ramBufferSizeMB != DISABLE_AUTO_FLUSH && ramBufferSizeMB <= 0.0) - throw new IllegalArgumentException( - "ramBufferSize should be > 0.0 MB when enabled"); - if (ramBufferSizeMB == DISABLE_AUTO_FLUSH && maxBufferedDocs == DISABLE_AUTO_FLUSH) - throw new IllegalArgumentException( - "at least one of ramBufferSize and maxBufferedDocs must be enabled"); - this.ramBufferSizeMB = ramBufferSizeMB; - return this; - } - - /** Returns the value set by {@link #setRAMBufferSizeMB(double)} if enabled. */ - public double getRAMBufferSizeMB() { - return ramBufferSizeMB; - } - - /** - * Determines the minimal number of documents required before the buffered - * in-memory documents are flushed as a new Segment. Large values generally - * give faster indexing. - * - *

    - * When this is set, the writer will flush every maxBufferedDocs added - * documents. Pass in {@link #DISABLE_AUTO_FLUSH} to prevent triggering a - * flush due to number of buffered documents. Note that if flushing by RAM - * usage is also enabled, then the flush will be triggered by whichever comes - * first. - * - *

    - * Disabled by default (writer flushes by RAM usage). - * - *

    Takes effect immediately, but only the next time a - * document is added, updated or deleted. - * - * @see #setRAMBufferSizeMB(double) - * @see #setFlushPolicy(FlushPolicy) - * @throws IllegalArgumentException - * if maxBufferedDocs is enabled but smaller than 2, or it disables - * maxBufferedDocs when ramBufferSize is already disabled - */ - public IndexWriterConfig setMaxBufferedDocs(int maxBufferedDocs) { - if (maxBufferedDocs != DISABLE_AUTO_FLUSH && maxBufferedDocs < 2) - throw new IllegalArgumentException( - "maxBufferedDocs must at least be 2 when enabled"); - if (maxBufferedDocs == DISABLE_AUTO_FLUSH - && ramBufferSizeMB == DISABLE_AUTO_FLUSH) - throw new IllegalArgumentException( - "at least one of ramBufferSize and maxBufferedDocs must be enabled"); - this.maxBufferedDocs = maxBufferedDocs; - return this; - } - - /** - * Returns the number of buffered added documents that will trigger a flush if - * enabled. - * - * @see #setMaxBufferedDocs(int) - */ - public int getMaxBufferedDocs() { - return maxBufferedDocs; - } - - /** Set the merged segment warmer. See {@link IndexReaderWarmer}. - * - *

    Takes effect on the next merge. */ - public IndexWriterConfig setMergedSegmentWarmer(IndexReaderWarmer mergeSegmentWarmer) { - this.mergedSegmentWarmer = mergeSegmentWarmer; - return this; - } - - /** Returns the current merged segment warmer. See {@link IndexReaderWarmer}. */ - public IndexReaderWarmer getMergedSegmentWarmer() { - return mergedSegmentWarmer; - } - - /** * Expert: {@link MergePolicy} is invoked whenever there are changes to the * segments in the index. Its role is to select which merges to do, if any, * and return a {@link MergePolicy.MergeSpecification} describing the merges. @@ -544,25 +281,24 @@ public final class IndexWriterConfig implements Cloneable { return this; } - /** Set the Codec. See {@link Codec}. - * - *

    Only takes effect when IndexWriter is first created. */ + /** + * Set the {@link Codec}. + * + *

    + * Only takes effect when IndexWriter is first created. + */ public IndexWriterConfig setCodec(Codec codec) { this.codec = codec; return this; } - /** Returns the current Codec. See {@link Codec}. */ + @Override public Codec getCodec() { return codec; } - /** - * Returns the current MergePolicy in use by this writer. - * - * @see #setMergePolicy(MergePolicy) - */ + @Override public MergePolicy getMergePolicy() { return mergePolicy; } @@ -583,17 +319,15 @@ public final class IndexWriterConfig implements Cloneable { * NOTE: This only takes effect when IndexWriter is first created.

    */ IndexWriterConfig setIndexerThreadPool(DocumentsWriterPerThreadPool threadPool) { if (threadPool == null) { - throw new IllegalArgumentException("DocumentsWriterPerThreadPool must not be nul"); + throw new IllegalArgumentException("threadPool must not be null"); } this.indexerThreadPool = threadPool; return this; } - /** Returns the configured {@link DocumentsWriterPerThreadPool} instance. - * @see #setIndexerThreadPool(DocumentsWriterPerThreadPool) - * @return the configured {@link DocumentsWriterPerThreadPool} instance.*/ + @Override DocumentsWriterPerThreadPool getIndexerThreadPool() { - return this.indexerThreadPool; + return indexerThreadPool; } /** @@ -608,8 +342,7 @@ public final class IndexWriterConfig implements Cloneable { return this; } - /** Returns the max number of simultaneous threads that - * may be indexing documents at once in IndexWriter. */ + @Override public int getMaxThreadStates() { try { return ((ThreadAffinityDocumentsWriterThreadPool) indexerThreadPool).getMaxThreadStates(); @@ -633,8 +366,7 @@ public final class IndexWriterConfig implements Cloneable { return this; } - /** Returns true if IndexWriter should pool readers even - * if {@link DirectoryReader#open(IndexWriter, boolean)} has not been called. */ + @Override public boolean getReaderPooling() { return readerPooling; } @@ -647,34 +379,11 @@ public final class IndexWriterConfig implements Cloneable { return this; } - /** Returns the indexing chain set on {@link #setIndexingChain(IndexingChain)}. */ + @Override IndexingChain getIndexingChain() { return indexingChain; } - /** Sets the termsIndexDivisor passed to any readers that - * IndexWriter opens, for example when applying deletes - * or creating a near-real-time reader in {@link - * DirectoryReader#open(IndexWriter, boolean)}. If you pass -1, the terms index - * won't be loaded by the readers. This is only useful in - * advanced situations when you will only .next() through - * all terms; attempts to seek will hit an exception. - * - *

    Takes effect immediately, but only applies to - * readers opened after this call */ - public IndexWriterConfig setReaderTermsIndexDivisor(int divisor) { - if (divisor <= 0 && divisor != -1) { - throw new IllegalArgumentException("divisor must be >= 1, or -1 (got " + divisor + ")"); - } - readerTermsIndexDivisor = divisor; - return this; - } - - /** @see #setReaderTermsIndexDivisor(int) */ - public int getReaderTermsIndexDivisor() { - return readerTermsIndexDivisor; - } - /** * Expert: Controls when segments are flushed to disk during indexing. * The {@link FlushPolicy} initialized during {@link IndexWriter} instantiation and once initialized @@ -707,28 +416,56 @@ public final class IndexWriterConfig implements Cloneable { return this; } - /** - * Returns the max amount of memory each {@link DocumentsWriterPerThread} can - * consume until forcefully flushed. - * @see #setRAMPerThreadHardLimitMB(int) - */ + @Override public int getRAMPerThreadHardLimitMB() { return perThreadHardLimitMB; } - /** - * @see #setFlushPolicy(FlushPolicy) - */ + + @Override public FlushPolicy getFlushPolicy() { return flushPolicy; } - /** - * @see #setInfoStream(InfoStream) - */ + @Override public InfoStream getInfoStream() { return infoStream; } + @Override + public Analyzer getAnalyzer() { + return super.getAnalyzer(); + } + + @Override + public int getMaxBufferedDeleteTerms() { + return super.getMaxBufferedDeleteTerms(); + } + + @Override + public int getMaxBufferedDocs() { + return super.getMaxBufferedDocs(); + } + + @Override + public IndexReaderWarmer getMergedSegmentWarmer() { + return super.getMergedSegmentWarmer(); + } + + @Override + public double getRAMBufferSizeMB() { + return super.getRAMBufferSizeMB(); + } + + @Override + public int getReaderTermsIndexDivisor() { + return super.getReaderTermsIndexDivisor(); + } + + @Override + public int getTermIndexInterval() { + return super.getTermIndexInterval(); + } + /** If non-null, information about merges, deletes and a * message when maxFieldLength is reached will be printed * to this. @@ -742,40 +479,39 @@ public final class IndexWriterConfig implements Cloneable { return this; } - /** - * Convenience method that uses {@link PrintStreamInfoStream} - */ + /** Convenience method that uses {@link PrintStreamInfoStream} */ public IndexWriterConfig setInfoStream(PrintStream printStream) { return setInfoStream(printStream == null ? InfoStream.NO_OUTPUT : new PrintStreamInfoStream(printStream)); } - + + @Override + public IndexWriterConfig setMaxBufferedDeleteTerms(int maxBufferedDeleteTerms) { + return (IndexWriterConfig) super.setMaxBufferedDeleteTerms(maxBufferedDeleteTerms); + } + + @Override + public IndexWriterConfig setMaxBufferedDocs(int maxBufferedDocs) { + return (IndexWriterConfig) super.setMaxBufferedDocs(maxBufferedDocs); + } + + @Override + public IndexWriterConfig setMergedSegmentWarmer(IndexReaderWarmer mergeSegmentWarmer) { + return (IndexWriterConfig) super.setMergedSegmentWarmer(mergeSegmentWarmer); + } + + @Override + public IndexWriterConfig setRAMBufferSizeMB(double ramBufferSizeMB) { + return (IndexWriterConfig) super.setRAMBufferSizeMB(ramBufferSizeMB); + } + + @Override + public IndexWriterConfig setReaderTermsIndexDivisor(int divisor) { + return (IndexWriterConfig) super.setReaderTermsIndexDivisor(divisor); + } + @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("matchVersion=").append(matchVersion).append("\n"); - sb.append("analyzer=").append(analyzer == null ? "null" : analyzer.getClass().getName()).append("\n"); - sb.append("delPolicy=").append(delPolicy.getClass().getName()).append("\n"); - sb.append("commit=").append(commit == null ? "null" : commit).append("\n"); - sb.append("openMode=").append(openMode).append("\n"); - sb.append("similarity=").append(similarity.getClass().getName()).append("\n"); - sb.append("termIndexInterval=").append(termIndexInterval).append("\n"); // TODO: this should be private to the codec, not settable here - sb.append("mergeScheduler=").append(mergeScheduler.getClass().getName()).append("\n"); - sb.append("default WRITE_LOCK_TIMEOUT=").append(WRITE_LOCK_TIMEOUT).append("\n"); - sb.append("writeLockTimeout=").append(writeLockTimeout).append("\n"); - sb.append("maxBufferedDeleteTerms=").append(maxBufferedDeleteTerms).append("\n"); - sb.append("ramBufferSizeMB=").append(ramBufferSizeMB).append("\n"); - sb.append("maxBufferedDocs=").append(maxBufferedDocs).append("\n"); - sb.append("mergedSegmentWarmer=").append(mergedSegmentWarmer).append("\n"); - sb.append("codec=").append(codec).append("\n"); - sb.append("infoStream=").append(infoStream.getClass().getName()).append("\n"); - sb.append("mergePolicy=").append(mergePolicy).append("\n"); - sb.append("indexerThreadPool=").append(indexerThreadPool).append("\n"); - sb.append("readerPooling=").append(readerPooling).append("\n"); - sb.append("readerTermsIndexDivisor=").append(readerTermsIndexDivisor).append("\n"); - sb.append("flushPolicy=").append(flushPolicy).append("\n"); - sb.append("perThreadHardLimitMB=").append(perThreadHardLimitMB).append("\n"); - - return sb.toString(); + public IndexWriterConfig setTermIndexInterval(int interval) { + return (IndexWriterConfig) super.setTermIndexInterval(interval); } } diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexableField.java b/lucene/core/src/java/org/apache/lucene/index/IndexableField.java index fac46e0..0408bab 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexableField.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexableField.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/IntBlockPool.java b/lucene/core/src/java/org/apache/lucene/index/IntBlockPool.java index 16093a5..531b287 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IntBlockPool.java +++ b/lucene/core/src/java/org/apache/lucene/index/IntBlockPool.java @@ -2,7 +2,7 @@ package org.apache.lucene.index; import java.util.Arrays; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumer.java b/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumer.java index 4093285..d9df651 100644 --- a/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumer.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java b/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java index 20f10fb..2ea307f 100644 --- a/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java +++ b/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/InvertedDocEndConsumer.java b/lucene/core/src/java/org/apache/lucene/index/InvertedDocEndConsumer.java index 9e97d40..bd20aa5 100644 --- a/lucene/core/src/java/org/apache/lucene/index/InvertedDocEndConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/index/InvertedDocEndConsumer.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/InvertedDocEndConsumerPerField.java b/lucene/core/src/java/org/apache/lucene/index/InvertedDocEndConsumerPerField.java index e46ae25..a4df838 100644 --- a/lucene/core/src/java/org/apache/lucene/index/InvertedDocEndConsumerPerField.java +++ b/lucene/core/src/java/org/apache/lucene/index/InvertedDocEndConsumerPerField.java @@ -2,7 +2,7 @@ package org.apache.lucene.index; import java.io.IOException; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/KeepOnlyLastCommitDeletionPolicy.java b/lucene/core/src/java/org/apache/lucene/index/KeepOnlyLastCommitDeletionPolicy.java index 489fd24..d52a1c1 100644 --- a/lucene/core/src/java/org/apache/lucene/index/KeepOnlyLastCommitDeletionPolicy.java +++ b/lucene/core/src/java/org/apache/lucene/index/KeepOnlyLastCommitDeletionPolicy.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java b/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java new file mode 100755 index 0000000..7bcdff9 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java @@ -0,0 +1,505 @@ +package org.apache.lucene.index; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain; +import org.apache.lucene.index.IndexWriter.IndexReaderWarmer; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.Version; + +/** + * Holds all the configuration used by {@link IndexWriter} with few setters for + * settings that can be changed on an {@link IndexWriter} instance "live". + * + * @since 4.0 + */ +public class LiveIndexWriterConfig { + + private final Analyzer analyzer; + + private volatile int maxBufferedDocs; + private volatile double ramBufferSizeMB; + private volatile int maxBufferedDeleteTerms; + private volatile int readerTermsIndexDivisor; + private volatile IndexReaderWarmer mergedSegmentWarmer; + private volatile int termIndexInterval; // TODO: this should be private to the codec, not settable here + + // modified by IndexWriterConfig + protected volatile IndexDeletionPolicy delPolicy; + protected volatile IndexCommit commit; + protected volatile OpenMode openMode; + protected volatile Similarity similarity; + protected volatile MergeScheduler mergeScheduler; + protected volatile long writeLockTimeout; + protected volatile IndexingChain indexingChain; + protected volatile Codec codec; + protected volatile InfoStream infoStream; + protected volatile MergePolicy mergePolicy; + protected volatile DocumentsWriterPerThreadPool indexerThreadPool; + protected volatile boolean readerPooling; + protected volatile FlushPolicy flushPolicy; + protected volatile int perThreadHardLimitMB; + + protected final Version matchVersion; + + // used by IndexWriterConfig + LiveIndexWriterConfig(Analyzer analyzer, Version matchVersion) { + this.analyzer = analyzer; + this.matchVersion = matchVersion; + ramBufferSizeMB = IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB; + maxBufferedDocs = IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS; + maxBufferedDeleteTerms = IndexWriterConfig.DEFAULT_MAX_BUFFERED_DELETE_TERMS; + readerTermsIndexDivisor = IndexWriterConfig.DEFAULT_READER_TERMS_INDEX_DIVISOR; + mergedSegmentWarmer = null; + termIndexInterval = IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL; // TODO: this should be private to the codec, not settable here + delPolicy = new KeepOnlyLastCommitDeletionPolicy(); + commit = null; + openMode = OpenMode.CREATE_OR_APPEND; + similarity = IndexSearcher.getDefaultSimilarity(); + mergeScheduler = new ConcurrentMergeScheduler(); + writeLockTimeout = IndexWriterConfig.WRITE_LOCK_TIMEOUT; + indexingChain = DocumentsWriterPerThread.defaultIndexingChain; + codec = Codec.getDefault(); + infoStream = InfoStream.getDefault(); + mergePolicy = new TieredMergePolicy(); + flushPolicy = new FlushByRamOrCountsPolicy(); + readerPooling = IndexWriterConfig.DEFAULT_READER_POOLING; + indexerThreadPool = new ThreadAffinityDocumentsWriterThreadPool(IndexWriterConfig.DEFAULT_MAX_THREAD_STATES); + perThreadHardLimitMB = IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB; + } + + /** + * Creates a new config that that handles the live {@link IndexWriter} + * settings. + */ + LiveIndexWriterConfig(IndexWriterConfig config) { + maxBufferedDeleteTerms = config.getMaxBufferedDeleteTerms(); + maxBufferedDocs = config.getMaxBufferedDocs(); + mergedSegmentWarmer = config.getMergedSegmentWarmer(); + ramBufferSizeMB = config.getRAMBufferSizeMB(); + readerTermsIndexDivisor = config.getReaderTermsIndexDivisor(); + termIndexInterval = config.getTermIndexInterval(); + matchVersion = config.matchVersion; + analyzer = config.getAnalyzer(); + delPolicy = config.getIndexDeletionPolicy(); + commit = config.getIndexCommit(); + openMode = config.getOpenMode(); + similarity = config.getSimilarity(); + mergeScheduler = config.getMergeScheduler(); + writeLockTimeout = config.getWriteLockTimeout(); + indexingChain = config.getIndexingChain(); + codec = config.getCodec(); + infoStream = config.getInfoStream(); + mergePolicy = config.getMergePolicy(); + indexerThreadPool = config.getIndexerThreadPool(); + readerPooling = config.getReaderPooling(); + flushPolicy = config.getFlushPolicy(); + perThreadHardLimitMB = config.getRAMPerThreadHardLimitMB(); + } + + /** Returns the default analyzer to use for indexing documents. */ + public Analyzer getAnalyzer() { + return analyzer; + } + + /** + * Expert: set the interval between indexed terms. Large values cause less + * memory to be used by IndexReader, but slow random-access to terms. Small + * values cause more memory to be used by an IndexReader, and speed + * random-access to terms. + *

    + * This parameter determines the amount of computation required per query + * term, regardless of the number of documents that contain that term. In + * particular, it is the maximum number of other terms that must be scanned + * before a term is located and its frequency and position information may be + * processed. In a large index with user-entered query terms, query processing + * time is likely to be dominated not by term lookup but rather by the + * processing of frequency and positional data. In a small index or when many + * uncommon query terms are generated (e.g., by wildcard queries) term lookup + * may become a dominant cost. + *

    + * In particular, numUniqueTerms/interval terms are read into + * memory by an IndexReader, and, on average, interval/2 terms + * must be scanned for each random term access. + * + *

    + * Takes effect immediately, but only applies to newly flushed/merged + * segments. + * + * @see IndexWriterConfig#DEFAULT_TERM_INDEX_INTERVAL + */ + public LiveIndexWriterConfig setTermIndexInterval(int interval) { // TODO: this should be private to the codec, not settable here + this.termIndexInterval = interval; + return this; + } + + /** + * Returns the interval between indexed terms. + * + * @see #setTermIndexInterval(int) + */ + public int getTermIndexInterval() { // TODO: this should be private to the codec, not settable here + return termIndexInterval; + } + + /** + * Determines the minimal number of delete terms required before the buffered + * in-memory delete terms and queries are applied and flushed. + *

    + * Disabled by default (writer flushes by RAM usage). + *

    + * NOTE: This setting won't trigger a segment flush. + * + *

    + * Takes effect immediately, but only the next time a document is added, + * updated or deleted. + * + * @throws IllegalArgumentException + * if maxBufferedDeleteTerms is enabled but smaller than 1 + * + * @see #setRAMBufferSizeMB + */ + public LiveIndexWriterConfig setMaxBufferedDeleteTerms(int maxBufferedDeleteTerms) { + if (maxBufferedDeleteTerms != IndexWriterConfig.DISABLE_AUTO_FLUSH && maxBufferedDeleteTerms < 1) { + throw new IllegalArgumentException("maxBufferedDeleteTerms must at least be 1 when enabled"); + } + this.maxBufferedDeleteTerms = maxBufferedDeleteTerms; + return this; + } + + /** + * Returns the number of buffered deleted terms that will trigger a flush of all + * buffered deletes if enabled. + * + * @see #setMaxBufferedDeleteTerms(int) + */ + public int getMaxBufferedDeleteTerms() { + return maxBufferedDeleteTerms; + } + + /** + * Determines the amount of RAM that may be used for buffering added documents + * and deletions before they are flushed to the Directory. Generally for + * faster indexing performance it's best to flush by RAM usage instead of + * document count and use as large a RAM buffer as you can. + *

    + * When this is set, the writer will flush whenever buffered documents and + * deletions use this much RAM. Pass in + * {@link IndexWriterConfig#DISABLE_AUTO_FLUSH} to prevent triggering a flush + * due to RAM usage. Note that if flushing by document count is also enabled, + * then the flush will be triggered by whichever comes first. + *

    + * The maximum RAM limit is inherently determined by the JVMs available + * memory. Yet, an {@link IndexWriter} session can consume a significantly + * larger amount of memory than the given RAM limit since this limit is just + * an indicator when to flush memory resident documents to the Directory. + * Flushes are likely happen concurrently while other threads adding documents + * to the writer. For application stability the available memory in the JVM + * should be significantly larger than the RAM buffer used for indexing. + *

    + * NOTE: the account of RAM usage for pending deletions is only + * approximate. Specifically, if you delete by Query, Lucene currently has no + * way to measure the RAM usage of individual Queries so the accounting will + * under-estimate and you should compensate by either calling commit() + * periodically yourself, or by using {@link #setMaxBufferedDeleteTerms(int)} + * to flush and apply buffered deletes by count instead of RAM usage (for each + * buffered delete Query a constant number of bytes is used to estimate RAM + * usage). Note that enabling {@link #setMaxBufferedDeleteTerms(int)} will not + * trigger any segment flushes. + *

    + * NOTE: It's not guaranteed that all memory resident documents are + * flushed once this limit is exceeded. Depending on the configured + * {@link FlushPolicy} only a subset of the buffered documents are flushed and + * therefore only parts of the RAM buffer is released. + *

    + * + * The default value is {@link IndexWriterConfig#DEFAULT_RAM_BUFFER_SIZE_MB}. + * + *

    + * Takes effect immediately, but only the next time a document is added, + * updated or deleted. + * + * @see IndexWriterConfig#setRAMPerThreadHardLimitMB(int) + * + * @throws IllegalArgumentException + * if ramBufferSize is enabled but non-positive, or it disables + * ramBufferSize when maxBufferedDocs is already disabled + */ + public LiveIndexWriterConfig setRAMBufferSizeMB(double ramBufferSizeMB) { + if (ramBufferSizeMB != IndexWriterConfig.DISABLE_AUTO_FLUSH && ramBufferSizeMB <= 0.0) { + throw new IllegalArgumentException("ramBufferSize should be > 0.0 MB when enabled"); + } + if (ramBufferSizeMB == IndexWriterConfig.DISABLE_AUTO_FLUSH + && maxBufferedDocs == IndexWriterConfig.DISABLE_AUTO_FLUSH) { + throw new IllegalArgumentException("at least one of ramBufferSize and maxBufferedDocs must be enabled"); + } + this.ramBufferSizeMB = ramBufferSizeMB; + return this; + } + + /** Returns the value set by {@link #setRAMBufferSizeMB(double)} if enabled. */ + public double getRAMBufferSizeMB() { + return ramBufferSizeMB; + } + + /** + * Determines the minimal number of documents required before the buffered + * in-memory documents are flushed as a new Segment. Large values generally + * give faster indexing. + * + *

    + * When this is set, the writer will flush every maxBufferedDocs added + * documents. Pass in {@link IndexWriterConfig#DISABLE_AUTO_FLUSH} to prevent + * triggering a flush due to number of buffered documents. Note that if + * flushing by RAM usage is also enabled, then the flush will be triggered by + * whichever comes first. + * + *

    + * Disabled by default (writer flushes by RAM usage). + * + *

    + * Takes effect immediately, but only the next time a document is added, + * updated or deleted. + * + * @see #setRAMBufferSizeMB(double) + * @throws IllegalArgumentException + * if maxBufferedDocs is enabled but smaller than 2, or it disables + * maxBufferedDocs when ramBufferSize is already disabled + */ + public LiveIndexWriterConfig setMaxBufferedDocs(int maxBufferedDocs) { + if (maxBufferedDocs != IndexWriterConfig.DISABLE_AUTO_FLUSH && maxBufferedDocs < 2) { + throw new IllegalArgumentException("maxBufferedDocs must at least be 2 when enabled"); + } + if (maxBufferedDocs == IndexWriterConfig.DISABLE_AUTO_FLUSH + && ramBufferSizeMB == IndexWriterConfig.DISABLE_AUTO_FLUSH) { + throw new IllegalArgumentException("at least one of ramBufferSize and maxBufferedDocs must be enabled"); + } + this.maxBufferedDocs = maxBufferedDocs; + return this; + } + + /** + * Returns the number of buffered added documents that will trigger a flush if + * enabled. + * + * @see #setMaxBufferedDocs(int) + */ + public int getMaxBufferedDocs() { + return maxBufferedDocs; + } + + /** + * Set the merged segment warmer. See {@link IndexReaderWarmer}. + * + *

    + * Takes effect on the next merge. + */ + public LiveIndexWriterConfig setMergedSegmentWarmer(IndexReaderWarmer mergeSegmentWarmer) { + this.mergedSegmentWarmer = mergeSegmentWarmer; + return this; + } + + /** Returns the current merged segment warmer. See {@link IndexReaderWarmer}. */ + public IndexReaderWarmer getMergedSegmentWarmer() { + return mergedSegmentWarmer; + } + + /** + * Sets the termsIndexDivisor passed to any readers that IndexWriter opens, + * for example when applying deletes or creating a near-real-time reader in + * {@link DirectoryReader#open(IndexWriter, boolean)}. If you pass -1, the + * terms index won't be loaded by the readers. This is only useful in advanced + * situations when you will only .next() through all terms; attempts to seek + * will hit an exception. + * + *

    + * Takes effect immediately, but only applies to readers opened after this + * call + */ + public LiveIndexWriterConfig setReaderTermsIndexDivisor(int divisor) { + if (divisor <= 0 && divisor != -1) { + throw new IllegalArgumentException("divisor must be >= 1, or -1 (got " + divisor + ")"); + } + readerTermsIndexDivisor = divisor; + return this; + } + + /** @see #setReaderTermsIndexDivisor(int) */ + public int getReaderTermsIndexDivisor() { + return readerTermsIndexDivisor; + } + + /** Returns the {@link OpenMode} set by {@link IndexWriterConfig#setOpenMode(OpenMode)}. */ + public OpenMode getOpenMode() { + return openMode; + } + + /** + * Returns the {@link IndexDeletionPolicy} specified in + * {@link IndexWriterConfig#setIndexDeletionPolicy(IndexDeletionPolicy)} or + * the default {@link KeepOnlyLastCommitDeletionPolicy}/ + */ + public IndexDeletionPolicy getIndexDeletionPolicy() { + return delPolicy; + } + + /** + * Returns the {@link IndexCommit} as specified in + * {@link IndexWriterConfig#setIndexCommit(IndexCommit)} or the default, + * {@code null} which specifies to open the latest index commit point. + */ + public IndexCommit getIndexCommit() { + return commit; + } + + /** + * Expert: returns the {@link Similarity} implementation used by this + * {@link IndexWriter}. + */ + public Similarity getSimilarity() { + return similarity; + } + + /** + * Returns the {@link MergeScheduler} that was set by + * {@link IndexWriterConfig#setMergeScheduler(MergeScheduler)}. + */ + public MergeScheduler getMergeScheduler() { + return mergeScheduler; + } + + /** + * Returns allowed timeout when acquiring the write lock. + * + * @see IndexWriterConfig#setWriteLockTimeout(long) + */ + public long getWriteLockTimeout() { + return writeLockTimeout; + } + + /** Returns the current {@link Codec}. */ + public Codec getCodec() { + return codec; + } + + /** + * Returns the current MergePolicy in use by this writer. + * + * @see IndexWriterConfig#setMergePolicy(MergePolicy) + */ + public MergePolicy getMergePolicy() { + return mergePolicy; + } + + /** + * Returns the configured {@link DocumentsWriterPerThreadPool} instance. + * + * @see IndexWriterConfig#setIndexerThreadPool(DocumentsWriterPerThreadPool) + * @return the configured {@link DocumentsWriterPerThreadPool} instance. + */ + DocumentsWriterPerThreadPool getIndexerThreadPool() { + return indexerThreadPool; + } + + /** + * Returns the max number of simultaneous threads that may be indexing + * documents at once in IndexWriter. + */ + public int getMaxThreadStates() { + try { + return ((ThreadAffinityDocumentsWriterThreadPool) indexerThreadPool).getMaxThreadStates(); + } catch (ClassCastException cce) { + throw new IllegalStateException(cce); + } + } + + /** + * Returns {@code true} if {@link IndexWriter} should pool readers even if + * {@link DirectoryReader#open(IndexWriter, boolean)} has not been called. + */ + public boolean getReaderPooling() { + return readerPooling; + } + + /** + * Returns the indexing chain set on + * {@link IndexWriterConfig#setIndexingChain(IndexingChain)}. + */ + IndexingChain getIndexingChain() { + return indexingChain; + } + + /** + * Returns the max amount of memory each {@link DocumentsWriterPerThread} can + * consume until forcefully flushed. + * + * @see IndexWriterConfig#setRAMPerThreadHardLimitMB(int) + */ + public int getRAMPerThreadHardLimitMB() { + return perThreadHardLimitMB; + } + + /** + * @see IndexWriterConfig#setFlushPolicy(FlushPolicy) + */ + public FlushPolicy getFlushPolicy() { + return flushPolicy; + } + + /** + * @see IndexWriterConfig#setInfoStream(InfoStream) + */ + public InfoStream getInfoStream() { + return infoStream; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("matchVersion=").append(matchVersion).append("\n"); + sb.append("analyzer=").append(analyzer == null ? "null" : analyzer.getClass().getName()).append("\n"); + sb.append("ramBufferSizeMB=").append(getRAMBufferSizeMB()).append("\n"); + sb.append("maxBufferedDocs=").append(getMaxBufferedDocs()).append("\n"); + sb.append("maxBufferedDeleteTerms=").append(getMaxBufferedDeleteTerms()).append("\n"); + sb.append("mergedSegmentWarmer=").append(getMergeScheduler()).append("\n"); + sb.append("readerTermsIndexDivisor=").append(getReaderTermsIndexDivisor()).append("\n"); + sb.append("termIndexInterval=").append(getTermIndexInterval()).append("\n"); // TODO: this should be private to the codec, not settable here + sb.append("delPolicy=").append(getIndexDeletionPolicy().getClass().getName()).append("\n"); + IndexCommit commit = getIndexCommit(); + sb.append("commit=").append(commit == null ? "null" : commit).append("\n"); + sb.append("openMode=").append(getOpenMode()).append("\n"); + sb.append("similarity=").append(getSimilarity().getClass().getName()).append("\n"); + sb.append("mergeScheduler=").append(getMergeScheduler().getClass().getName()).append("\n"); + sb.append("default WRITE_LOCK_TIMEOUT=").append(IndexWriterConfig.WRITE_LOCK_TIMEOUT).append("\n"); + sb.append("writeLockTimeout=").append(getWriteLockTimeout()).append("\n"); + sb.append("codec=").append(getCodec()).append("\n"); + sb.append("infoStream=").append(getInfoStream().getClass().getName()).append("\n"); + sb.append("mergePolicy=").append(getMergePolicy()).append("\n"); + sb.append("indexerThreadPool=").append(getIndexerThreadPool()).append("\n"); + sb.append("readerPooling=").append(getReaderPooling()).append("\n"); + sb.append("flushPolicy=").append(getFlushPolicy()).append("\n"); + sb.append("perThreadHardLimitMB=").append(getRAMPerThreadHardLimitMB()).append("\n"); + return sb.toString(); + } + +} diff --git a/lucene/core/src/java/org/apache/lucene/index/LogByteSizeMergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/LogByteSizeMergePolicy.java index cc46cfd..99941bd 100644 --- a/lucene/core/src/java/org/apache/lucene/index/LogByteSizeMergePolicy.java +++ b/lucene/core/src/java/org/apache/lucene/index/LogByteSizeMergePolicy.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/LogDocMergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/LogDocMergePolicy.java index bb0a198..622847c 100644 --- a/lucene/core/src/java/org/apache/lucene/index/LogDocMergePolicy.java +++ b/lucene/core/src/java/org/apache/lucene/index/LogDocMergePolicy.java @@ -2,7 +2,7 @@ package org.apache.lucene.index; import java.io.IOException; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java index 5dd9307..53e2767 100644 --- a/lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java +++ b/lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -232,7 +232,6 @@ public abstract class LogMergePolicy extends MergePolicy { assert w != null; boolean hasDeletions = w.numDeletedDocs(info) > 0; return !hasDeletions && - !info.info.hasSeparateNorms() && info.info.dir == w.getDirectory() && (info.info.getUseCompoundFile() == useCompoundFile || noCFSRatio < 1.0); } @@ -421,7 +420,7 @@ public abstract class LogMergePolicy extends MergePolicy { */ @Override public MergeSpecification findForcedDeletesMerges(SegmentInfos segmentInfos) - throws CorruptIndexException, IOException { + throws IOException { final List segments = segmentInfos.asList(); final int numSegments = segments.size(); diff --git a/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java index d6a95c5..be5420c 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java +++ b/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -24,7 +24,6 @@ import java.util.Map; import org.apache.lucene.store.Directory; import org.apache.lucene.store.MergeInfo; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.SetOnce.AlreadySetException; import org.apache.lucene.util.SetOnce; @@ -57,7 +56,7 @@ import org.apache.lucene.util.SetOnce; * @lucene.experimental */ -public abstract class MergePolicy implements java.io.Closeable { +public abstract class MergePolicy implements java.io.Closeable, Cloneable { /** OneMerge provides the information necessary to perform * an individual primitive merge operation, resulting in @@ -74,7 +73,6 @@ public abstract class MergePolicy implements java.io.Closeable { int maxNumSegments = -1; // used by IndexWriter public long estimatedMergeBytes; // used by IndexWriter List readers; // used by IndexWriter - List readerLiveDocs; // used by IndexWriter public final List segments; public final int totalDocCount; boolean aborted; @@ -254,7 +252,20 @@ public abstract class MergePolicy implements java.io.Closeable { } } - protected final SetOnce writer; + protected SetOnce writer; + + @Override + public MergePolicy clone() { + MergePolicy clone; + try { + clone = (MergePolicy) super.clone(); + } catch (CloneNotSupportedException e) { + // should not happen + throw new RuntimeException(e); + } + clone.writer = new SetOnce(); + return clone; + } /** * Creates a new merge policy instance. Note that if you intend to use it @@ -286,7 +297,7 @@ public abstract class MergePolicy implements java.io.Closeable { * the total set of segments in the index */ public abstract MergeSpecification findMerges(SegmentInfos segmentInfos) - throws CorruptIndexException, IOException; + throws IOException; /** * Determine what set of merge operations is necessary in @@ -311,7 +322,7 @@ public abstract class MergePolicy implements java.io.Closeable { */ public abstract MergeSpecification findForcedMerges( SegmentInfos segmentInfos, int maxSegmentCount, Map segmentsToMerge) - throws CorruptIndexException, IOException; + throws IOException; /** * Determine what set of merge operations is necessary in order to expunge all @@ -321,7 +332,7 @@ public abstract class MergePolicy implements java.io.Closeable { * the total set of segments in the index */ public abstract MergeSpecification findForcedDeletesMerges( - SegmentInfos segmentInfos) throws CorruptIndexException, IOException; + SegmentInfos segmentInfos) throws IOException; /** * Release all resources for the policy. diff --git a/lucene/core/src/java/org/apache/lucene/index/MergeScheduler.java b/lucene/core/src/java/org/apache/lucene/index/MergeScheduler.java index 0c57c5f..7be73ff 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MergeScheduler.java +++ b/lucene/core/src/java/org/apache/lucene/index/MergeScheduler.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -29,10 +29,8 @@ import java.io.IOException; public abstract class MergeScheduler { /** Run the merges provided by {@link IndexWriter#getNextMerge()}. */ - public abstract void merge(IndexWriter writer) - throws CorruptIndexException, IOException; + public abstract void merge(IndexWriter writer) throws IOException; /** Close this MergeScheduler. */ - public abstract void close() - throws CorruptIndexException, IOException; + public abstract void close() throws IOException; } diff --git a/lucene/core/src/java/org/apache/lucene/index/MergeState.java b/lucene/core/src/java/org/apache/lucene/index/MergeState.java index 367d231..abad2a8 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MergeState.java +++ b/lucene/core/src/java/org/apache/lucene/index/MergeState.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,31 +19,174 @@ package org.apache.lucene.index; import java.util.List; -import org.apache.lucene.index.PayloadProcessorProvider.ReaderPayloadProcessor; import org.apache.lucene.index.PayloadProcessorProvider.PayloadProcessor; +import org.apache.lucene.index.PayloadProcessorProvider.ReaderPayloadProcessor; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.packed.PackedInts; /** Holds common state used during segment merging * * @lucene.experimental */ public class MergeState { - public static class IndexReaderAndLiveDocs { - public final AtomicReader reader; - public final Bits liveDocs; + public static abstract class DocMap { + private final Bits liveDocs; - public IndexReaderAndLiveDocs(AtomicReader reader, Bits liveDocs) { - this.reader = reader; + protected DocMap(Bits liveDocs) { this.liveDocs = liveDocs; } + + public static DocMap build(AtomicReader reader) { + final int maxDoc = reader.maxDoc(); + final int numDeletes = reader.numDeletedDocs(); + final int numDocs = maxDoc - numDeletes; + assert reader.getLiveDocs() != null || numDeletes == 0; + if (numDeletes == 0) { + return new NoDelDocMap(maxDoc); + } else if (numDeletes < numDocs) { + return buildDelCountDocmap(maxDoc, numDeletes, reader.getLiveDocs(), PackedInts.COMPACT); + } else { + return buildDirectDocMap(maxDoc, numDocs, reader.getLiveDocs(), PackedInts.COMPACT); + } + } + + static DocMap buildDelCountDocmap(int maxDoc, int numDeletes, Bits liveDocs, float acceptableOverheadRatio) { + PackedInts.Mutable numDeletesSoFar = PackedInts.getMutable(maxDoc, + PackedInts.bitsRequired(numDeletes), acceptableOverheadRatio); + int del = 0; + for (int i = 0; i < maxDoc; ++i) { + if (!liveDocs.get(i)) { + ++del; + } + numDeletesSoFar.set(i, del); + } + assert del == numDeletes : "del=" + del + ", numdeletes=" + numDeletes; + return new DelCountDocMap(liveDocs, numDeletesSoFar); + } + + static DocMap buildDirectDocMap(int maxDoc, int numDocs, Bits liveDocs, float acceptableOverheadRatio) { + PackedInts.Mutable docIds = PackedInts.getMutable(maxDoc, + PackedInts.bitsRequired(Math.max(0, numDocs - 1)), acceptableOverheadRatio); + int del = 0; + for (int i = 0; i < maxDoc; ++i) { + if (liveDocs.get(i)) { + docIds.set(i, i - del); + } else { + ++del; + } + } + assert numDocs + del == maxDoc : "maxDoc=" + maxDoc + ", del=" + del + ", numDocs=" + numDocs; + return new DirectDocMap(liveDocs, docIds, del); + } + + public int get(int docId) { + if (liveDocs == null || liveDocs.get(docId)) { + return remap(docId); + } else { + return -1; + } + } + + public abstract int remap(int docId); + + public abstract int maxDoc(); + + public final int numDocs() { + return maxDoc() - numDeletedDocs(); + } + + public abstract int numDeletedDocs(); + + public boolean hasDeletions() { + return numDeletedDocs() > 0; + } + + } + + private static class NoDelDocMap extends DocMap { + + private final int maxDoc; + + private NoDelDocMap(int maxDoc) { + super(null); + this.maxDoc = maxDoc; + } + + @Override + public int remap(int docId) { + return docId; + } + + @Override + public int maxDoc() { + return maxDoc; + } + + @Override + public int numDeletedDocs() { + return 0; + } + } + + private static class DirectDocMap extends DocMap { + + private final PackedInts.Mutable docIds; + private final int numDeletedDocs; + + private DirectDocMap(Bits liveDocs, PackedInts.Mutable docIds, int numDeletedDocs) { + super(liveDocs); + this.docIds = docIds; + this.numDeletedDocs = numDeletedDocs; + } + + @Override + public int remap(int docId) { + return (int) docIds.get(docId); + } + + @Override + public int maxDoc() { + return docIds.size(); + } + + @Override + public int numDeletedDocs() { + return numDeletedDocs; + } + } + + private static class DelCountDocMap extends DocMap { + + private final PackedInts.Mutable numDeletesSoFar; + + private DelCountDocMap(Bits liveDocs, PackedInts.Mutable numDeletesSoFar) { + super(liveDocs); + this.numDeletesSoFar = numDeletesSoFar; + } + + @Override + public int remap(int docId) { + return docId - (int) numDeletesSoFar.get(docId); + } + + @Override + public int maxDoc() { + return numDeletesSoFar.size(); + } + + @Override + public int numDeletedDocs() { + final int maxDoc = maxDoc(); + return (int) numDeletesSoFar.get(maxDoc - 1); + } } public SegmentInfo segmentInfo; public FieldInfos fieldInfos; - public List readers; // Readers & liveDocs being merged - public int[][] docMaps; // Maps docIDs around deletions + public List readers; // Readers being merged + public DocMap[] docMaps; // Maps docIDs around deletions public int[] docBase; // New docID base per reader public CheckAbort checkAbort; public InfoStream infoStream; @@ -65,8 +208,8 @@ public class MergeState { public static class CheckAbort { private double workCount; - private MergePolicy.OneMerge merge; - private Directory dir; + private final MergePolicy.OneMerge merge; + private final Directory dir; public CheckAbort(MergePolicy.OneMerge merge, Directory dir) { this.merge = merge; this.dir = dir; @@ -92,7 +235,7 @@ public class MergeState { * @lucene.internal */ static final MergeState.CheckAbort NONE = new MergeState.CheckAbort(null, null) { @Override - public void work(double units) throws MergePolicy.MergeAbortedException { + public void work(double units) { // do nothing } }; diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiBits.java b/lucene/core/src/java/org/apache/lucene/index/MultiBits.java new file mode 100644 index 0000000..cb45220 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/index/MultiBits.java @@ -0,0 +1,115 @@ +package org.apache.lucene.index; + +import org.apache.lucene.util.Bits; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Concatenates multiple Bits together, on every lookup. + * + *

    NOTE: This is very costly, as every lookup must + * do a binary search to locate the right sub-reader. + * + * @lucene.experimental + */ +final class MultiBits implements Bits { + private final Bits[] subs; + + // length is 1+subs.length (the last entry has the maxDoc): + private final int[] starts; + + private final boolean defaultValue; + + public MultiBits(Bits[] subs, int[] starts, boolean defaultValue) { + assert starts.length == 1+subs.length; + this.subs = subs; + this.starts = starts; + this.defaultValue = defaultValue; + } + + private boolean checkLength(int reader, int doc) { + final int length = starts[1+reader]-starts[reader]; + assert doc - starts[reader] < length: "doc=" + doc + " reader=" + reader + " starts[reader]=" + starts[reader] + " length=" + length; + return true; + } + + public boolean get(int doc) { + final int reader = ReaderUtil.subIndex(doc, starts); + assert reader != -1; + final Bits bits = subs[reader]; + if (bits == null) { + return defaultValue; + } else { + assert checkLength(reader, doc); + return bits.get(doc-starts[reader]); + } + } + + @Override + public String toString() { + StringBuilder b = new StringBuilder(); + b.append(subs.length + " subs: "); + for(int i=0;i 0; - if (liveDocs.size() == 1) { - // Only one actual sub reader -- optimize this case - return liveDocs.get(0); - } else { - return new MultiBits(liveDocs, starts, true); + final Bits[] liveDocs = new Bits[size]; + final int[] starts = new int[size + 1]; + for (int i = 0; i < size; i++) { + // record all liveDocs, even if they are null + final AtomicReaderContext ctx = leaves.get(i); + liveDocs[i] = ctx.reader().getLiveDocs(); + starts[i] = ctx.docBase; } + starts[size] = reader.maxDoc(); + return new MultiBits(liveDocs, starts, true); } else { return null; } @@ -170,7 +153,7 @@ public final class MultiFields extends Fields { return null; } - public MultiFields(Fields[] subs, ReaderUtil.Slice[] subSlices) { + public MultiFields(Fields[] subs, ReaderSlice[] subSlices) { this.subs = subs; this.subSlices = subSlices; } @@ -179,7 +162,7 @@ public final class MultiFields extends Fields { public FieldsEnum iterator() throws IOException { final List fieldsEnums = new ArrayList(); - final List fieldsSlices = new ArrayList(); + final List fieldsSlices = new ArrayList(); for(int i=0;i subs2 = new ArrayList(); - final List slices2 = new ArrayList(); + final List slices2 = new ArrayList(); // Gather all sub-readers that share this field for(int i=0;i subReaders = new ArrayList(); - ReaderUtil.gatherSubReaders(subReaders, reader); final FieldInfos.Builder builder = new FieldInfos.Builder(); - for(AtomicReader subReader : subReaders) { - builder.add(subReader.getFieldInfos()); + for(final AtomicReaderContext ctx : reader.getTopReaderContext().leaves()) { + builder.add(ctx.reader().getFieldInfos()); } return builder.finish(); } public static Collection getIndexedFields(IndexReader reader) { final Collection fields = new HashSet(); - for(FieldInfo fieldInfo : getMergedFieldInfos(reader)) { + for(final FieldInfo fieldInfo : getMergedFieldInfos(reader)) { if (fieldInfo.isIndexed()) { fields.add(fieldInfo.name); } diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiFieldsEnum.java b/lucene/core/src/java/org/apache/lucene/index/MultiFieldsEnum.java index 7db5f02..2e75c47 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MultiFieldsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/index/MultiFieldsEnum.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,7 +18,6 @@ package org.apache.lucene.index; */ import org.apache.lucene.util.PriorityQueue; -import org.apache.lucene.util.ReaderUtil; import java.io.IOException; import java.util.List; @@ -47,7 +46,7 @@ public final class MultiFieldsEnum extends FieldsEnum { /** The subs array must be newly initialized FieldsEnum * (ie, {@link FieldsEnum#next} has not been called. */ - public MultiFieldsEnum(MultiFields fields, FieldsEnum[] subs, ReaderUtil.Slice[] subSlices) throws IOException { + public MultiFieldsEnum(MultiFields fields, FieldsEnum[] subs, ReaderSlice[] subSlices) throws IOException { this.fields = fields; queue = new FieldMergeQueue(subs.length); top = new FieldsEnumWithSlice[subs.length]; @@ -107,11 +106,11 @@ public final class MultiFieldsEnum extends FieldsEnum { public final static class FieldsEnumWithSlice { public static final FieldsEnumWithSlice[] EMPTY_ARRAY = new FieldsEnumWithSlice[0]; final FieldsEnum fields; - final ReaderUtil.Slice slice; + final ReaderSlice slice; final int index; String current; - public FieldsEnumWithSlice(FieldsEnum fields, ReaderUtil.Slice slice, int index) throws IOException { + public FieldsEnumWithSlice(FieldsEnum fields, ReaderSlice slice, int index) { this.slice = slice; this.index = index; assert slice.length >= 0: "length=" + slice.length; diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiReader.java b/lucene/core/src/java/org/apache/lucene/index/MultiReader.java index 39627e8..4610203 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MultiReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/MultiReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -45,7 +45,7 @@ public class MultiReader extends BaseCompositeReader { *

    Note that all subreaders are closed if this Multireader is closed.

    * @param subReaders set of (sub)readers */ - public MultiReader(IndexReader... subReaders) throws IOException { + public MultiReader(IndexReader... subReaders) { this(subReaders, true); } @@ -55,7 +55,7 @@ public class MultiReader extends BaseCompositeReader { * @param closeSubReaders indicates whether the subreaders should be closed * when this MultiReader is closed */ - public MultiReader(IndexReader[] subReaders, boolean closeSubReaders) throws IOException { + public MultiReader(IndexReader[] subReaders, boolean closeSubReaders) { super(subReaders.clone()); this.closeSubReaders = closeSubReaders; if (!closeSubReaders) { @@ -68,12 +68,12 @@ public class MultiReader extends BaseCompositeReader { @Override protected synchronized void doClose() throws IOException { IOException ioe = null; - for (int i = 0; i < subReaders.length; i++) { + for (final IndexReader r : getSequentialSubReaders()) { try { if (closeSubReaders) { - subReaders[i].close(); + r.close(); } else { - subReaders[i].decRef(); + r.decRef(); } } catch (IOException e) { if (ioe == null) ioe = e; diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiTerms.java b/lucene/core/src/java/org/apache/lucene/index/MultiTerms.java index c18ac6c..8b8c3aa 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MultiTerms.java +++ b/lucene/core/src/java/org/apache/lucene/index/MultiTerms.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,7 +23,6 @@ import java.util.Comparator; import java.util.List; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.ReaderUtil; import org.apache.lucene.util.automaton.CompiledAutomaton; @@ -36,10 +35,10 @@ import org.apache.lucene.util.automaton.CompiledAutomaton; public final class MultiTerms extends Terms { private final Terms[] subs; - private final ReaderUtil.Slice[] subSlices; + private final ReaderSlice[] subSlices; private final Comparator termComp; - public MultiTerms(Terms[] subs, ReaderUtil.Slice[] subSlices) throws IOException { + public MultiTerms(Terms[] subs, ReaderSlice[] subSlices) throws IOException { this.subs = subs; this.subSlices = subSlices; @@ -96,7 +95,7 @@ public final class MultiTerms extends Terms { } @Override - public long size() throws IOException { + public long size() { return -1; } diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java b/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java index a84785d..8a8f97d 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,9 +20,6 @@ package org.apache.lucene.index; import org.apache.lucene.util.PriorityQueue; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.BitsSlice; -import org.apache.lucene.util.MultiBits; -import org.apache.lucene.util.ReaderUtil; import java.io.IOException; import java.util.Arrays; @@ -71,7 +68,7 @@ public final class MultiTermsEnum extends TermsEnum { return top; } - public MultiTermsEnum(ReaderUtil.Slice[] slices) { + public MultiTermsEnum(ReaderSlice[] slices) { queue = new TermMergeQueue(slices.length); top = new TermsEnumWithSlice[slices.length]; subs = new TermsEnumWithSlice[slices.length]; @@ -261,12 +258,12 @@ public final class MultiTermsEnum extends TermsEnum { } @Override - public void seekExact(long ord) throws IOException { + public void seekExact(long ord) { throw new UnsupportedOperationException(); } @Override - public long ord() throws IOException { + public long ord() { throw new UnsupportedOperationException(); } @@ -494,12 +491,12 @@ public final class MultiTermsEnum extends TermsEnum { } private final static class TermsEnumWithSlice { - private final ReaderUtil.Slice subSlice; + private final ReaderSlice subSlice; private TermsEnum terms; public BytesRef current; final int index; - public TermsEnumWithSlice(int index, ReaderUtil.Slice subSlice) { + public TermsEnumWithSlice(int index, ReaderSlice subSlice) { this.subSlice = subSlice; this.index = index; assert subSlice.length >= 0: "length=" + subSlice.length; diff --git a/lucene/core/src/java/org/apache/lucene/index/NoDeletionPolicy.java b/lucene/core/src/java/org/apache/lucene/index/NoDeletionPolicy.java index 87b3b2e..9ea0501 100644 --- a/lucene/core/src/java/org/apache/lucene/index/NoDeletionPolicy.java +++ b/lucene/core/src/java/org/apache/lucene/index/NoDeletionPolicy.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,7 +17,6 @@ package org.apache.lucene.index; * limitations under the License. */ -import java.io.IOException; import java.util.List; /** @@ -34,8 +33,8 @@ public final class NoDeletionPolicy implements IndexDeletionPolicy { // keep private to avoid instantiation } - public void onCommit(List commits) throws IOException {} + public void onCommit(List commits) {} - public void onInit(List commits) throws IOException {} + public void onInit(List commits) {} } diff --git a/lucene/core/src/java/org/apache/lucene/index/NoMergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/NoMergePolicy.java index 212d701..9d83e35 100644 --- a/lucene/core/src/java/org/apache/lucene/index/NoMergePolicy.java +++ b/lucene/core/src/java/org/apache/lucene/index/NoMergePolicy.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,7 +17,6 @@ package org.apache.lucene.index; * limitations under the License. */ -import java.io.IOException; import java.util.Map; /** @@ -54,17 +53,14 @@ public final class NoMergePolicy extends MergePolicy { public void close() {} @Override - public MergeSpecification findMerges(SegmentInfos segmentInfos) - throws CorruptIndexException, IOException { return null; } + public MergeSpecification findMerges(SegmentInfos segmentInfos) { return null; } @Override public MergeSpecification findForcedMerges(SegmentInfos segmentInfos, - int maxSegmentCount, Map segmentsToMerge) - throws CorruptIndexException, IOException { return null; } + int maxSegmentCount, Map segmentsToMerge) { return null; } @Override - public MergeSpecification findForcedDeletesMerges(SegmentInfos segmentInfos) - throws CorruptIndexException, IOException { return null; } + public MergeSpecification findForcedDeletesMerges(SegmentInfos segmentInfos) { return null; } @Override public boolean useCompoundFile(SegmentInfos segments, SegmentInfoPerCommit newSegment) { return useCompoundFile; } diff --git a/lucene/core/src/java/org/apache/lucene/index/NoMergeScheduler.java b/lucene/core/src/java/org/apache/lucene/index/NoMergeScheduler.java index 1f6fce7..e3aba03 100644 --- a/lucene/core/src/java/org/apache/lucene/index/NoMergeScheduler.java +++ b/lucene/core/src/java/org/apache/lucene/index/NoMergeScheduler.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,8 +17,6 @@ package org.apache.lucene.index; * limitations under the License. */ -import java.io.IOException; - /** * A {@link MergeScheduler} which never executes any merges. It is also a * singleton and can be accessed through {@link NoMergeScheduler#INSTANCE}. Use @@ -42,5 +40,5 @@ public final class NoMergeScheduler extends MergeScheduler { public void close() {} @Override - public void merge(IndexWriter writer) throws CorruptIndexException, IOException {} + public void merge(IndexWriter writer) {} } diff --git a/lucene/core/src/java/org/apache/lucene/index/Norm.java b/lucene/core/src/java/org/apache/lucene/index/Norm.java index 9c3c3b2..4f338c3 100644 --- a/lucene/core/src/java/org/apache/lucene/index/Norm.java +++ b/lucene/core/src/java/org/apache/lucene/index/Norm.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/NormsConsumer.java b/lucene/core/src/java/org/apache/lucene/index/NormsConsumer.java index e7a68d4..25f6932 100644 --- a/lucene/core/src/java/org/apache/lucene/index/NormsConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/index/NormsConsumer.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -87,10 +87,10 @@ final class NormsConsumer extends InvertedDocEndConsumer { } @Override - void finishDocument() throws IOException {} + void finishDocument() {} @Override - void startDocument() throws IOException {} + void startDocument() {} @Override InvertedDocEndConsumerPerField addField(DocInverterPerField docInverterPerField, diff --git a/lucene/core/src/java/org/apache/lucene/index/NormsConsumerPerField.java b/lucene/core/src/java/org/apache/lucene/index/NormsConsumerPerField.java index e9ed264..6532330 100644 --- a/lucene/core/src/java/org/apache/lucene/index/NormsConsumerPerField.java +++ b/lucene/core/src/java/org/apache/lucene/index/NormsConsumerPerField.java @@ -1,5 +1,5 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/OrdTermState.java b/lucene/core/src/java/org/apache/lucene/index/OrdTermState.java index 20e8a84..eb8a51f 100644 --- a/lucene/core/src/java/org/apache/lucene/index/OrdTermState.java +++ b/lucene/core/src/java/org/apache/lucene/index/OrdTermState.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java b/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java index d546c3b..10c8dd6 100644 --- a/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -162,7 +162,7 @@ public final class ParallelAtomicReader extends AtomicReader { } @Override - public String next() throws IOException { + public String next() { if (keys.hasNext()) { currentField = keys.next(); } else { @@ -172,7 +172,7 @@ public final class ParallelAtomicReader extends AtomicReader { } @Override - public Terms terms() throws IOException { + public Terms terms() { return fields.terms(currentField); } @@ -185,22 +185,22 @@ public final class ParallelAtomicReader extends AtomicReader { ParallelFields() { } - void addField(String fieldName, Terms terms) throws IOException { + void addField(String fieldName, Terms terms) { fields.put(fieldName, terms); } @Override - public FieldsEnum iterator() throws IOException { + public FieldsEnum iterator() { return new ParallelFieldsEnum(this); } @Override - public Terms terms(String field) throws IOException { + public Terms terms(String field) { return fields.get(field); } @Override - public int size() throws IOException { + public int size() { return fields.size(); } } @@ -249,7 +249,7 @@ public final class ParallelAtomicReader extends AtomicReader { } @Override - public void document(int docID, StoredFieldVisitor visitor) throws CorruptIndexException, IOException { + public void document(int docID, StoredFieldVisitor visitor) throws IOException { ensureOpen(); for (final AtomicReader reader: storedFieldsReaders) { reader.document(docID, visitor); diff --git a/lucene/core/src/java/org/apache/lucene/index/ParallelCompositeReader.java b/lucene/core/src/java/org/apache/lucene/index/ParallelCompositeReader.java index d85e4e8..3fcc76e 100644 --- a/lucene/core/src/java/org/apache/lucene/index/ParallelCompositeReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/ParallelCompositeReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.Collections; import java.util.IdentityHashMap; import java.util.Iterator; +import java.util.List; import java.util.Set; /** An {@link CompositeReader} which reads multiple, parallel indexes. Each index added @@ -85,44 +86,45 @@ public final class ParallelCompositeReader extends BaseCompositeReader firstSubReaders = readers[0].getSequentialSubReaders(); // check compatibility: - final int maxDoc = readers[0].maxDoc(); - final int[] childMaxDoc = new int[firstSubReaders.length]; - final boolean[] childAtomic = new boolean[firstSubReaders.length]; - for (int i = 0; i < firstSubReaders.length; i++) { - childMaxDoc[i] = firstSubReaders[i].maxDoc(); - childAtomic[i] = firstSubReaders[i] instanceof AtomicReader; + final int maxDoc = readers[0].maxDoc(), noSubs = firstSubReaders.size(); + final int[] childMaxDoc = new int[noSubs]; + final boolean[] childAtomic = new boolean[noSubs]; + for (int i = 0; i < noSubs; i++) { + final IndexReader r = firstSubReaders.get(i); + childMaxDoc[i] = r.maxDoc(); + childAtomic[i] = r instanceof AtomicReader; } validate(readers, maxDoc, childMaxDoc, childAtomic); validate(storedFieldsReaders, maxDoc, childMaxDoc, childAtomic); // hierarchically build the same subreader structure as the first CompositeReader with Parallel*Readers: - final IndexReader[] subReaders = new IndexReader[firstSubReaders.length]; + final IndexReader[] subReaders = new IndexReader[noSubs]; for (int i = 0; i < subReaders.length; i++) { - if (firstSubReaders[i] instanceof AtomicReader) { + if (firstSubReaders.get(i) instanceof AtomicReader) { final AtomicReader[] atomicSubs = new AtomicReader[readers.length]; for (int j = 0; j < readers.length; j++) { - atomicSubs[j] = (AtomicReader) readers[j].getSequentialSubReaders()[i]; + atomicSubs[j] = (AtomicReader) readers[j].getSequentialSubReaders().get(i); } final AtomicReader[] storedSubs = new AtomicReader[storedFieldsReaders.length]; for (int j = 0; j < storedFieldsReaders.length; j++) { - storedSubs[j] = (AtomicReader) storedFieldsReaders[j].getSequentialSubReaders()[i]; + storedSubs[j] = (AtomicReader) storedFieldsReaders[j].getSequentialSubReaders().get(i); } // we simply enable closing of subReaders, to prevent incRefs on subReaders // -> for synthetic subReaders, close() is never // called by our doClose() subReaders[i] = new ParallelAtomicReader(true, atomicSubs, storedSubs); } else { - assert firstSubReaders[i] instanceof CompositeReader; + assert firstSubReaders.get(i) instanceof CompositeReader; final CompositeReader[] compositeSubs = new CompositeReader[readers.length]; for (int j = 0; j < readers.length; j++) { - compositeSubs[j] = (CompositeReader) readers[j].getSequentialSubReaders()[i]; + compositeSubs[j] = (CompositeReader) readers[j].getSequentialSubReaders().get(i); } final CompositeReader[] storedSubs = new CompositeReader[storedFieldsReaders.length]; for (int j = 0; j < storedFieldsReaders.length; j++) { - storedSubs[j] = (CompositeReader) storedFieldsReaders[j].getSequentialSubReaders()[i]; + storedSubs[j] = (CompositeReader) storedFieldsReaders[j].getSequentialSubReaders().get(i); } // we simply enable closing of subReaders, to prevent incRefs on subReaders // -> for synthetic subReaders, close() is never called by our doClose() @@ -136,18 +138,20 @@ public final class ParallelCompositeReader extends BaseCompositeReader subs = reader.getSequentialSubReaders(); if (reader.maxDoc() != maxDoc) { throw new IllegalArgumentException("All readers must have same maxDoc: "+maxDoc+"!="+reader.maxDoc()); } - if (subs.length != childMaxDoc.length) { + final int noSubs = subs.size(); + if (noSubs != childMaxDoc.length) { throw new IllegalArgumentException("All readers must have same number of subReaders"); } - for (int subIDX = 0; subIDX < subs.length; subIDX++) { - if (subs[subIDX].maxDoc() != childMaxDoc[subIDX]) { + for (int subIDX = 0; subIDX < noSubs; subIDX++) { + final IndexReader r = subs.get(subIDX); + if (r.maxDoc() != childMaxDoc[subIDX]) { throw new IllegalArgumentException("All readers must have same corresponding subReader maxDoc"); } - if (!(childAtomic[subIDX] ? (subs[subIDX] instanceof AtomicReader) : (subs[subIDX] instanceof CompositeReader))) { + if (!(childAtomic[subIDX] ? (r instanceof AtomicReader) : (r instanceof CompositeReader))) { throw new IllegalArgumentException("All readers must have same corresponding subReader types (atomic or composite)"); } } diff --git a/lucene/core/src/java/org/apache/lucene/index/ParallelPostingsArray.java b/lucene/core/src/java/org/apache/lucene/index/ParallelPostingsArray.java index 0af1bc5..75f223f 100644 --- a/lucene/core/src/java/org/apache/lucene/index/ParallelPostingsArray.java +++ b/lucene/core/src/java/org/apache/lucene/index/ParallelPostingsArray.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/Payload.java b/lucene/core/src/java/org/apache/lucene/index/Payload.java deleted file mode 100644 index ec8b8c4..0000000 --- a/lucene/core/src/java/org/apache/lucene/index/Payload.java +++ /dev/null @@ -1,199 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.util.ArrayUtil; - -/** - * A Payload is metadata that can be stored together with each occurrence - * of a term. This metadata is stored inline in the posting list of the - * specific term. - *

    - * To store payloads in the index a {@link TokenStream} has to be used that - * produces payload data. - *

    - * Use {@link DocsAndPositionsEnum#getPayload()} - * to retrieve the payloads from the index.
    - * - */ -public class Payload implements Cloneable { - /** the byte array containing the payload data */ - protected byte[] data; - - /** the offset within the byte array */ - protected int offset; - - /** the length of the payload data */ - protected int length; - - /** Creates an empty payload and does not allocate a byte array. */ - public Payload() { - // nothing to do - } - - /** - * Creates a new payload with the the given array as data. - * A reference to the passed-in array is held, i. e. no - * copy is made. - * - * @param data the data of this payload - */ - public Payload(byte[] data) { - this(data, 0, data.length); - } - - /** - * Creates a new payload with the the given array as data. - * A reference to the passed-in array is held, i. e. no - * copy is made. - * - * @param data the data of this payload - * @param offset the offset in the data byte array - * @param length the length of the data - */ - public Payload(byte[] data, int offset, int length) { - if (offset < 0 || offset + length > data.length) { - throw new IllegalArgumentException(); - } - this.data = data; - this.offset = offset; - this.length = length; - } - - /** - * Sets this payloads data. - * A reference to the passed-in array is held, i. e. no - * copy is made. - */ - public void setData(byte[] data) { - setData(data, 0, data.length); - } - - /** - * Sets this payloads data. - * A reference to the passed-in array is held, i. e. no - * copy is made. - */ - public void setData(byte[] data, int offset, int length) { - this.data = data; - this.offset = offset; - this.length = length; - } - - /** - * Returns a reference to the underlying byte array - * that holds this payloads data. - */ - public byte[] getData() { - return this.data; - } - - /** - * Returns the offset in the underlying byte array - */ - public int getOffset() { - return this.offset; - } - - /** - * Returns the length of the payload data. - */ - public int length() { - return this.length; - } - - /** - * Returns the byte at the given index. - */ - public byte byteAt(int index) { - if (0 <= index && index < this.length) { - return this.data[this.offset + index]; - } - throw new ArrayIndexOutOfBoundsException(index); - } - - /** - * Allocates a new byte array, copies the payload data into it and returns it. - */ - public byte[] toByteArray() { - byte[] retArray = new byte[this.length]; - System.arraycopy(this.data, this.offset, retArray, 0, this.length); - return retArray; - } - - /** - * Copies the payload data to a byte array. - * - * @param target the target byte array - * @param targetOffset the offset in the target byte array - */ - public void copyTo(byte[] target, int targetOffset) { - if (this.length > target.length + targetOffset) { - throw new ArrayIndexOutOfBoundsException(); - } - System.arraycopy(this.data, this.offset, target, targetOffset, this.length); - } - - /** - * Clones this payload by creating a copy of the underlying - * byte array. - */ - @Override - public Payload clone() { - try { - // Start with a shallow copy of data - Payload clone = (Payload) super.clone(); - // Only copy the part of data that belongs to this Payload - if (offset == 0 && length == data.length) { - // It is the whole thing, so just clone it. - clone.data = data.clone(); - } - else { - // Just get the part - clone.data = this.toByteArray(); - clone.offset = 0; - } - return clone; - } catch (CloneNotSupportedException e) { - throw new RuntimeException(e); // shouldn't happen - } - } - - @Override - public boolean equals(Object obj) { - if (obj == this) - return true; - if (obj instanceof Payload) { - Payload other = (Payload) obj; - if (length == other.length) { - for(int i=0;i { + + /** + * Creates and returns a new ReaderManager from the given + * {@link IndexWriter}. + * + * @param writer + * the IndexWriter to open the IndexReader from. + * @param applyAllDeletes + * If true, all buffered deletes will be applied (made + * visible) in the {@link IndexSearcher} / {@link DirectoryReader}. + * If false, the deletes may or may not be applied, but + * remain buffered (in IndexWriter) so that they will be applied in + * the future. Applying deletes can be costly, so if your app can + * tolerate deleted documents being returned you might gain some + * performance by passing false. See + * {@link DirectoryReader#openIfChanged(DirectoryReader, IndexWriter, boolean)}. + * + * @throws IOException + */ + public ReaderManager(IndexWriter writer, boolean applyAllDeletes) throws IOException { + current = DirectoryReader.open(writer, applyAllDeletes); + } + + /** + * Creates and returns a new ReaderManager from the given {@link Directory}. + * @param dir the directory to open the DirectoryReader on. + * + * @throws IOException + */ + public ReaderManager(Directory dir) throws IOException { + current = DirectoryReader.open(dir); + } + + @Override + protected void decRef(DirectoryReader reference) throws IOException { + reference.decRef(); + } + + @Override + protected DirectoryReader refreshIfNeeded(DirectoryReader referenceToRefresh) throws IOException { + return DirectoryReader.openIfChanged(referenceToRefresh); + } + + @Override + protected boolean tryIncRef(DirectoryReader reference) { + return reference.tryIncRef(); + } + +} diff --git a/lucene/core/src/java/org/apache/lucene/index/ReaderSlice.java b/lucene/core/src/java/org/apache/lucene/index/ReaderSlice.java new file mode 100644 index 0000000..62eabc2 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/index/ReaderSlice.java @@ -0,0 +1,41 @@ +package org.apache.lucene.index; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Subreader slice from a parent composite reader. + * + * @lucene.internal + */ +public final class ReaderSlice { + public static final ReaderSlice[] EMPTY_ARRAY = new ReaderSlice[0]; + public final int start; + public final int length; + public final int readerIndex; + + public ReaderSlice(int start, int length, int readerIndex) { + this.start = start; + this.length = length; + this.readerIndex = readerIndex; + } + + @Override + public String toString() { + return "slice start=" + start + " length=" + length + " readerIndex=" + readerIndex; + } +} \ No newline at end of file diff --git a/lucene/core/src/java/org/apache/lucene/index/ReaderUtil.java b/lucene/core/src/java/org/apache/lucene/index/ReaderUtil.java new file mode 100644 index 0000000..ae97143 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/index/ReaderUtil.java @@ -0,0 +1,94 @@ +package org.apache.lucene.index; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.List; + + +/** + * Common util methods for dealing with {@link IndexReader}s and {@link IndexReaderContext}s. + * + * @lucene.internal + */ +public final class ReaderUtil { + + private ReaderUtil() {} // no instance + + /** + * Walks up the reader tree and return the given context's top level reader + * context, or in other words the reader tree's root context. + */ + public static IndexReaderContext getTopLevelContext(IndexReaderContext context) { + while (context.parent != null) { + context = context.parent; + } + return context; + } + + /** + * Returns index of the searcher/reader for document n in the + * array used to construct this searcher/reader. + */ + public static int subIndex(int n, int[] docStarts) { // find + // searcher/reader for doc n: + int size = docStarts.length; + int lo = 0; // search starts array + int hi = size - 1; // for first element less than n, return its index + while (hi >= lo) { + int mid = (lo + hi) >>> 1; + int midValue = docStarts[mid]; + if (n < midValue) + hi = mid - 1; + else if (n > midValue) + lo = mid + 1; + else { // found a match + while (mid + 1 < size && docStarts[mid + 1] == midValue) { + mid++; // scan to last match + } + return mid; + } + } + return hi; + } + + /** + * Returns index of the searcher/reader for document n in the + * array used to construct this searcher/reader. + */ + public static int subIndex(int n, List leaves) { // find + // searcher/reader for doc n: + int size = leaves.size(); + int lo = 0; // search starts array + int hi = size - 1; // for first element less than n, return its index + while (hi >= lo) { + int mid = (lo + hi) >>> 1; + int midValue = leaves.get(mid).docBase; + if (n < midValue) + hi = mid - 1; + else if (n > midValue) + lo = mid + 1; + else { // found a match + while (mid + 1 < size && leaves.get(mid + 1).docBase == midValue) { + mid++; // scan to last match + } + return mid; + } + } + return hi; + } +} diff --git a/lucene/core/src/java/org/apache/lucene/index/ReadersAndLiveDocs.java b/lucene/core/src/java/org/apache/lucene/index/ReadersAndLiveDocs.java index febe85b..e5a2468 100644 --- a/lucene/core/src/java/org/apache/lucene/index/ReadersAndLiveDocs.java +++ b/lucene/core/src/java/org/apache/lucene/index/ReadersAndLiveDocs.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java b/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java index a4eb902..92a33d2 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java +++ b/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java b/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java index ddd157b..e6f2b0e 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java +++ b/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -26,7 +26,6 @@ import java.util.Map; import java.util.Set; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene3x.Lucene3xSegmentInfoFormat; import org.apache.lucene.store.Directory; import org.apache.lucene.store.TrackingDirectoryWrapper; @@ -107,14 +106,6 @@ public final class SegmentInfo { } /** - * @deprecated separate norms are not supported in >= 4.0 - */ - @Deprecated - boolean hasSeparateNorms() { - return getAttribute(Lucene3xSegmentInfoFormat.NORMGEN_KEY) != null; - } - - /** * Mark whether this segment is stored as a compound file. * * @param isCompoundFile true if this is a compound file; @@ -166,7 +157,7 @@ public final class SegmentInfo { * modify it. */ - public Set files() throws IOException { + public Set files() { if (setFiles == null) { throw new IllegalStateException("files were not computed yet"); } diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java b/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java index ed4792a..60c2bcf 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java +++ b/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -132,6 +132,15 @@ public class SegmentInfoPerCommit { } @Override + public String toString() { + String s = info.toString(info.dir, delCount); + if (delGen != -1) { + s += ":delGen=" + delGen; + } + return s; + } + + @Override public SegmentInfoPerCommit clone() { return new SegmentInfoPerCommit(info, delCount, delGen); } diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java index 238b558..b92d31a 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java +++ b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -32,10 +32,8 @@ import java.util.Map; import java.util.Set; import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.LiveDocsFormat; -import org.apache.lucene.codecs.lucene3x.Lucene3xCodec; -import org.apache.lucene.codecs.lucene3x.Lucene3xSegmentInfoFormat; -import org.apache.lucene.codecs.lucene3x.Lucene3xSegmentInfoReader; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.ChecksumIndexOutput; import org.apache.lucene.store.DataOutput; // javadocs @@ -44,9 +42,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.NoSuchDirectoryException; -import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.IOUtils; -import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.ThreadInterruptedException; /** @@ -194,7 +190,7 @@ public final class SegmentInfos implements Cloneable, Iterable(); + mergeState.readers = new ArrayList(); mergeState.checkAbort = checkAbort; mergeState.payloadProcessorProvider = payloadProcessorProvider; directory = dir; @@ -76,21 +75,14 @@ final class SegmentMerger { * @param reader */ final void add(IndexReader reader) { - try { - new ReaderUtil.Gather(reader) { - @Override - protected void add(int base, AtomicReader r) { - mergeState.readers.add(new MergeState.IndexReaderAndLiveDocs(r, r.getLiveDocs())); - } - }.run(); - } catch (IOException ioe) { - // won't happen - throw new RuntimeException(ioe); + for (final AtomicReaderContext ctx : reader.getTopReaderContext().leaves()) { + final AtomicReader r = ctx.reader(); + mergeState.readers.add(r); } } - final void add(SegmentReader reader, Bits liveDocs) { - mergeState.readers.add(new MergeState.IndexReaderAndLiveDocs(reader, liveDocs)); + final void add(SegmentReader reader) { + mergeState.readers.add(reader); } /** @@ -99,7 +91,7 @@ final class SegmentMerger { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - final MergeState merge() throws CorruptIndexException, IOException { + final MergeState merge() throws IOException { // NOTE: it's important to add calls to // checkAbort.work(...) if you make any changes to this // method that will spend alot of time. The frequency @@ -146,14 +138,14 @@ final class SegmentMerger { // FieldInfos, then we can do a bulk copy of the // stored fields: for (int i = 0; i < numReaders; i++) { - MergeState.IndexReaderAndLiveDocs reader = mergeState.readers.get(i); + AtomicReader reader = mergeState.readers.get(i); // TODO: we may be able to broaden this to // non-SegmentReaders, since FieldInfos is now // required? But... this'd also require exposing // bulk-copy (TVs and stored fields) API in foreign // readers.. - if (reader.reader instanceof SegmentReader) { - SegmentReader segmentReader = (SegmentReader) reader.reader; + if (reader instanceof SegmentReader) { + SegmentReader segmentReader = (SegmentReader) reader; boolean same = true; FieldInfos segmentFieldInfos = segmentReader.getFieldInfos(); for (FieldInfo fi : segmentFieldInfos) { @@ -185,13 +177,7 @@ final class SegmentMerger { if (previous == null) { previous = TypePromoter.getIdentityPromoter(); } - TypePromoter promoted = previous.promote(incoming); - if (promoted == null) { - // type is incompatible: promote to BYTES_VAR_STRAIGHT - return TypePromoter.create(DocValues.Type.BYTES_VAR_STRAIGHT, TypePromoter.VAR_TYPE_VALUE_SIZE); - } else { - return promoted; - } + return previous.promote(incoming); } // NOTE: this is actually merging all the fieldinfos @@ -202,8 +188,7 @@ final class SegmentMerger { Map docValuesTypes = new HashMap(); Map normValuesTypes = new HashMap(); - for (MergeState.IndexReaderAndLiveDocs readerAndLiveDocs : mergeState.readers) { - final AtomicReader reader = readerAndLiveDocs.reader; + for (AtomicReader reader : mergeState.readers) { FieldInfos readerFieldInfos = reader.getFieldInfos(); for (FieldInfo fi : readerFieldInfos) { FieldInfo merged = fieldInfosBuilder.add(fi); @@ -258,7 +243,7 @@ final class SegmentMerger { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - private int mergeFields() throws CorruptIndexException, IOException { + private int mergeFields() throws IOException { final StoredFieldsWriter fieldsWriter = codec.storedFieldsFormat().fieldsWriter(directory, mergeState.segmentInfo, context); try { @@ -287,7 +272,7 @@ final class SegmentMerger { final int numReaders = mergeState.readers.size(); // Remap docIDs - mergeState.docMaps = new int[numReaders][]; + mergeState.docMaps = new MergeState.DocMap[numReaders]; mergeState.docBase = new int[numReaders]; mergeState.readerPayloadProcessor = new PayloadProcessorProvider.ReaderPayloadProcessor[numReaders]; mergeState.currentPayloadProcessor = new PayloadProcessorProvider.PayloadProcessor[numReaders]; @@ -297,36 +282,15 @@ final class SegmentMerger { int i = 0; while(i < mergeState.readers.size()) { - final MergeState.IndexReaderAndLiveDocs reader = mergeState.readers.get(i); + final AtomicReader reader = mergeState.readers.get(i); mergeState.docBase[i] = docBase; - final int maxDoc = reader.reader.maxDoc(); - final int docCount; - final Bits liveDocs = reader.liveDocs; - final int[] docMap; - if (liveDocs != null) { - int delCount = 0; - docMap = new int[maxDoc]; - int newDocID = 0; - for(int j=0;j fields = new ArrayList(); - final List slices = new ArrayList(); + final List slices = new ArrayList(); int docBase = 0; for(int readerIndex=0;readerIndex= maxDoc()) { throw new IllegalArgumentException("docID must be >= 0 and < maxDoc=" + maxDoc() + " (got docID=" + docID + ")"); } @@ -139,7 +139,7 @@ public final class SegmentReader extends AtomicReader { } @Override - public Fields fields() throws IOException { + public Fields fields() { ensureOpen(); return core.fields; } diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java b/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java index 9224869..6b161df 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java +++ b/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/SerialMergeScheduler.java b/lucene/core/src/java/org/apache/lucene/index/SerialMergeScheduler.java index ece5dce..2035441 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SerialMergeScheduler.java +++ b/lucene/core/src/java/org/apache/lucene/index/SerialMergeScheduler.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -27,8 +27,7 @@ public class SerialMergeScheduler extends MergeScheduler { * "synchronized" so that even if the application is using * multiple threads, only one merge may run at a time. */ @Override - synchronized public void merge(IndexWriter writer) - throws CorruptIndexException, IOException { + synchronized public void merge(IndexWriter writer) throws IOException { while(true) { MergePolicy.OneMerge merge = writer.getNextMerge(); diff --git a/lucene/core/src/java/org/apache/lucene/index/SingleTermsEnum.java b/lucene/core/src/java/org/apache/lucene/index/SingleTermsEnum.java index 1c09c44..e33c10d 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SingleTermsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/index/SingleTermsEnum.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,8 +17,6 @@ package org.apache.lucene.index; * limitations under the License. */ -import java.io.IOException; - import org.apache.lucene.search.MultiTermQuery; // javadocs import org.apache.lucene.util.BytesRef; @@ -39,7 +37,7 @@ public final class SingleTermsEnum extends FilteredTermsEnum { * After calling the constructor the enumeration is already pointing to the term, * if it exists. */ - public SingleTermsEnum(TermsEnum tenum, BytesRef termText) throws IOException { + public SingleTermsEnum(TermsEnum tenum, BytesRef termText) { super(tenum); singleRef = termText; setInitialSeekTerm(termText); diff --git a/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java b/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java index bb6dacc..a616661 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java +++ b/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,7 +22,6 @@ import java.util.HashMap; import java.util.Map; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.ReaderUtil; // javadoc import org.apache.lucene.index.DirectoryReader; // javadoc import org.apache.lucene.index.MultiReader; // javadoc @@ -32,17 +31,16 @@ import org.apache.lucene.index.MultiReader; // javadoc * MultiReader} or {@link DirectoryReader}) to emulate an * atomic reader. This requires implementing the postings * APIs on-the-fly, using the static methods in {@link - * MultiFields}, {@link MultiDocValues}, - * by stepping through the sub-readers to merge fields/terms, - * appending docs, etc. + * MultiFields}, {@link MultiDocValues}, by stepping through + * the sub-readers to merge fields/terms, appending docs, etc. * *

    NOTE: this class almost always results in a * performance hit. If this is important to your use case, - * it's better to get the sequential sub readers (see {@link - * ReaderUtil#gatherSubReaders}, instead, and iterate through them - * yourself.

    + * you'll get better performance by gathering the sub readers using + * {@link IndexReader#getTopReaderContext()} to get the + * atomic leaves and then operate per-AtomicReader, + * instead of using this class. */ - public final class SlowCompositeReaderWrapper extends AtomicReader { private final CompositeReader in; @@ -77,7 +75,7 @@ public final class SlowCompositeReaderWrapper extends AtomicReader { } @Override - public Fields fields() throws IOException { + public Fields fields() { ensureOpen(); return fields; } @@ -119,7 +117,7 @@ public final class SlowCompositeReaderWrapper extends AtomicReader { } @Override - public void document(int docID, StoredFieldVisitor visitor) throws CorruptIndexException, IOException { + public void document(int docID, StoredFieldVisitor visitor) throws IOException { ensureOpen(); in.document(docID, visitor); } diff --git a/lucene/core/src/java/org/apache/lucene/index/SnapshotDeletionPolicy.java b/lucene/core/src/java/org/apache/lucene/index/SnapshotDeletionPolicy.java index 1bf50cc..06eaf57 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SnapshotDeletionPolicy.java +++ b/lucene/core/src/java/org/apache/lucene/index/SnapshotDeletionPolicy.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/SortedBytesMergeUtils.java b/lucene/core/src/java/org/apache/lucene/index/SortedBytesMergeUtils.java index 188a6d1..023af13 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SortedBytesMergeUtils.java +++ b/lucene/core/src/java/org/apache/lucene/index/SortedBytesMergeUtils.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -81,7 +81,8 @@ public final class SortedBytesMergeUtils { } } - public static List buildSlices(int[] docBases, int[][] docMaps, + public static List buildSlices( + int[] docBases, MergeState.DocMap[] docMaps, DocValues[] docValues, MergeContext ctx) throws IOException { final List slices = new ArrayList(); for (int i = 0; i < docValues.length; i++) { @@ -111,15 +112,15 @@ public final class SortedBytesMergeUtils { * mapping in docIDToRelativeOrd. After the merge SortedSourceSlice#ordMapping * contains the new global ordinals for the relative index. */ - private static void createOrdMapping(int[] docBases, int[][] docMaps, + private static void createOrdMapping(int[] docBases, MergeState.DocMap[] docMaps, SortedSourceSlice currentSlice) { final int readerIdx = currentSlice.readerIdx; - final int[] currentDocMap = docMaps[readerIdx]; + final MergeState.DocMap currentDocMap = docMaps[readerIdx]; final int docBase = currentSlice.docToOrdStart; assert docBase == docBases[readerIdx]; - if (currentDocMap != null) { // we have deletes - for (int i = 0; i < currentDocMap.length; i++) { - final int doc = currentDocMap[i]; + if (currentDocMap != null && currentDocMap.hasDeletions()) { // we have deletes + for (int i = 0; i < currentDocMap.maxDoc(); i++) { + final int doc = currentDocMap.get(i); if (doc != -1) { // not deleted final int ord = currentSlice.source.ord(i); // collect ords strictly // increasing @@ -217,7 +218,7 @@ public final class SortedBytesMergeUtils { current = top[0].current; } - private void pushTop() throws IOException { + private void pushTop() { // call next() on each top, and put back into queue for (int i = 0; i < numTop; i++) { top[i].current = top[i].next(); diff --git a/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java b/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java index 23511e2..fc02117 100644 --- a/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -38,7 +38,7 @@ final class StandardDirectoryReader extends DirectoryReader { /** called only from static open() methods */ StandardDirectoryReader(Directory directory, AtomicReader[] readers, IndexWriter writer, - SegmentInfos sis, int termInfosIndexDivisor, boolean applyAllDeletes) throws IOException { + SegmentInfos sis, int termInfosIndexDivisor, boolean applyAllDeletes) { super(directory, readers); this.writer = writer; this.segmentInfos = sis; @@ -48,10 +48,10 @@ final class StandardDirectoryReader extends DirectoryReader { /** called from DirectoryReader.open(...) methods */ static DirectoryReader open(final Directory directory, final IndexCommit commit, - final int termInfosIndexDivisor) throws CorruptIndexException, IOException { + final int termInfosIndexDivisor) throws IOException { return (DirectoryReader) new SegmentInfos.FindSegmentsFile(directory) { @Override - protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException { + protected Object doBody(String segmentFileName) throws IOException { SegmentInfos sis = new SegmentInfos(); sis.read(directory, segmentFileName); final SegmentReader[] readers = new SegmentReader[sis.size()]; @@ -118,8 +118,8 @@ final class StandardDirectoryReader extends DirectoryReader { writer, segmentInfos, writer.getConfig().getReaderTermsIndexDivisor(), applyAllDeletes); } - /** This constructor is only used for {@link #doOpenIfChanged()} */ - private static DirectoryReader open(Directory directory, IndexWriter writer, SegmentInfos infos, AtomicReader[] oldReaders, + /** This constructor is only used for {@link #doOpenIfChanged(SegmentInfos, IndexWriter)} */ + private static DirectoryReader open(Directory directory, IndexWriter writer, SegmentInfos infos, List oldReaders, int termInfosIndexDivisor) throws IOException { // we put the old SegmentReaders in a map, that allows us // to lookup a reader using its segment name @@ -127,8 +127,9 @@ final class StandardDirectoryReader extends DirectoryReader { if (oldReaders != null) { // create a Map SegmentName->SegmentReader - for (int i = 0; i < oldReaders.length; i++) { - segmentReaders.put(((SegmentReader) oldReaders[i]).getSegmentName(), Integer.valueOf(i)); + for (int i = 0, c = oldReaders.size(); i < c; i++) { + final SegmentReader sr = (SegmentReader) oldReaders.get(i); + segmentReaders.put(sr.getSegmentName(), Integer.valueOf(i)); } } @@ -146,7 +147,7 @@ final class StandardDirectoryReader extends DirectoryReader { newReaders[i] = null; } else { // there is an old reader for this segment - we'll try to reopen it - newReaders[i] = (SegmentReader) oldReaders[oldReaderIndex.intValue()]; + newReaders[i] = (SegmentReader) oldReaders.get(oldReaderIndex.intValue()); } boolean success = false; @@ -216,21 +217,21 @@ final class StandardDirectoryReader extends DirectoryReader { if (writer != null) { buffer.append(":nrt"); } - for(int i=0;i + * Note: the given context must be a top-level context. + */ + public static TermContext build(IndexReaderContext context, Term term, boolean cache) + throws IOException { + assert context != null && context.isTopLevel; + final String field = term.field(); + final BytesRef bytes = term.bytes(); + final TermContext perReaderTermState = new TermContext(context); + //if (DEBUG) System.out.println("prts.build term=" + term); + for (final AtomicReaderContext ctx : context.leaves()) { + //if (DEBUG) System.out.println(" r=" + leaves[i].reader); + final Fields fields = ctx.reader().fields(); + if (fields != null) { + final Terms terms = fields.terms(field); + if (terms != null) { + final TermsEnum termsEnum = terms.iterator(null); + if (termsEnum.seekExact(bytes, cache)) { + final TermState termState = termsEnum.termState(); + //if (DEBUG) System.out.println(" found"); + perReaderTermState.register(termState, ctx.ord, termsEnum.docFreq(), termsEnum.totalTermFreq()); + } + } + } + } + return perReaderTermState; + } + + /** + * Clears the {@link TermContext} internal state and removes all + * registered {@link TermState}s + */ + public void clear() { + docFreq = 0; + Arrays.fill(states, null); + } + + /** + * Registers and associates a {@link TermState} with an leaf ordinal. The leaf ordinal + * should be derived from a {@link IndexReaderContext}'s leaf ord. + */ + public void register(TermState state, final int ord, final int docFreq, final long totalTermFreq) { + assert state != null : "state must not be null"; + assert ord >= 0 && ord < states.length; + assert states[ord] == null : "state for ord: " + ord + + " already registered"; + this.docFreq += docFreq; + if (this.totalTermFreq >= 0 && totalTermFreq >= 0) + this.totalTermFreq += totalTermFreq; + else + this.totalTermFreq = -1; + states[ord] = state; + } + + /** + * Returns the {@link TermState} for an leaf ordinal or null if no + * {@link TermState} for the ordinal was registered. + * + * @param ord + * the readers leaf ordinal to get the {@link TermState} for. + * @return the {@link TermState} for the given readers ord or null if no + * {@link TermState} for the reader was registered + */ + public TermState get(int ord) { + assert ord >= 0 && ord < states.length; + return states[ord]; + } + + /** + * Returns the accumulated document frequency of all {@link TermState} + * instances passed to {@link #register(TermState, int, int, long)}. + * @return the accumulated document frequency of all {@link TermState} + * instances passed to {@link #register(TermState, int, int, long)}. + */ + public int docFreq() { + return docFreq; + } + + /** + * Returns the accumulated term frequency of all {@link TermState} + * instances passed to {@link #register(TermState, int, int, long)}. + * @return the accumulated term frequency of all {@link TermState} + * instances passed to {@link #register(TermState, int, int, long)}. + */ + public long totalTermFreq() { + return totalTermFreq; + } + + /** expert: only available for queries that want to lie about docfreq + * @lucene.internal */ + public void setDocFreq(int docFreq) { + this.docFreq = docFreq; + } +} \ No newline at end of file diff --git a/lucene/core/src/java/org/apache/lucene/index/TermState.java b/lucene/core/src/java/org/apache/lucene/index/TermState.java index 9355d67..e52b889 100644 --- a/lucene/core/src/java/org/apache/lucene/index/TermState.java +++ b/lucene/core/src/java/org/apache/lucene/index/TermState.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java b/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java index 1e65c12..40756e1 100644 --- a/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -158,7 +158,7 @@ final class TermVectorsConsumer extends TermsHashConsumer { } @Override - void startDocument() throws IOException { + void startDocument() { assert clearLastVectorFieldName(); reset(); } diff --git a/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java b/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java index b212848..e133840 100644 --- a/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java +++ b/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -90,7 +90,7 @@ final class TermVectorsConsumerPerField extends TermsHashConsumerPerField { * are enabled, to write the vectors to * RAMOutputStream, which is then quickly flushed to * the real term vectors files in the Directory. */ @Override - void finish() throws IOException { + void finish() { if (!doVectors || termsHashPerField.bytesHash.size() == 0) { return; } diff --git a/lucene/core/src/java/org/apache/lucene/index/Terms.java b/lucene/core/src/java/org/apache/lucene/index/Terms.java index 435837a..65b2894 100644 --- a/lucene/core/src/java/org/apache/lucene/index/Terms.java +++ b/lucene/core/src/java/org/apache/lucene/index/Terms.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java b/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java index 8b44e98..7d0a580 100644 --- a/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -58,7 +58,7 @@ public abstract class TermsEnum implements BytesRefIterator { * was hit. */ public static enum SeekStatus {END, FOUND, NOT_FOUND}; - /** Attemps to seek to the exact term, returning + /** Attempts to seek to the exact term, returning * true if the term is found. If this returns false, the * enum is unpositioned. For some codecs, seekExact may * be substantially faster than {@link #seekCeil}. */ @@ -240,12 +240,12 @@ public abstract class TermsEnum implements BytesRefIterator { } @Override - public TermState termState() throws IOException { + public TermState termState() { throw new IllegalStateException("this method should never be called"); } @Override - public void seekExact(BytesRef term, TermState state) throws IOException { + public void seekExact(BytesRef term, TermState state) { throw new IllegalStateException("this method should never be called"); } }; diff --git a/lucene/core/src/java/org/apache/lucene/index/TermsHash.java b/lucene/core/src/java/org/apache/lucene/index/TermsHash.java index 0a06aff..090540d 100644 --- a/lucene/core/src/java/org/apache/lucene/index/TermsHash.java +++ b/lucene/core/src/java/org/apache/lucene/index/TermsHash.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/TermsHashConsumer.java b/lucene/core/src/java/org/apache/lucene/index/TermsHashConsumer.java index 1e5a55d..af4a94b 100644 --- a/lucene/core/src/java/org/apache/lucene/index/TermsHashConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/index/TermsHashConsumer.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/TermsHashConsumerPerField.java b/lucene/core/src/java/org/apache/lucene/index/TermsHashConsumerPerField.java index 9b5649b..3ba1685 100644 --- a/lucene/core/src/java/org/apache/lucene/index/TermsHashConsumerPerField.java +++ b/lucene/core/src/java/org/apache/lucene/index/TermsHashConsumerPerField.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java b/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java index 2de4f05..a3b56bf 100644 --- a/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java +++ b/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java b/lucene/core/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java index cabd50d..82a0477 100644 --- a/lucene/core/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java +++ b/lucene/core/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java @@ -1,5 +1,5 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -74,4 +74,11 @@ class ThreadAffinityDocumentsWriterThreadPool extends DocumentsWriterPerThreadPo minThreadState.lock(); return minThreadState; } + + @Override + public ThreadAffinityDocumentsWriterThreadPool clone() { + ThreadAffinityDocumentsWriterThreadPool clone = (ThreadAffinityDocumentsWriterThreadPool) super.clone(); + clone.threadBindings = new ConcurrentHashMap(); + return clone; + } } diff --git a/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java index 72c6873..f181160 100644 --- a/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java +++ b/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -38,7 +38,7 @@ import java.util.ArrayList; * policy also does not over-merge (i.e. cascade merges). * *

    For normal merging, this policy first computes a - * "budget" of how many segments are allowed by be in the + * "budget" of how many segments are allowed to be in the * index. If the index is over-budget, then the policy * sorts segments by decreasing size (pro-rating by percent * deletes), and then finds the least-cost merge. Merge @@ -256,8 +256,6 @@ public class TieredMergePolicy extends MergePolicy { } } - private final Comparator segmentByteSizeDescending = new SegmentByteSizeDescending(); - /** Holds score and explanation for a single candidate * merge. */ protected static abstract class MergeScore { @@ -277,7 +275,7 @@ public class TieredMergePolicy extends MergePolicy { final Collection toBeMerged = new HashSet(); final List infosSorted = new ArrayList(infos.asList()); - Collections.sort(infosSorted, segmentByteSizeDescending); + Collections.sort(infosSorted, new SegmentByteSizeDescending()); // Compute total index bytes & print details about the index long totIndexBytes = 0; @@ -516,7 +514,7 @@ public class TieredMergePolicy extends MergePolicy { return null; } - Collections.sort(eligible, segmentByteSizeDescending); + Collections.sort(eligible, new SegmentByteSizeDescending()); if (verbose()) { message("eligible=" + eligible); @@ -555,8 +553,7 @@ public class TieredMergePolicy extends MergePolicy { } @Override - public MergeSpecification findForcedDeletesMerges(SegmentInfos infos) - throws CorruptIndexException, IOException { + public MergeSpecification findForcedDeletesMerges(SegmentInfos infos) throws IOException { if (verbose()) { message("findForcedDeletesMerges infos=" + writer.get().segString(infos) + " forceMergeDeletesPctAllowed=" + forceMergeDeletesPctAllowed); } @@ -573,7 +570,7 @@ public class TieredMergePolicy extends MergePolicy { return null; } - Collections.sort(eligible, segmentByteSizeDescending); + Collections.sort(eligible, new SegmentByteSizeDescending()); if (verbose()) { message("eligible=" + eligible); @@ -625,13 +622,11 @@ public class TieredMergePolicy extends MergePolicy { public void close() { } - private boolean isMerged(SegmentInfoPerCommit info) - throws IOException { + private boolean isMerged(SegmentInfoPerCommit info) { IndexWriter w = writer.get(); assert w != null; boolean hasDeletions = w.numDeletedDocs(info) > 0; return !hasDeletions && - !info.info.hasSeparateNorms() && info.info.dir == w.getDirectory() && (info.info.getUseCompoundFile() == useCompoundFile || noCFSRatio < 1.0); } diff --git a/lucene/core/src/java/org/apache/lucene/index/TwoPhaseCommit.java b/lucene/core/src/java/org/apache/lucene/index/TwoPhaseCommit.java new file mode 100644 index 0000000..165f093 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/index/TwoPhaseCommit.java @@ -0,0 +1,77 @@ +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.Map; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * An interface for implementations that support 2-phase commit. You can use + * {@link TwoPhaseCommitTool} to execute a 2-phase commit algorithm over several + * {@link TwoPhaseCommit}s. + * + * @lucene.experimental + */ +public interface TwoPhaseCommit { + + /** + * The first stage of a 2-phase commit. Implementations should do as much work + * as possible in this method, but avoid actual committing changes. If the + * 2-phase commit fails, {@link #rollback()} is called to discard all changes + * since last successful commit. + */ + public void prepareCommit() throws IOException; + + /** + * Like {@link #commit()}, but takes an additional commit data to be included + * w/ the commit. + *

    + * NOTE: some implementations may not support any custom data to be + * included w/ the commit and may discard it altogether. Consult the actual + * implementation documentation for verifying if this is supported. + * + * @see #prepareCommit() + */ + public void prepareCommit(Map commitData) throws IOException; + + /** + * The second phase of a 2-phase commit. Implementations should ideally do + * very little work in this method (following {@link #prepareCommit()}, and + * after it returns, the caller can assume that the changes were successfully + * committed to the underlying storage. + */ + public void commit() throws IOException; + + /** + * Like {@link #commit()}, but takes an additional commit data to be included + * w/ the commit. + * + * @see #commit() + * @see #prepareCommit(Map) + */ + public void commit(Map commitData) throws IOException; + + /** + * Discards any changes that have occurred since the last commit. In a 2-phase + * commit algorithm, where one of the objects failed to {@link #commit()} or + * {@link #prepareCommit()}, this method is used to roll all other objects + * back to their previous state. + */ + public void rollback() throws IOException; + +} diff --git a/lucene/core/src/java/org/apache/lucene/index/TwoPhaseCommitTool.java b/lucene/core/src/java/org/apache/lucene/index/TwoPhaseCommitTool.java new file mode 100644 index 0000000..10f7817 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/index/TwoPhaseCommitTool.java @@ -0,0 +1,162 @@ +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.Map; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A utility for executing 2-phase commit on several objects. + * + * @see TwoPhaseCommit + * @lucene.experimental + */ +public final class TwoPhaseCommitTool { + + /** + * A wrapper of a {@link TwoPhaseCommit}, which delegates all calls to the + * wrapped object, passing the specified commitData. This object is useful for + * use with {@link TwoPhaseCommitTool#execute(TwoPhaseCommit...)} if one would + * like to store commitData as part of the commit. + */ + public static final class TwoPhaseCommitWrapper implements TwoPhaseCommit { + + private final TwoPhaseCommit tpc; + private final Map commitData; + + public TwoPhaseCommitWrapper(TwoPhaseCommit tpc, Map commitData) { + this.tpc = tpc; + this.commitData = commitData; + } + + public void prepareCommit() throws IOException { + prepareCommit(commitData); + } + + public void prepareCommit(Map commitData) throws IOException { + tpc.prepareCommit(this.commitData); + } + + public void commit() throws IOException { + commit(commitData); + } + + public void commit(Map commitData) throws IOException { + tpc.commit(this.commitData); + } + + public void rollback() throws IOException { + tpc.rollback(); + } + } + + /** + * Thrown by {@link TwoPhaseCommitTool#execute(TwoPhaseCommit...)} when an + * object fails to prepareCommit(). + */ + public static class PrepareCommitFailException extends IOException { + + public PrepareCommitFailException(Throwable cause, TwoPhaseCommit obj) { + super("prepareCommit() failed on " + obj); + initCause(cause); + } + + } + + /** + * Thrown by {@link TwoPhaseCommitTool#execute(TwoPhaseCommit...)} when an + * object fails to commit(). + */ + public static class CommitFailException extends IOException { + + public CommitFailException(Throwable cause, TwoPhaseCommit obj) { + super("commit() failed on " + obj); + initCause(cause); + } + + } + + /** rollback all objects, discarding any exceptions that occur. */ + private static void rollback(TwoPhaseCommit... objects) { + for (TwoPhaseCommit tpc : objects) { + // ignore any exception that occurs during rollback - we want to ensure + // all objects are rolled-back. + if (tpc != null) { + try { + tpc.rollback(); + } catch (Throwable t) {} + } + } + } + + /** + * Executes a 2-phase commit algorithm by first + * {@link TwoPhaseCommit#prepareCommit()} all objects and only if all succeed, + * it proceeds with {@link TwoPhaseCommit#commit()}. If any of the objects + * fail on either the preparation or actual commit, it terminates and + * {@link TwoPhaseCommit#rollback()} all of them. + *

    + * NOTE: it may happen that an object fails to commit, after few have + * already successfully committed. This tool will still issue a rollback + * instruction on them as well, but depending on the implementation, it may + * not have any effect. + *

    + * NOTE: if any of the objects are {@code null}, this method simply + * skips over them. + * + * @throws PrepareCommitFailException + * if any of the objects fail to + * {@link TwoPhaseCommit#prepareCommit()} + * @throws CommitFailException + * if any of the objects fail to {@link TwoPhaseCommit#commit()} + */ + public static void execute(TwoPhaseCommit... objects) + throws PrepareCommitFailException, CommitFailException { + TwoPhaseCommit tpc = null; + try { + // first, all should successfully prepareCommit() + for (int i = 0; i < objects.length; i++) { + tpc = objects[i]; + if (tpc != null) { + tpc.prepareCommit(); + } + } + } catch (Throwable t) { + // first object that fails results in rollback all of them and + // throwing an exception. + rollback(objects); + throw new PrepareCommitFailException(t, tpc); + } + + // If all successfully prepareCommit(), attempt the actual commit() + try { + for (int i = 0; i < objects.length; i++) { + tpc = objects[i]; + if (tpc != null) { + tpc.commit(); + } + } + } catch (Throwable t) { + // first object that fails results in rollback all of them and + // throwing an exception. + rollback(objects); + throw new CommitFailException(t, tpc); + } + } + +} diff --git a/lucene/core/src/java/org/apache/lucene/index/TypePromoter.java b/lucene/core/src/java/org/apache/lucene/index/TypePromoter.java index db94b0a..5d257cc 100644 --- a/lucene/core/src/java/org/apache/lucene/index/TypePromoter.java +++ b/lucene/core/src/java/org/apache/lucene/index/TypePromoter.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,26 +20,27 @@ import java.util.HashMap; import java.util.Map; import org.apache.lucene.index.DocValues.Type; +import org.apache.lucene.util.BytesRef; // TODO: maybe we should not automagically promote // types... and instead require a given field always has the // same type? /** - * Type promoter that promotes {@link DocValues} during merge based on - * their {@link Type} and {@link #getValueSize()} + * Type promoter that promotes {@link DocValues} during merge based on their + * {@link Type} and {@link #getValueSize()} * * @lucene.internal */ class TypePromoter { - - private final static Map FLAGS_MAP = new HashMap(); + + private final static Map FLAGS_MAP = new HashMap(); private static final TypePromoter IDENTITY_PROMOTER = new IdentityTypePromoter(); public static final int VAR_TYPE_VALUE_SIZE = -1; - - private static final int IS_INT = 1 << 0; + + private static final int IS_INT = 1 << 0 | 1 << 2; private static final int IS_BYTE = 1 << 1; - private static final int IS_FLOAT = 1 << 2; + private static final int IS_FLOAT = 1 << 2 ; /* VAR & FIXED == VAR */ private static final int IS_VAR = 1 << 3; private static final int IS_FIXED = 1 << 3 | 1 << 4; @@ -50,15 +51,17 @@ class TypePromoter { private static final int IS_DEREF = 1 << 5 | 1 << 6; private static final int IS_SORTED = 1 << 7; /* more bits wins (int16 & int32 == int32) */ - private static final int IS_8_BIT = 1 << 8 | 1 << 9 | 1 << 10 | 1 << 11; - private static final int IS_16_BIT = 1 << 9 | 1 << 10 | 1 << 11; - private static final int IS_32_BIT = 1 << 10 | 1 << 11; + private static final int IS_8_BIT = 1 << 8 | 1 << 9 | 1 << 10 | 1 << 11 | 1 << 12 | 1 << 13; // 8 + private static final int IS_16_BIT = 1 << 9 | 1 << 10 | 1 << 11 | 1 << 12 | 1 << 13; // 9 + private static final int IS_32_BIT = 1 << 10 | 1 << 11 | 1 << 13; private static final int IS_64_BIT = 1 << 11; - - private final Type type; - private final int flags; - private final int valueSize; - + private static final int IS_32_BIT_FLOAT = 1 << 12 | 1 << 13; + private static final int IS_64_BIT_FLOAT = 1 << 13; + + private Type type; + private int flags; + private int valueSize; + /** * Returns a positive value size if this {@link TypePromoter} represents a * fixed variant, otherwise -1 @@ -69,19 +72,26 @@ class TypePromoter { public int getValueSize() { return valueSize; } - + static { for (Type type : Type.values()) { TypePromoter create = create(type, VAR_TYPE_VALUE_SIZE); FLAGS_MAP.put(create.flags, type); } } - + + /** + * Creates a new {@link TypePromoter} + * + */ + protected TypePromoter() {} + /** * Creates a new {@link TypePromoter} * * @param type * the {@link Type} this promoter represents + * * @param flags * the promoters flags * @param valueSize @@ -92,7 +102,25 @@ class TypePromoter { this.flags = flags; this.valueSize = valueSize; } - + + /** + * Resets the {@link TypePromoter} + * + * @param type + * the {@link Type} this promoter represents + * + * @param flags + * the promoters flags + * @param valueSize + * the value size if {@link #IS_FIXED} or -1 otherwise. + */ + protected TypePromoter set(Type type, int flags, int valueSize) { + this.type = type; + this.flags = flags; + this.valueSize = valueSize; + return this; + } + /** * Creates a new promoted {@link TypePromoter} based on this and the given * {@link TypePromoter} or null iff the {@link TypePromoter} @@ -105,23 +133,29 @@ class TypePromoter { * {@link TypePromoter} aren't compatible. */ public TypePromoter promote(TypePromoter promoter) { - + return promote(promoter, newPromoter()); + } + + private TypePromoter promote(TypePromoter promoter, TypePromoter spare) { int promotedFlags = promoter.flags & this.flags; - TypePromoter promoted = create(FLAGS_MAP.get(promotedFlags), valueSize); + TypePromoter promoted = reset(FLAGS_MAP.get(promotedFlags), valueSize, + spare); if (promoted == null) { - return promoted; + return TypePromoter.create(DocValues.Type.BYTES_VAR_STRAIGHT, + TypePromoter.VAR_TYPE_VALUE_SIZE); } - if ((promoted.flags & IS_BYTE) != 0 && (promoted.flags & IS_FIXED) == IS_FIXED) { + if ((promoted.flags & IS_BYTE) != 0 + && (promoted.flags & IS_FIXED) == IS_FIXED) { if (this.valueSize == promoter.valueSize) { return promoted; } - return create(FLAGS_MAP.get(promoted.flags & PROMOTE_TO_VAR_SIZE_MASK), - VAR_TYPE_VALUE_SIZE); + return reset(FLAGS_MAP.get(promoted.flags & PROMOTE_TO_VAR_SIZE_MASK), + VAR_TYPE_VALUE_SIZE, spare); } - + return promoted; } - + /** * Returns the {@link Type} of this {@link TypePromoter} * @@ -130,12 +164,51 @@ class TypePromoter { public Type type() { return type; } - + + private boolean isTypeCompatible(TypePromoter promoter) { + int promotedFlags = promoter.flags & this.flags; + return (promotedFlags & 0x7) > 0; + } + + private boolean isBytesCompatible(TypePromoter promoter) { + int promotedFlags = promoter.flags & this.flags; + return (promotedFlags & IS_BYTE) > 0 + && (promotedFlags & (IS_FIXED | IS_VAR)) > 0; + } + + private boolean isNumericSizeCompatible(TypePromoter promoter) { + int promotedFlags = promoter.flags & this.flags; + return (promotedFlags & IS_BYTE) == 0 + && (((promotedFlags & IS_FIXED) > 0 && (promotedFlags & (IS_8_BIT)) > 0) || (promotedFlags & IS_VAR) > 0); + } + @Override public String toString() { return "TypePromoter [type=" + type + ", sizeInBytes=" + valueSize + "]"; } - + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + flags; + result = prime * result + ((type == null) ? 0 : type.hashCode()); + result = prime * result + valueSize; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + TypePromoter other = (TypePromoter) obj; + if (flags != other.flags) return false; + if (type != other.type) return false; + if (valueSize != other.valueSize) return false; + return true; + } + /** * Creates a new {@link TypePromoter} for the given type and size per value. * @@ -147,47 +220,84 @@ class TypePromoter { * @return a new {@link TypePromoter} */ public static TypePromoter create(Type type, int valueSize) { + return reset(type, valueSize, new TypePromoter()); + } + + private static TypePromoter reset(Type type, int valueSize, + TypePromoter promoter) { if (type == null) { return null; } switch (type) { - case BYTES_FIXED_DEREF: - return new TypePromoter(type, IS_BYTE | IS_FIXED | IS_DEREF, valueSize); - case BYTES_FIXED_SORTED: - return new TypePromoter(type, IS_BYTE | IS_FIXED | IS_SORTED, valueSize); - case BYTES_FIXED_STRAIGHT: - return new TypePromoter(type, IS_BYTE | IS_FIXED | IS_STRAIGHT, valueSize); - case BYTES_VAR_DEREF: - return new TypePromoter(type, IS_BYTE | IS_VAR | IS_DEREF, VAR_TYPE_VALUE_SIZE); - case BYTES_VAR_SORTED: - return new TypePromoter(type, IS_BYTE | IS_VAR | IS_SORTED, VAR_TYPE_VALUE_SIZE); - case BYTES_VAR_STRAIGHT: - return new TypePromoter(type, IS_BYTE | IS_VAR | IS_STRAIGHT, VAR_TYPE_VALUE_SIZE); - case FIXED_INTS_16: - return new TypePromoter(type, - IS_INT | IS_FIXED | IS_STRAIGHT | IS_16_BIT, valueSize); - case FIXED_INTS_32: - return new TypePromoter(type, - IS_INT | IS_FIXED | IS_STRAIGHT | IS_32_BIT, valueSize); - case FIXED_INTS_64: - return new TypePromoter(type, - IS_INT | IS_FIXED | IS_STRAIGHT | IS_64_BIT, valueSize); - case FIXED_INTS_8: - return new TypePromoter(type, IS_INT | IS_FIXED | IS_STRAIGHT | IS_8_BIT, - valueSize); - case FLOAT_32: - return new TypePromoter(type, IS_FLOAT | IS_FIXED | IS_STRAIGHT - | IS_32_BIT, valueSize); - case FLOAT_64: - return new TypePromoter(type, IS_FLOAT | IS_FIXED | IS_STRAIGHT - | IS_64_BIT, valueSize); - case VAR_INTS: - return new TypePromoter(type, IS_INT | IS_VAR | IS_STRAIGHT, VAR_TYPE_VALUE_SIZE); - default: - throw new IllegalStateException(); + case BYTES_FIXED_DEREF: + return promoter.set(type, IS_BYTE | IS_FIXED | IS_DEREF, valueSize); + case BYTES_FIXED_SORTED: + return promoter.set(type, IS_BYTE | IS_FIXED | IS_SORTED, valueSize); + case BYTES_FIXED_STRAIGHT: + return promoter.set(type, IS_BYTE | IS_FIXED | IS_STRAIGHT, valueSize); + case BYTES_VAR_DEREF: + return promoter.set(type, IS_BYTE | IS_VAR | IS_DEREF, + VAR_TYPE_VALUE_SIZE); + case BYTES_VAR_SORTED: + return promoter.set(type, IS_BYTE | IS_VAR | IS_SORTED, + VAR_TYPE_VALUE_SIZE); + case BYTES_VAR_STRAIGHT: + return promoter.set(type, IS_BYTE | IS_VAR | IS_STRAIGHT, + VAR_TYPE_VALUE_SIZE); + case FIXED_INTS_16: + return promoter.set(type, IS_INT | IS_FIXED | IS_STRAIGHT | IS_16_BIT, + valueSize); + case FIXED_INTS_32: + return promoter.set(type, IS_INT | IS_FIXED | IS_STRAIGHT | IS_32_BIT, + valueSize); + case FIXED_INTS_64: + return promoter.set(type, IS_INT | IS_FIXED | IS_STRAIGHT | IS_64_BIT, + valueSize); + case FIXED_INTS_8: + return promoter.set(type, IS_INT | IS_FIXED | IS_STRAIGHT | IS_8_BIT, + valueSize); + case FLOAT_32: + return promoter.set(type, + IS_FLOAT | IS_FIXED | IS_STRAIGHT | IS_32_BIT_FLOAT, valueSize); + case FLOAT_64: + return promoter.set(type, + IS_FLOAT | IS_FIXED | IS_STRAIGHT | IS_64_BIT_FLOAT, valueSize); + case VAR_INTS: + return promoter.set(type, IS_INT | IS_VAR | IS_STRAIGHT, + VAR_TYPE_VALUE_SIZE); + default: + throw new IllegalStateException(); } } - + + public static int getValueSize(DocValues.Type type, BytesRef ref) { + switch (type) { + case VAR_INTS: + case BYTES_VAR_DEREF: + case BYTES_VAR_SORTED: + case BYTES_VAR_STRAIGHT: + return -1; + case BYTES_FIXED_DEREF: + case BYTES_FIXED_SORTED: + case BYTES_FIXED_STRAIGHT: + assert ref != null; + return ref.length; + case FIXED_INTS_16: + return 2; + case FLOAT_32: + case FIXED_INTS_32: + return 4; + case FLOAT_64: + case FIXED_INTS_64: + return 8; + case FIXED_INTS_8: + return 1; + default: + throw new IllegalArgumentException("unknonw docvalues type: " + + type.name()); + } + } + /** * Returns a {@link TypePromoter} that always promotes to the type provided to * {@link #promote(TypePromoter)} @@ -195,16 +305,58 @@ class TypePromoter { public static TypePromoter getIdentityPromoter() { return IDENTITY_PROMOTER; } - + + private static TypePromoter newPromoter() { + return new TypePromoter(null, 0, -1); + } + private static class IdentityTypePromoter extends TypePromoter { - + public IdentityTypePromoter() { super(null, 0, -1); } - + + @Override + protected TypePromoter set(Type type, int flags, int valueSize) { + throw new UnsupportedOperationException("can not reset IdendityPromotoer"); + } + @Override public TypePromoter promote(TypePromoter promoter) { return promoter; } } + + static class TypeCompatibility { + private final TypePromoter base; + private final TypePromoter spare; + + TypeCompatibility(Type type, int valueSize) { + this.base = create(type, valueSize); + spare = newPromoter(); + } + + boolean isCompatible(Type type, int valueSize) { + TypePromoter reset = reset(type, valueSize, spare); + if (base.isTypeCompatible(reset)) { + if (base.isBytesCompatible(reset)) { + return base.valueSize == -1 || base.valueSize == valueSize; + } else if (base.flags == reset.flags) { + return true; + } else if (base.isNumericSizeCompatible(reset)) { + return base.valueSize == -1 + || (base.valueSize > valueSize && valueSize > 0); + } + } + return false; + } + + Type getBaseType() { + return base.type(); + } + + int getBaseSize() { + return base.valueSize; + } + } } \ No newline at end of file diff --git a/lucene/core/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java index f16ce5f..e4d4fc2 100644 --- a/lucene/core/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java +++ b/lucene/core/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -74,12 +74,12 @@ public class UpgradeIndexMergePolicy extends MergePolicy { } @Override - public MergeSpecification findMerges(SegmentInfos segmentInfos) throws CorruptIndexException, IOException { + public MergeSpecification findMerges(SegmentInfos segmentInfos) throws IOException { return base.findMerges(segmentInfos); } @Override - public MergeSpecification findForcedMerges(SegmentInfos segmentInfos, int maxSegmentCount, Map segmentsToMerge) throws CorruptIndexException, IOException { + public MergeSpecification findForcedMerges(SegmentInfos segmentInfos, int maxSegmentCount, Map segmentsToMerge) throws IOException { // first find all old segments final Map oldSegments = new HashMap(); for (final SegmentInfoPerCommit si : segmentInfos) { @@ -129,7 +129,7 @@ public class UpgradeIndexMergePolicy extends MergePolicy { } @Override - public MergeSpecification findForcedDeletesMerges(SegmentInfos segmentInfos) throws CorruptIndexException, IOException { + public MergeSpecification findForcedDeletesMerges(SegmentInfos segmentInfos) throws IOException { return base.findForcedDeletesMerges(segmentInfos); } diff --git a/lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java b/lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java index 8096e7f..41f1503 100644 --- a/lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/BitsFilteredDocIdSet.java b/lucene/core/src/java/org/apache/lucene/search/BitsFilteredDocIdSet.java index f196ba9..9177c72 100644 --- a/lucene/core/src/java/org/apache/lucene/search/BitsFilteredDocIdSet.java +++ b/lucene/core/src/java/org/apache/lucene/search/BitsFilteredDocIdSet.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanClause.java b/lucene/core/src/java/org/apache/lucene/search/BooleanClause.java index 91af8e6..5b17a03 100644 --- a/lucene/core/src/java/org/apache/lucene/search/BooleanClause.java +++ b/lucene/core/src/java/org/apache/lucene/search/BooleanClause.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java b/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java index 1360a7a..1deb09b 100644 --- a/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,14 +17,7 @@ package org.apache.lucene.search; * limitations under the License. */ -import java.io.IOException; -import java.util.*; - -import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.*; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.ConjunctionTermScorer.DocsAndFreqs; import org.apache.lucene.search.TermQuery.TermDocsEnumFactory; @@ -34,6 +27,12 @@ import org.apache.lucene.search.similarities.Similarity.ExactSimScorer; import org.apache.lucene.util.Bits; import org.apache.lucene.util.ToStringUtils; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + /** A Query that matches documents matching boolean combinations of other * queries, e.g. {@link TermQuery}s, {@link PhraseQuery}s or other * BooleanQuerys. @@ -376,8 +375,8 @@ public class BooleanQuery extends Query implements Iterable { // and fallback to full match-only scorer: return createMatchOnlyConjunctionTermScorer(context, acceptDocs); } - TermDocsEnumFactory factory = new TermDocsEnumFactory(termsEnum, docsAndFreqsEnum, docsAndFreqsEnum, acceptDocs); - docsAndFreqs[i] = new DocsAndFreqs(termsEnum.docFreq(), docScorer, factory); + TermDocsEnumFactory factory = new TermDocsEnumFactory(termsEnum, acceptDocs); + docsAndFreqs[i] = new DocsAndFreqs(termsEnum.docFreq(), docScorer, docsAndFreqsEnum, factory); } return new ConjunctionTermScorer(this, disableCoord ? 1.0f : coord( docsAndFreqs.length, docsAndFreqs.length), docsAndFreqs); @@ -394,8 +393,8 @@ public class BooleanQuery extends Query implements Iterable { return null; } final ExactSimScorer docScorer = weight.createDocScorer(context); - TermDocsEnumFactory factory = new TermDocsEnumFactory(termsEnum, termsEnum.docs(acceptDocs, null, false), null, acceptDocs); - docsAndFreqs[i] = new DocsAndFreqs(termsEnum.docFreq(), docScorer, factory); + TermDocsEnumFactory factory = new TermDocsEnumFactory(termsEnum, acceptDocs); + docsAndFreqs[i] = new DocsAndFreqs(termsEnum.docFreq(), docScorer, termsEnum.docs(acceptDocs, null, false), factory); } return new MatchOnlyConjunctionTermScorer(this, disableCoord ? 1.0f : coord( diff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java b/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java index c02553a..60c3914 100644 --- a/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -101,7 +101,7 @@ final class BooleanScorer extends Scorer { } @Override - public void setScorer(Scorer scorer) throws IOException { + public void setScorer(Scorer scorer) { this.scorer = scorer; } @@ -125,7 +125,7 @@ final class BooleanScorer extends Scorer { public BucketScorer(Weight weight) { super(weight); } @Override - public int advance(int target) throws IOException { return NO_MORE_DOCS; } + public int advance(int target) { return NO_MORE_DOCS; } @Override public int docID() { return doc; } @@ -134,15 +134,15 @@ final class BooleanScorer extends Scorer { public float freq() { return freq; } @Override - public int nextDoc() throws IOException { return NO_MORE_DOCS; } - - @Override - public float score() throws IOException { return score; } + public int nextDoc() { return NO_MORE_DOCS; } @Override public PositionIntervalIterator positions(boolean needsPayloads, boolean needsOffsets) throws IOException { return PositionIntervalIterator.NO_MORE_POSITIONS; } + + @Override + public float score() { return score; } } @@ -189,8 +189,7 @@ final class BooleanScorer extends Scorer { public SubScorer next; public SubScorer(Scorer scorer, boolean required, boolean prohibited, - Collector collector, SubScorer next) - throws IOException { + Collector collector, SubScorer next) { if (required) { throw new IllegalArgumentException("this scorer cannot handle required=true"); } @@ -307,7 +306,7 @@ final class BooleanScorer extends Scorer { } @Override - public int advance(int target) throws IOException { + public int advance(int target) { throw new UnsupportedOperationException(); } @@ -317,7 +316,7 @@ final class BooleanScorer extends Scorer { } @Override - public int nextDoc() throws IOException { + public int nextDoc() { throw new UnsupportedOperationException(); } diff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java b/lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java index 0598735..ddf754a 100644 --- a/lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java +++ b/lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/BoostAttribute.java b/lucene/core/src/java/org/apache/lucene/search/BoostAttribute.java index 87c7b6e..b60db85 100644 --- a/lucene/core/src/java/org/apache/lucene/search/BoostAttribute.java +++ b/lucene/core/src/java/org/apache/lucene/search/BoostAttribute.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/BoostAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/search/BoostAttributeImpl.java index f079090..7552fec 100644 --- a/lucene/core/src/java/org/apache/lucene/search/BoostAttributeImpl.java +++ b/lucene/core/src/java/org/apache/lucene/search/BoostAttributeImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/CachingCollector.java b/lucene/core/src/java/org/apache/lucene/search/CachingCollector.java index 0fe9f60..eb7bbe9 100644 --- a/lucene/core/src/java/org/apache/lucene/search/CachingCollector.java +++ b/lucene/core/src/java/org/apache/lucene/search/CachingCollector.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -342,13 +342,13 @@ public abstract class CachingCollector extends Collector { } @Override - public void setScorer(Scorer scorer) throws IOException {} + public void setScorer(Scorer scorer) {} @Override - public void collect(int doc) throws IOException {} + public void collect(int doc) {} @Override - public void setNextReader(AtomicReaderContext context) throws IOException {} + public void setNextReader(AtomicReaderContext context) {} }; return create(other, cacheScores, maxRAMMB); diff --git a/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java b/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java index 0a382bb..0780c4c 100644 --- a/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java +++ b/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/CollectionStatistics.java b/lucene/core/src/java/org/apache/lucene/search/CollectionStatistics.java index e0bc3e8..09cf54c 100644 --- a/lucene/core/src/java/org/apache/lucene/search/CollectionStatistics.java +++ b/lucene/core/src/java/org/apache/lucene/search/CollectionStatistics.java @@ -3,7 +3,7 @@ package org.apache.lucene.search; import org.apache.lucene.index.IndexReader; // javadocs import org.apache.lucene.index.Terms; // javadocs -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/Collector.java b/lucene/core/src/java/org/apache/lucene/search/Collector.java index 09aad8a..3ed5694 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Collector.java +++ b/lucene/core/src/java/org/apache/lucene/search/Collector.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/ComplexExplanation.java b/lucene/core/src/java/org/apache/lucene/search/ComplexExplanation.java index d2eceaf..511082c 100644 --- a/lucene/core/src/java/org/apache/lucene/search/ComplexExplanation.java +++ b/lucene/core/src/java/org/apache/lucene/search/ComplexExplanation.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java b/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java index a29465a..ae4e33d 100644 --- a/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java b/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java index dbf1c0e..0dd4093 100644 --- a/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -38,7 +38,7 @@ class ConjunctionTermScorer extends Scorer { private DocsAndFreqs[] origDocsAndFreqs; ConjunctionTermScorer(Weight weight, float coord, - DocsAndFreqs[] docsAndFreqs) throws IOException { + DocsAndFreqs[] docsAndFreqs) { super(weight); this.coord = coord; this.docsAndFreqs = docsAndFreqs; @@ -105,16 +105,16 @@ class ConjunctionTermScorer extends Scorer { } static final class DocsAndFreqs { - final DocsEnum docsAndFreqs; + //final DocsEnum docsAndFreqs; final DocsEnum docs; final int docFreq; final ExactSimScorer docScorer; int doc = -1; private final TermDocsEnumFactory factory; - DocsAndFreqs( int docFreq, ExactSimScorer docScorer, TermDocsEnumFactory factory) throws IOException { - this.docsAndFreqs = factory.docsAndFreqsEnum(); - this.docs = factory.docsEnum(); + DocsAndFreqs( int docFreq, ExactSimScorer docScorer, DocsEnum docs, TermDocsEnumFactory factory) throws IOException { + //this.docsAndFreqs = factory.docsAndFreqsEnum(); + this.docs = docs; this.docFreq = docFreq; this.docScorer = docScorer; this.factory = factory; diff --git a/lucene/core/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java b/lucene/core/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java index 56e3000..d36b1a3 100644 --- a/lucene/core/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java +++ b/lucene/core/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,13 +21,13 @@ import java.io.IOException; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefHash; -import org.apache.lucene.util.TermContext; import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.BytesRefHash.DirectBytesStartArray; @@ -122,7 +122,7 @@ class ConstantScoreAutoRewrite extends TermCollectingRewrite { } @Override - public void setNextEnum(TermsEnum termsEnum) throws IOException { + public void setNextEnum(TermsEnum termsEnum) { this.termsEnum = termsEnum; } diff --git a/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java b/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java index f56159d..b1cee9c 100644 --- a/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -173,7 +173,7 @@ public class ConstantScoreQuery extends Query { final DocIdSetIterator docIdSetIterator; final float theScore; - public ConstantScorer(DocIdSetIterator docIdSetIterator, Weight w, float theScore) throws IOException { + public ConstantScorer(DocIdSetIterator docIdSetIterator, Weight w, float theScore) { super(w); this.theScore = theScore; this.docIdSetIterator = docIdSetIterator; diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java index 0563132..13d45c4 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java @@ -58,7 +58,7 @@ class DisjunctionMaxScorer extends Scorer { * length may be larger than the actual number of scorers. */ public DisjunctionMaxScorer(Weight weight, float tieBreakerMultiplier, - Scorer[] subScorers, int numScorers) throws IOException { + Scorer[] subScorers, int numScorers) { super(weight); this.tieBreakerMultiplier = tieBreakerMultiplier; // The passed subScorers array includes only scorers which have documents diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java index fcd006e..8389647 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/DocIdSet.java b/lucene/core/src/java/org/apache/lucene/search/DocIdSet.java index e9b788c..6345e6a 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DocIdSet.java +++ b/lucene/core/src/java/org/apache/lucene/search/DocIdSet.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -31,11 +31,11 @@ public abstract class DocIdSet { private final DocIdSetIterator iterator = new DocIdSetIterator() { @Override - public int advance(int target) throws IOException { return NO_MORE_DOCS; } + public int advance(int target) { return NO_MORE_DOCS; } @Override public int docID() { return NO_MORE_DOCS; } @Override - public int nextDoc() throws IOException { return NO_MORE_DOCS; } + public int nextDoc() { return NO_MORE_DOCS; } }; @Override @@ -50,7 +50,7 @@ public abstract class DocIdSet { // we explicitely provide no random access, as this filter is 100% sparse and iterator exits faster @Override - public Bits bits() throws IOException { + public Bits bits() { return null; } }; diff --git a/lucene/core/src/java/org/apache/lucene/search/DocIdSetIterator.java b/lucene/core/src/java/org/apache/lucene/search/DocIdSetIterator.java index f34f78c..3a77fc7 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DocIdSetIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/DocIdSetIterator.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java b/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java index 4f0dceb..7862847 100644 --- a/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -201,7 +201,7 @@ final class ExactPhraseScorer extends Scorer { } @Override - public float score() throws IOException { + public float score() { return docScorer.score(docID, freq); } diff --git a/lucene/core/src/java/org/apache/lucene/search/Explanation.java b/lucene/core/src/java/org/apache/lucene/search/Explanation.java index 0425f24..ce30602 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Explanation.java +++ b/lucene/core/src/java/org/apache/lucene/search/Explanation.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldCache.java b/lucene/core/src/java/org/apache/lucene/search/FieldCache.java index e7e30f7..590d784 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FieldCache.java +++ b/lucene/core/src/java/org/apache/lucene/search/FieldCache.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -53,6 +53,7 @@ public interface FieldCache { * Hack: When thrown from a Parser (NUMERIC_UTILS_* ones), this stops * processing terms and returns the current FieldCache * array. + * @lucene.internal */ public static final class StopFillCacheException extends RuntimeException { } @@ -213,7 +214,7 @@ public interface FieldCache { public static final IntParser NUMERIC_UTILS_INT_PARSER=new IntParser(){ public int parseInt(BytesRef term) { if (NumericUtils.getPrefixCodedIntShift(term) > 0) - throw new FieldCacheImpl.StopFillCacheException(); + throw new StopFillCacheException(); return NumericUtils.prefixCodedToInt(term); } @Override @@ -229,7 +230,7 @@ public interface FieldCache { public static final FloatParser NUMERIC_UTILS_FLOAT_PARSER=new FloatParser(){ public float parseFloat(BytesRef term) { if (NumericUtils.getPrefixCodedIntShift(term) > 0) - throw new FieldCacheImpl.StopFillCacheException(); + throw new StopFillCacheException(); return NumericUtils.sortableIntToFloat(NumericUtils.prefixCodedToInt(term)); } @Override @@ -245,7 +246,7 @@ public interface FieldCache { public static final LongParser NUMERIC_UTILS_LONG_PARSER = new LongParser(){ public long parseLong(BytesRef term) { if (NumericUtils.getPrefixCodedLongShift(term) > 0) - throw new FieldCacheImpl.StopFillCacheException(); + throw new StopFillCacheException(); return NumericUtils.prefixCodedToLong(term); } @Override @@ -261,7 +262,7 @@ public interface FieldCache { public static final DoubleParser NUMERIC_UTILS_DOUBLE_PARSER = new DoubleParser(){ public double parseDouble(BytesRef term) { if (NumericUtils.getPrefixCodedLongShift(term) > 0) - throw new FieldCacheImpl.StopFillCacheException(); + throw new StopFillCacheException(); return NumericUtils.sortableLongToDouble(NumericUtils.prefixCodedToLong(term)); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldCacheDocIdSet.java b/lucene/core/src/java/org/apache/lucene/search/FieldCacheDocIdSet.java index 11210cd..11d88f3 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FieldCacheDocIdSet.java +++ b/lucene/core/src/java/org/apache/lucene/search/FieldCacheDocIdSet.java @@ -1,5 +1,5 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java b/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java index 4659723..f3ec347 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java +++ b/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -380,7 +380,7 @@ class FieldCacheImpl implements FieldCache { } } } - } catch (StopFillCacheException stop) { + } catch (FieldCache.StopFillCacheException stop) { } } if (setDocsWithField) { @@ -453,7 +453,7 @@ class FieldCacheImpl implements FieldCache { } } } - } catch (StopFillCacheException stop) { + } catch (FieldCache.StopFillCacheException stop) { } } if (setDocsWithField) { @@ -557,7 +557,7 @@ class FieldCacheImpl implements FieldCache { } } } - } catch (StopFillCacheException stop) { + } catch (FieldCache.StopFillCacheException stop) { } } @@ -707,7 +707,7 @@ class FieldCacheImpl implements FieldCache { } } } - } catch (StopFillCacheException stop) { + } catch (FieldCache.StopFillCacheException stop) { } } @@ -795,7 +795,7 @@ class FieldCacheImpl implements FieldCache { } } } - } catch (StopFillCacheException stop) { + } catch (FieldCache.StopFillCacheException stop) { } } @@ -884,7 +884,7 @@ class FieldCacheImpl implements FieldCache { } } } - } catch (StopFillCacheException stop) { + } catch (FieldCache.StopFillCacheException stop) { } } if (retArray == null) { // no values diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java b/lucene/core/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java index aa3da4e..54c3fb5 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java +++ b/lucene/core/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java @@ -1,5 +1,5 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldCacheTermsFilter.java b/lucene/core/src/java/org/apache/lucene/search/FieldCacheTermsFilter.java index 172c71d..bb837c4 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FieldCacheTermsFilter.java +++ b/lucene/core/src/java/org/apache/lucene/search/FieldCacheTermsFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java b/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java index e6d96b1..83b44bf 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java +++ b/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -1063,12 +1063,12 @@ public abstract class FieldComparator { /** Sorts by field's natural Term sort order, using * ordinals. This is functionally equivalent to {@link - * TermValComparator}, but it first resolves the string + * org.apache.lucene.search.FieldComparator.TermValComparator}, but it first resolves the string * to their relative ordinal positions (using the index * returned by {@link FieldCache#getTermsIndex}), and * does most comparisons using the ordinals. For medium * to large results, this comparator will be much faster - * than {@link TermValComparator}. For very small + * than {@link org.apache.lucene.search.FieldComparator.TermValComparator}. For very small * result sets it may be slower. */ public static final class TermOrdValComparator extends FieldComparator { /* Ords for each slot. @@ -1469,7 +1469,7 @@ public abstract class FieldComparator { /** Sorts by field's natural Term sort order, using * ordinals; this is just like {@link - * TermOrdValComparator} except it uses DocValues to + * org.apache.lucene.search.FieldComparator.TermValComparator} except it uses DocValues to * retrieve the sort ords saved during indexing. */ public static final class TermOrdValDocValuesComparator extends FieldComparator { /* Ords for each slot. diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldComparatorSource.java b/lucene/core/src/java/org/apache/lucene/search/FieldComparatorSource.java index 6ad27db..b612ab5 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FieldComparatorSource.java +++ b/lucene/core/src/java/org/apache/lucene/search/FieldComparatorSource.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldDoc.java b/lucene/core/src/java/org/apache/lucene/search/FieldDoc.java index c72dd68..7bf8c30 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FieldDoc.java +++ b/lucene/core/src/java/org/apache/lucene/search/FieldDoc.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldValueFilter.java b/lucene/core/src/java/org/apache/lucene/search/FieldValueFilter.java index 1fb0342..0770da1 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FieldValueFilter.java +++ b/lucene/core/src/java/org/apache/lucene/search/FieldValueFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldValueHitQueue.java b/lucene/core/src/java/org/apache/lucene/search/FieldValueHitQueue.java index 9211f09..5a4d444 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FieldValueHitQueue.java +++ b/lucene/core/src/java/org/apache/lucene/search/FieldValueHitQueue.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/Filter.java b/lucene/core/src/java/org/apache/lucene/search/Filter.java index 290f3c4..440875b 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Filter.java +++ b/lucene/core/src/java/org/apache/lucene/search/Filter.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSet.java b/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSet.java index 4b316f5..0d94ca9 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSet.java +++ b/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSet.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSetIterator.java b/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSetIterator.java index 519bdeb..f16da33 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSetIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSetIterator.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java b/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java index c305489..ee020aa 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/FuzzyQuery.java b/lucene/core/src/java/org/apache/lucene/search/FuzzyQuery.java index c66e5eb..c849045 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FuzzyQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/FuzzyQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java b/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java index 72db835..673f011 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -337,8 +337,7 @@ public class FuzzyTermsEnum extends TermsEnum { private final BoostAttribute boostAtt = attributes().addAttribute(BoostAttribute.class); - public AutomatonFuzzyTermsEnum(TermsEnum tenum, CompiledAutomaton compiled[]) - throws IOException { + public AutomatonFuzzyTermsEnum(TermsEnum tenum, CompiledAutomaton compiled[]) { super(tenum, false); this.matchers = new ByteRunAutomaton[compiled.length]; for (int i = 0; i < compiled.length; i++) diff --git a/lucene/core/src/java/org/apache/lucene/search/HitQueue.java b/lucene/core/src/java/org/apache/lucene/search/HitQueue.java index e5ce5bc..3880bad 100644 --- a/lucene/core/src/java/org/apache/lucene/search/HitQueue.java +++ b/lucene/core/src/java/org/apache/lucene/search/HitQueue.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java index 5dd9fef..c26b16d 100644 --- a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java +++ b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,24 +17,42 @@ package org.apache.lucene.search; * limitations under the License. */ -import org.apache.lucene.document.Document; -import org.apache.lucene.index.*; -import org.apache.lucene.search.positions.PositionIntervalIterator; -import org.apache.lucene.search.similarities.DefaultSimilarity; -import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.store.NIOFSDirectory; -import org.apache.lucene.util.ReaderUtil; -import org.apache.lucene.util.TermContext; -import org.apache.lucene.util.ThreadInterruptedException; +import org.apache.lucene.search.positions.PositionIntervalIterator; import java.io.IOException; +import java.util.Arrays; import java.util.Iterator; +import java.util.List; import java.util.NoSuchElementException; import java.util.Set; -import java.util.concurrent.*; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.DirectoryReader; // javadocs +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.IndexReaderContext; +import org.apache.lucene.index.ReaderUtil; +import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; +import org.apache.lucene.index.Terms; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.store.NIOFSDirectory; // javadoc + +import org.apache.lucene.util.ThreadInterruptedException; + + + /** Implements search over a single IndexReader. * *

    Applications usually need only call the inherited @@ -67,7 +85,7 @@ public class IndexSearcher { // NOTE: these members might change in incompatible ways // in the next release protected final IndexReaderContext readerContext; - protected final AtomicReaderContext[] leafContexts; + protected final List leafContexts; // used with executor - each slice holds a set of leafs executed within one thread protected final LeafSlice[] leafSlices; @@ -152,10 +170,10 @@ public class IndexSearcher { * Each {@link LeafSlice} is executed in a single thread. By default there * will be one {@link LeafSlice} per leaf ({@link AtomicReaderContext}). */ - protected LeafSlice[] slices(AtomicReaderContext...leaves) { - LeafSlice[] slices = new LeafSlice[leaves.length]; + protected LeafSlice[] slices(List leaves) { + LeafSlice[] slices = new LeafSlice[leaves.size()]; for (int i = 0; i < slices.length; i++) { - slices[i] = new LeafSlice(leaves[i]); + slices[i] = new LeafSlice(leaves.get(i)); } return slices; } @@ -167,17 +185,17 @@ public class IndexSearcher { } /** Sugar for .getIndexReader().document(docID) */ - public Document doc(int docID) throws CorruptIndexException, IOException { + public Document doc(int docID) throws IOException { return reader.document(docID); } /** Sugar for .getIndexReader().document(docID, fieldVisitor) */ - public void doc(int docID, StoredFieldVisitor fieldVisitor) throws CorruptIndexException, IOException { + public void doc(int docID, StoredFieldVisitor fieldVisitor) throws IOException { reader.document(docID, fieldVisitor); } /** Sugar for .getIndexReader().document(docID, fieldsToLoad) */ - public final Document document(int docID, Set fieldsToLoad) throws CorruptIndexException, IOException { + public final Document document(int docID, Set fieldsToLoad) throws IOException { return reader.document(docID, fieldsToLoad); } @@ -250,12 +268,6 @@ public class IndexSearcher { * *

    {@link Collector#collect(int)} is called for every matching * document. - *
    Collector-based access to remote indexes is discouraged. - * - *

    Applications should only use this if they need all of the - * matching documents. The high-level search API ({@link - * IndexSearcher#search(Query, Filter, int)}) is usually more efficient, as it skips - * non-high-scoring hits. * * @param query to match documents * @param filter if non-null, used to permit documents to be collected. @@ -271,13 +283,6 @@ public class IndexSearcher { * *

    {@link Collector#collect(int)} is called for every matching document. * - *

    Applications should only use this if they need all of the - * matching documents. The high-level search API ({@link - * IndexSearcher#search(Query, int)}) is usually more efficient, as it skips - * non-high-scoring hits. - *

    Note: The score passed to this method is a raw score. - * In other words, the score will not necessarily be a float whose value is - * between 0 and 1. * @throws BooleanQuery.TooManyClauses */ public void search(Query query, Collector results) @@ -440,7 +445,7 @@ public class IndexSearcher { * {@link IndexSearcher#search(Query,Filter,int)} instead. * @throws BooleanQuery.TooManyClauses */ - protected TopDocs search(AtomicReaderContext[] leaves, Weight weight, ScoreDoc after, int nDocs) throws IOException { + protected TopDocs search(List leaves, Weight weight, ScoreDoc after, int nDocs) throws IOException { // single thread int limit = reader.maxDoc(); if (limit == 0) { @@ -477,7 +482,7 @@ public class IndexSearcher { *

    NOTE: this does not compute scores by default. If you * need scores, create a {@link TopFieldCollector} * instance by calling {@link TopFieldCollector#create} and - * then pass that to {@link #search(AtomicReaderContext[], Weight, + * then pass that to {@link #search(List, Weight, * Collector)}.

    */ protected TopFieldDocs search(Weight weight, FieldDoc after, int nDocs, @@ -525,7 +530,7 @@ public class IndexSearcher { * whether or not the fields in the returned {@link FieldDoc} instances should * be set by specifying fillFields. */ - protected TopFieldDocs search(AtomicReaderContext[] leaves, Weight weight, FieldDoc after, int nDocs, + protected TopFieldDocs search(List leaves, Weight weight, FieldDoc after, int nDocs, Sort sort, boolean fillFields, boolean doDocScores, boolean doMaxScore) throws IOException { // single thread int limit = reader.maxDoc(); @@ -546,12 +551,6 @@ public class IndexSearcher { * *

    * {@link Collector#collect(int)} is called for every document.
    - * Collector-based access to remote indexes is discouraged. - * - *

    - * Applications should only use this if they need all of the matching - * documents. The high-level search API ({@link IndexSearcher#search(Query,int)}) is - * usually more efficient, as it skips non-high-scoring hits. * *

    * NOTE: this method executes the searches on all given leaves exclusively. @@ -565,15 +564,15 @@ public class IndexSearcher { * to receive hits * @throws BooleanQuery.TooManyClauses */ - protected void search(AtomicReaderContext[] leaves, Weight weight, Collector collector) + protected void search(List leaves, Weight weight, Collector collector) throws IOException { // TODO: should we make this // threaded...? the Collector could be sync'd? // always use single thread: - for (int i = 0; i < leaves.length; i++) { // search each subreader - collector.setNextReader(leaves[i]); - Scorer scorer = weight.scorer(leaves[i], !collector.acceptsDocsOutOfOrder(), true, leaves[i].reader().getLiveDocs()); + for (AtomicReaderContext ctx : leaves) { // search each subreader + collector.setNextReader(ctx); + Scorer scorer = weight.scorer(ctx, !collector.acceptsDocsOutOfOrder(), true, ctx.reader().getLiveDocs()); if (scorer != null) { scorer.score(collector); } @@ -617,9 +616,10 @@ public class IndexSearcher { */ protected Explanation explain(Weight weight, int doc) throws IOException { int n = ReaderUtil.subIndex(doc, leafContexts); - int deBasedDoc = doc - leafContexts[n].docBase; + final AtomicReaderContext ctx = leafContexts.get(n); + int deBasedDoc = doc - ctx.docBase; - return weight.explain(leafContexts[n], deBasedDoc); + return weight.explain(ctx, deBasedDoc); } /** @@ -675,7 +675,7 @@ public class IndexSearcher { } public TopDocs call() throws IOException { - final TopDocs docs = searcher.search(slice.leaves, weight, after, nDocs); + final TopDocs docs = searcher.search(Arrays.asList(slice.leaves), weight, after, nDocs); final ScoreDoc[] scoreDocs = docs.scoreDocs; //it would be so nice if we had a thread-safe insert lock.lock(); @@ -769,11 +769,13 @@ public class IndexSearcher { public TopFieldDocs call() throws IOException { assert slice.leaves.length == 1; - final TopFieldDocs docs = searcher.search(slice.leaves, weight, after, nDocs, sort, true, doDocScores, doMaxScore); + final TopFieldDocs docs = searcher.search(Arrays.asList(slice.leaves), + weight, after, nDocs, sort, true, doDocScores, doMaxScore); lock.lock(); try { - final int base = slice.leaves[0].docBase; - hq.setNextReader(slice.leaves[0]); + final AtomicReaderContext ctx = slice.leaves[0]; + final int base = ctx.docBase; + hq.setNextReader(ctx); hq.setScorer(fakeScorer); for(ScoreDoc scoreDoc : docs.scoreDocs) { fakeScorer.doc = scoreDoc.doc - base; @@ -849,7 +851,7 @@ public class IndexSearcher { public static class LeafSlice { final AtomicReaderContext[] leaves; - public LeafSlice(AtomicReaderContext...leaves) { + public LeafSlice(AtomicReaderContext... leaves) { this.leaves = leaves; } } diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java b/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java index 247f412..8c04317 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -39,7 +39,7 @@ public class MatchAllDocsQuery extends Query { private final int maxDoc; private final Bits liveDocs; - MatchAllScorer(IndexReader reader, Bits liveDocs, Weight w, float score) throws IOException { + MatchAllScorer(IndexReader reader, Bits liveDocs, Weight w, float score) { super(w); this.liveDocs = liveDocs; this.score = score; diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchOnlyConjunctionTermsScorer.java b/lucene/core/src/java/org/apache/lucene/search/MatchOnlyConjunctionTermsScorer.java index 9f39d91..a8336f2 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MatchOnlyConjunctionTermsScorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/MatchOnlyConjunctionTermsScorer.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,17 +17,15 @@ package org.apache.lucene.search; * limitations under the License. */ -import java.io.IOException; - /** Scorer for conjunctions, sets of terms, all of which are required. */ final class MatchOnlyConjunctionTermScorer extends ConjunctionTermScorer { MatchOnlyConjunctionTermScorer(Weight weight, float coord, - DocsAndFreqs[] docsAndFreqs) throws IOException { + DocsAndFreqs[] docsAndFreqs) { super(weight, coord, docsAndFreqs); } @Override - public float score() throws IOException { + public float score() { float sum = 0.0f; for (DocsAndFreqs docs : docsAndFreqs) { sum += docs.docScorer.score(lastDoc, 1); diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java b/lucene/core/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java index 7c28376..b0e792e 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -43,7 +43,7 @@ final class MatchOnlyTermScorer extends Scorer { * The Similarity.ExactSimScorer implementation * to be used for score computations. */ - MatchOnlyTermScorer(Weight weight, DocsEnum td, Similarity.ExactSimScorer docScorer) throws IOException { + MatchOnlyTermScorer(Weight weight, DocsEnum td, Similarity.ExactSimScorer docScorer) { super(weight); this.docScorer = docScorer; this.docsEnum = td; diff --git a/lucene/core/src/java/org/apache/lucene/search/MaxNonCompetitiveBoostAttribute.java b/lucene/core/src/java/org/apache/lucene/search/MaxNonCompetitiveBoostAttribute.java index ecee934..34e431e 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MaxNonCompetitiveBoostAttribute.java +++ b/lucene/core/src/java/org/apache/lucene/search/MaxNonCompetitiveBoostAttribute.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/MaxNonCompetitiveBoostAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/search/MaxNonCompetitiveBoostAttributeImpl.java index 629f600..e78a068 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MaxNonCompetitiveBoostAttributeImpl.java +++ b/lucene/core/src/java/org/apache/lucene/search/MaxNonCompetitiveBoostAttributeImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiCollector.java b/lucene/core/src/java/org/apache/lucene/search/MultiCollector.java index ea15a68..40c0838 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MultiCollector.java +++ b/lucene/core/src/java/org/apache/lucene/search/MultiCollector.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java index c7fecc8..2b6155b 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,6 +18,7 @@ package org.apache.lucene.search; */ import org.apache.lucene.index.*; +import org.apache.lucene.index.TermContext; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.search.similarities.Similarity.SloppySimScorer; import org.apache.lucene.util.*; @@ -206,7 +207,7 @@ public class MultiPhraseQuery extends Query { // None of the terms are in this reader return null; } - factory = null; // nocommit - what to do here + factory = new MultiTermDocsEnumFactory(liveDocs, context, terms, termContexts, termsEnum); } else { final Term term = terms[0]; TermState termState = termContexts.get(term).get(context.ord); @@ -223,8 +224,7 @@ public class MultiPhraseQuery extends Query { throw new IllegalStateException("field \"" + term.field() + "\" was indexed without position data; cannot run PhraseQuery (term=" + term.text() + ")"); } - docFreq = termsEnum.docFreq(); - factory = new TermQuery.TermDocsEnumFactory(BytesRef.deepCopyOf(term.bytes()), termState, termsEnum, postingsEnum, postingsEnum, acceptDocs); + factory = new TermQuery.TermDocsEnumFactory(BytesRef.deepCopyOf(term.bytes()), termsEnum, acceptDocs); } postingsFreqs[pos] = new PhraseQuery.PostingsAndFreq(postingsEnum, factory, termsEnum.docFreq() , positions.get(pos).intValue(), terms); @@ -393,6 +393,27 @@ public class MultiPhraseQuery extends Query { } return true; } + + private static class MultiTermDocsEnumFactory extends TermQuery.TermDocsEnumFactory { + + AtomicReaderContext context; + Term[] terms; + Map termContexts; + + MultiTermDocsEnumFactory(Bits liveDocs, AtomicReaderContext context, Term[] terms, + Map termContexts, TermsEnum termsEnum) throws IOException { + super(termsEnum, liveDocs); + this.context = context; + this.terms = terms; + this.termContexts = termContexts; + } + + @Override + public DocsAndPositionsEnum docsAndPositionsEnum(boolean offsets) throws IOException { + return new UnionDocsAndPositionsEnum(liveDocs, context, terms, termContexts, termsEnum, offsets); + } + + } } /** @@ -421,25 +442,41 @@ class UnionDocsAndPositionsEnum extends DocsAndPositionsEnum { } } - private static final class IntQueue { - private int _arraySize = 16; + // TODO: Reimplement this as int[_arraySize * 3], storing position at i * 3, + // startOffset at i * 3 + 1 and endOffset at i * 3 + 2. Will need to also + // implement a new SorterTemplate to sort the array. + + private static final class PositionQueue { + private int _arraySize = 48; private int _index = 0; private int _lastIndex = 0; private int[] _array = new int[_arraySize]; - final void add(int i) { - if (_lastIndex == _arraySize) + final void add(int pos, int start, int end) { + if (_lastIndex * 3 == _arraySize) growArray(); - _array[_lastIndex++] = i; + _array[_lastIndex * 3] = pos; + _array[_lastIndex * 3 + 1] = start; + _array[_lastIndex * 3 + 2] = end; + _lastIndex += 1; } final int next() { - return _array[_index++]; + return _array[_index++ * 3]; + } + + final int startOffset() { + return _array[(_index - 1) * 3 + 1]; + } + + final int endOffset() { + return _array[(_index - 1) * 3 + 2]; } final void sort() { - Arrays.sort(_array, _index, _lastIndex); + //Arrays.sort(_array, _index, _lastIndex); + sorter.quickSort(_index, _lastIndex - 1); } final void clear() { @@ -457,14 +494,52 @@ class UnionDocsAndPositionsEnum extends DocsAndPositionsEnum { _array = newArray; _arraySize *= 2; } + + private SorterTemplate sorter = new SorterTemplate() { + private int pivot; + + @Override + protected void swap(int i, int j) { + int ti = _array[i * 3]; + int ts = _array[i * 3 + 1]; + int te = _array[i * 3 + 2]; + _array[i * 3] = _array[j * 3]; + _array[i * 3 + 1] = _array[j * 3 + 1]; + _array[i * 3 + 2] = _array[j * 3 + 2]; + _array[j * 3] = ti; + _array[j * 3 + 1] = ts; + _array[j * 3 + 2] = te; + } + + @Override + protected int compare(int i, int j) { + return _array[i * 3] - _array[j * 3]; + } + + @Override + protected void setPivot(int i) { + pivot = i; + } + + @Override + protected int comparePivot(int j) { + return pivot - _array[j * 3]; + } + }; } private int _doc; private int _freq; private DocsQueue _queue; - private IntQueue _posList; + private PositionQueue _posList; + + public UnionDocsAndPositionsEnum(Bits liveDocs, AtomicReaderContext context, Term[] terms, + Map termContexts, TermsEnum termsEnum) throws IOException { + this(liveDocs, context, terms, termContexts, termsEnum, false); + } - public UnionDocsAndPositionsEnum(Bits liveDocs, AtomicReaderContext context, Term[] terms, Map termContexts, TermsEnum termsEnum) throws IOException { + public UnionDocsAndPositionsEnum(Bits liveDocs, AtomicReaderContext context, Term[] terms, + Map termContexts, TermsEnum termsEnum, boolean needsOffsets) throws IOException { List docsEnums = new LinkedList(); for (int i = 0; i < terms.length; i++) { final Term term = terms[i]; @@ -474,7 +549,7 @@ class UnionDocsAndPositionsEnum extends DocsAndPositionsEnum { continue; } termsEnum.seekExact(term.bytes(), termState); - DocsAndPositionsEnum postings = termsEnum.docsAndPositions(liveDocs, null, false); + DocsAndPositionsEnum postings = termsEnum.docsAndPositions(liveDocs, null, needsOffsets); if (postings == null) { // term does exist, but has no positions throw new IllegalStateException("field \"" + term.field() + "\" was indexed without position data; cannot run PhraseQuery (term=" + term.text() + ")"); @@ -483,7 +558,7 @@ class UnionDocsAndPositionsEnum extends DocsAndPositionsEnum { } _queue = new DocsQueue(docsEnums); - _posList = new IntQueue(); + _posList = new PositionQueue(); } @Override @@ -505,7 +580,7 @@ class UnionDocsAndPositionsEnum extends DocsAndPositionsEnum { final int freq = postings.freq(); for (int i = 0; i < freq; i++) { - _posList.add(postings.nextPosition()); + _posList.add(postings.nextPosition(), postings.startOffset(), postings.endOffset()); } if (postings.nextDoc() != NO_MORE_DOCS) { @@ -528,12 +603,12 @@ class UnionDocsAndPositionsEnum extends DocsAndPositionsEnum { @Override public int startOffset() { - return -1; + return _posList.startOffset(); } @Override public int endOffset() { - return -1; + return _posList.endOffset(); } @Override @@ -558,7 +633,7 @@ class UnionDocsAndPositionsEnum extends DocsAndPositionsEnum { } @Override - public final int freq() throws IOException { + public final int freq() { return _freq; } diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java b/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java index 6bf3d23..e8c24bd 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,10 +23,10 @@ import org.apache.lucene.index.FilteredTermsEnum; // javadocs import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.SingleTermsEnum; // javadocs import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.AttributeSource; -import org.apache.lucene.util.TermContext; /** * An abstract {@link Query} that matches documents diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java index c3c09d5..a67d1c4 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java +++ b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/NGramPhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/NGramPhraseQuery.java index cec4e16..20c5c39 100644 --- a/lucene/core/src/java/org/apache/lucene/search/NGramPhraseQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/NGramPhraseQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/NRTManager.java b/lucene/core/src/java/org/apache/lucene/search/NRTManager.java index 2946dbc..8f6719c 100644 --- a/lucene/core/src/java/org/apache/lucene/search/NRTManager.java +++ b/lucene/core/src/java/org/apache/lucene/search/NRTManager.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -26,7 +26,6 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; // javadocs import org.apache.lucene.index.IndexWriter; @@ -232,13 +231,13 @@ public class NRTManager extends ReferenceManager { return indexingGen.get(); } - public long addIndexes(Directory... dirs) throws CorruptIndexException, IOException { + public long addIndexes(Directory... dirs) throws IOException { writer.addIndexes(dirs); // Return gen as of when indexing finished: return indexingGen.get(); } - public long addIndexes(IndexReader... readers) throws CorruptIndexException, IOException { + public long addIndexes(IndexReader... readers) throws IOException { writer.addIndexes(readers); // Return gen as of when indexing finished: return indexingGen.get(); diff --git a/lucene/core/src/java/org/apache/lucene/search/NRTManagerReopenThread.java b/lucene/core/src/java/org/apache/lucene/search/NRTManagerReopenThread.java index 3afc022..e21faf4 100644 --- a/lucene/core/src/java/org/apache/lucene/search/NRTManagerReopenThread.java +++ b/lucene/core/src/java/org/apache/lucene/search/NRTManagerReopenThread.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/NumericRangeFilter.java b/lucene/core/src/java/org/apache/lucene/search/NumericRangeFilter.java index 7d4e216..4c1090a 100644 --- a/lucene/core/src/java/org/apache/lucene/search/NumericRangeFilter.java +++ b/lucene/core/src/java/org/apache/lucene/search/NumericRangeFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java index 07ed12a..24fa0fb 100644 --- a/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -415,7 +415,7 @@ public final class NumericRangeQuery extends MultiTermQuery { private final LinkedList rangeBounds = new LinkedList(); private final Comparator termComp; - NumericRangeTermsEnum(final TermsEnum tenum) throws IOException { + NumericRangeTermsEnum(final TermsEnum tenum) { super(tenum); switch (dataType) { case LONG: @@ -517,7 +517,7 @@ public final class NumericRangeQuery extends MultiTermQuery { } @Override - protected final BytesRef nextSeekTerm(BytesRef term) throws IOException { + protected final BytesRef nextSeekTerm(BytesRef term) { while (rangeBounds.size() >= 2) { nextRange(); diff --git a/lucene/core/src/java/org/apache/lucene/search/PhrasePositions.java b/lucene/core/src/java/org/apache/lucene/search/PhrasePositions.java index b2d4afe..1ded536 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PhrasePositions.java +++ b/lucene/core/src/java/org/apache/lucene/search/PhrasePositions.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,8 +17,10 @@ package org.apache.lucene.search; * limitations under the License. */ +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.Term; + import java.io.IOException; -import org.apache.lucene.index.*; /** * Position of a term in a document that takes into account the term offset within the phrase. @@ -84,6 +86,10 @@ final class PhrasePositions { if (rptGroup >=0 ) { s += " rpt:"+rptGroup+",i"+rptInd; } + s += " t: [" + terms[0]; + for (int i = 1; i < terms.length; i++) + s += "," + terms[1]; + s += "]"; return s; } } diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java index 1f0c1f4..a2d617c 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,7 +20,10 @@ package org.apache.lucene.search; import org.apache.lucene.index.*; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.search.similarities.Similarity.SloppySimScorer; -import org.apache.lucene.util.*; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.ToStringUtils; import java.io.IOException; import java.util.ArrayList; @@ -124,7 +127,7 @@ public class PhraseQuery extends Query { static class PostingsAndFreq implements Comparable { final TermQuery.TermDocsEnumFactory factory; - final DocsAndPositionsEnum postings; + DocsAndPositionsEnum postings; final int docFreq; final int position; final Term[] terms; @@ -132,7 +135,8 @@ public class PhraseQuery extends Query { public PostingsAndFreq(DocsAndPositionsEnum postings, TermQuery.TermDocsEnumFactory factory, int docFreq, int position, Term... terms) throws IOException { this.factory = factory; - this.postings = postings; + //this.postings = postings; + this.postings = factory.docsAndPositionsEnum(false); this.docFreq = docFreq; this.position = position; nTerms = terms==null ? 0 : terms.length; @@ -150,6 +154,10 @@ public class PhraseQuery extends Query { } } + public void reset(boolean needsOffsets) throws IOException { + this.postings = factory.docsAndPositionsEnum(needsOffsets); + } + public int compareTo(PostingsAndFreq other) { if (docFreq != other.docFreq) { return docFreq - other.docFreq; @@ -263,7 +271,7 @@ public class PhraseQuery extends Query { // term does exist, but has no positions throw new IllegalStateException("field \"" + t.field() + "\" was indexed without position data; cannot run PhraseQuery (term=" + t.text() + ")"); } - TermQuery.TermDocsEnumFactory factory = new TermQuery.TermDocsEnumFactory(BytesRef.deepCopyOf(t.bytes()), state, te, null, null, acceptDocs); + TermQuery.TermDocsEnumFactory factory = new TermQuery.TermDocsEnumFactory(BytesRef.deepCopyOf(t.bytes()), te, acceptDocs); postingsFreqs[i] = new PostingsAndFreq(postingsEnum, factory, te.docFreq(), positions.get(i).intValue(), t); } diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQueue.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQueue.java index bac0a97..d2c8655 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PhraseQueue.java +++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQueue.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseScorer.java b/lucene/core/src/java/org/apache/lucene/search/PhraseScorer.java index 19644b3..73bf5fb 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PhraseScorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/PhraseScorer.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,10 +17,10 @@ package org.apache.lucene.search; * limitations under the License. */ -import java.io.IOException; - import org.apache.lucene.search.similarities.Similarity; +import java.io.IOException; + /** Expert: Scoring functionality for phrase queries. *
    A document is considered matching if it contains the phrase-query terms * at "valid" positions. What "valid positions" are @@ -37,22 +37,29 @@ abstract class PhraseScorer extends Scorer { private float freq; //phrase frequency in current doc as computed by phraseFreq(). final Similarity.SloppySimScorer docScorer; + private PhraseQuery.PostingsAndFreq[] postings; PhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings, - Similarity.SloppySimScorer docScorer) { + Similarity.SloppySimScorer docScorer) throws IOException { super(weight); this.docScorer = docScorer; + this.postings = postings; + reset(false); + } + void reset(boolean needsOffsets) throws IOException { // convert tps to a list of phrase positions. // note: phrase-position differs from term-position in that its position // reflects the phrase offset: pp.pos = tp.pos - offset. // this allows to easily identify a matching (exact) phrase // when all PhrasePositions have exactly the same position. if (postings.length > 0) { + postings[0].reset(needsOffsets); min = new PhrasePositions(postings[0].postings, postings[0].position, 0, postings[0].terms); max = min; max.doc = -1; for (int i = 1; i < postings.length; i++) { + postings[i].reset(needsOffsets); PhrasePositions pp = new PhrasePositions(postings[i].postings, postings[i].position, i, postings[i].terms); max.next = pp; max = pp; diff --git a/lucene/core/src/java/org/apache/lucene/search/PositiveScoresOnlyCollector.java b/lucene/core/src/java/org/apache/lucene/search/PositiveScoresOnlyCollector.java index 6770e98..d35a755 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PositiveScoresOnlyCollector.java +++ b/lucene/core/src/java/org/apache/lucene/search/PositiveScoresOnlyCollector.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/PrefixFilter.java b/lucene/core/src/java/org/apache/lucene/search/PrefixFilter.java index 7adf080..32aa44d 100755 --- a/lucene/core/src/java/org/apache/lucene/search/PrefixFilter.java +++ b/lucene/core/src/java/org/apache/lucene/search/PrefixFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/PrefixQuery.java b/lucene/core/src/java/org/apache/lucene/search/PrefixQuery.java index 31e88ea..e1aaa0c 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PrefixQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PrefixQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/PrefixTermsEnum.java b/lucene/core/src/java/org/apache/lucene/search/PrefixTermsEnum.java index e57f89c..96184b6 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PrefixTermsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/search/PrefixTermsEnum.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,8 +17,6 @@ package org.apache.lucene.search; * limitations under the License. */ -import java.io.IOException; - import org.apache.lucene.index.FilteredTermsEnum; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; @@ -35,7 +33,7 @@ public class PrefixTermsEnum extends FilteredTermsEnum { private final BytesRef prefixRef; - public PrefixTermsEnum(TermsEnum tenum, BytesRef prefixText) throws IOException { + public PrefixTermsEnum(TermsEnum tenum, BytesRef prefixText) { super(tenum); setInitialSeekTerm(this.prefixRef = prefixText); } diff --git a/lucene/core/src/java/org/apache/lucene/search/Query.java b/lucene/core/src/java/org/apache/lucene/search/Query.java index 1494963..5def3e6 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Query.java +++ b/lucene/core/src/java/org/apache/lucene/search/Query.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/QueryWrapperFilter.java b/lucene/core/src/java/org/apache/lucene/search/QueryWrapperFilter.java index 7963064..65ff254 100644 --- a/lucene/core/src/java/org/apache/lucene/search/QueryWrapperFilter.java +++ b/lucene/core/src/java/org/apache/lucene/search/QueryWrapperFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/ReferenceManager.java b/lucene/core/src/java/org/apache/lucene/search/ReferenceManager.java index 6bf13fc..a663914 100755 --- a/lucene/core/src/java/org/apache/lucene/search/ReferenceManager.java +++ b/lucene/core/src/java/org/apache/lucene/search/ReferenceManager.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -162,7 +162,7 @@ public abstract class ReferenceManager implements Closeable { public final boolean maybeRefresh() throws IOException { ensureOpen(); - // Ensure only 1 thread does reopen at once; other threads just return immediately: + // Ensure only 1 thread does refresh at once; other threads just return immediately: final boolean doTryRefresh = refreshLock.tryLock(); if (doTryRefresh) { try { @@ -186,10 +186,10 @@ public abstract class ReferenceManager implements Closeable { * will return a refreshed instance. Otherwise, consider using the * non-blocking {@link #maybeRefresh()}. */ - public final void maybeRefreshBlocking() throws IOException, InterruptedException { + public final void maybeRefreshBlocking() throws IOException { ensureOpen(); - // Ensure only 1 thread does reopen at once + // Ensure only 1 thread does refresh at once refreshLock.lock(); try { doMaybeRefresh(); diff --git a/lucene/core/src/java/org/apache/lucene/search/RegexpQuery.java b/lucene/core/src/java/org/apache/lucene/search/RegexpQuery.java index 3f77c99..54d0135 100644 --- a/lucene/core/src/java/org/apache/lucene/search/RegexpQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/RegexpQuery.java @@ -1,7 +1,5 @@ package org.apache.lucene.search; -import java.io.IOException; - import org.apache.lucene.index.Term; import org.apache.lucene.util.ToStringUtils; @@ -9,7 +7,7 @@ import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.AutomatonProvider; import org.apache.lucene.util.automaton.RegExp; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -53,7 +51,7 @@ public class RegexpQuery extends AutomatonQuery { * A provider that provides no named automata */ private static AutomatonProvider defaultProvider = new AutomatonProvider() { - public Automaton getAutomaton(String name) throws IOException { + public Automaton getAutomaton(String name) { return null; } }; diff --git a/lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java b/lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java index 16664ea..d7112af 100644 --- a/lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java b/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java index c744f75..896bc1c 100644 --- a/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java @@ -1,5 +1,5 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java b/lucene/core/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java index 68e2c90..5263413 100644 --- a/lucene/core/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/ScoreDoc.java b/lucene/core/src/java/org/apache/lucene/search/ScoreDoc.java index dfc0f22..512a1d4 100644 --- a/lucene/core/src/java/org/apache/lucene/search/ScoreDoc.java +++ b/lucene/core/src/java/org/apache/lucene/search/ScoreDoc.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/Scorer.java b/lucene/core/src/java/org/apache/lucene/search/Scorer.java index c1864d5..ecdb75e 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Scorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/Scorer.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java b/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java index f36253f..53b5dc2 100644 --- a/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java +++ b/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,6 +20,7 @@ package org.apache.lucene.search; import java.io.IOException; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.MultiTermQuery.RewriteMethod; @@ -28,7 +29,6 @@ import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefHash; -import org.apache.lucene.util.TermContext; import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.BytesRefHash.DirectBytesStartArray; @@ -126,7 +126,7 @@ public abstract class ScoringRewrite extends TermCollectingRewr private BoostAttribute boostAtt; @Override - public void setNextEnum(TermsEnum termsEnum) throws IOException { + public void setNextEnum(TermsEnum termsEnum) { this.termsEnum = termsEnum; this.boostAtt = termsEnum.attributes().addAttribute(BoostAttribute.class); } diff --git a/lucene/core/src/java/org/apache/lucene/search/SearcherFactory.java b/lucene/core/src/java/org/apache/lucene/search/SearcherFactory.java index c121761..03e67c6 100644 --- a/lucene/core/src/java/org/apache/lucene/search/SearcherFactory.java +++ b/lucene/core/src/java/org/apache/lucene/search/SearcherFactory.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/SearcherLifetimeManager.java b/lucene/core/src/java/org/apache/lucene/search/SearcherLifetimeManager.java index 4ffde26..6995682 100644 --- a/lucene/core/src/java/org/apache/lucene/search/SearcherLifetimeManager.java +++ b/lucene/core/src/java/org/apache/lucene/search/SearcherLifetimeManager.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/SearcherManager.java b/lucene/core/src/java/org/apache/lucene/search/SearcherManager.java index ac3f1a4..9c5c743 100644 --- a/lucene/core/src/java/org/apache/lucene/search/SearcherManager.java +++ b/lucene/core/src/java/org/apache/lucene/search/SearcherManager.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java b/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java index c02007b..bc0a3e9 100644 --- a/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -37,10 +37,11 @@ final class SloppyPhraseScorer extends PhraseScorer { private boolean checkedRpts; // flag to only check for repetitions in first candidate doc private boolean hasMultiTermRpts; // private PhrasePositions[][] rptGroups; // in each group are PPs that repeats each other (i.e. same term), sorted by (query) offset - private PhrasePositions[] rptStack; // temporary stack for switching colliding repeating pps + private PhrasePositions[] rptStack; // temporary stack for switching colliding repeating pps + PhrasePositionsIterator ppIt = new PhrasePositionsIterator(); SloppyPhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings, - int slop, Similarity.SloppySimScorer docScorer) { + int slop, Similarity.SloppySimScorer docScorer) throws IOException { super(weight, postings, docScorer); this.slop = slop; this.numPostings = postings==null ? 0 : postings.length; @@ -71,32 +72,56 @@ final class SloppyPhraseScorer extends PhraseScorer { return 0.0f; } float freq = 0.0f; - PhrasePositions pp = pq.pop(); - int matchLength = end - pp.position; - int next = pq.top().position; - while (advancePP(pp)) { - if (hasRpts && !advanceRpts(pp)) { - break; // pps exhausted - } - if (pp.position > next) { // done minimizing current match-length - if (matchLength <= slop) { - freq += docScorer.computeSlopFactor(matchLength); // score match - } - pq.add(pp); - pp = pq.pop(); - next = pq.top().position; - matchLength = end - pp.position; - } else { - int matchLength2 = end - pp.position; - if (matchLength2 < matchLength) { - matchLength = matchLength2; + + while (ppIt.next()) { + freq += docScorer.computeSlopFactor(ppIt.matchlength); + } + + return freq; + } + + private class PhrasePositionsIterator { + + public int matchlength; + public int headPosition = -1; + public int headStartOffset = 1; + public int headEndOffset = -1; + private PhrasePositions currentHead; + + public boolean next() throws IOException { + if (pq.size() < numPostings) + return false; + currentHead = pq.pop(); + matchlength = end - currentHead.position; + headPosition = currentHead.position + currentHead.offset; + headStartOffset = currentHead.postings.startOffset(); + headEndOffset = currentHead.postings.endOffset(); + int next = pq.top().position; + while (advancePP(currentHead)) { + if (hasRpts && !advanceRpts(currentHead)) + break; + if (currentHead.position > next) { // done minimizing current match-length + pq.add(currentHead); + if (matchlength <= slop) { + return true; + } + // not a match - keep going + currentHead = pq.pop(); + next = pq.top().position; + matchlength = end - currentHead.position; + } + else { + int newmatchlength = end - currentHead.position; + if (newmatchlength < matchlength) + matchlength = newmatchlength; } + headPosition = currentHead.position + currentHead.offset; + headStartOffset = currentHead.postings.startOffset(); + headEndOffset = currentHead.postings.endOffset(); } + return matchlength <= slop; } - if (matchLength <= slop) { - freq += docScorer.computeSlopFactor(matchLength); // score match - } - return freq; + } /** advance a PhrasePosition and update 'end', return false if exhausted */ @@ -481,8 +506,80 @@ final class SloppyPhraseScorer extends PhraseScorer { @Override public PositionIntervalIterator positions(boolean needsPayloads, boolean needsOffsets) throws IOException { - // nocommit implement this (and get a beer before you do so!) - throw new UnsupportedOperationException(); + // nocommit - payloads? + reset(needsOffsets); + return new SloppyPhrasePositionIntervalIterator(this); + } + + private class SloppyPhrasePositionIntervalIterator extends PositionIntervalIterator { + + private PositionInterval interval = new PositionInterval(Integer.MIN_VALUE, Integer.MIN_VALUE, -1, -1); + private PositionInterval[] subintervals; + + public SloppyPhrasePositionIntervalIterator(Scorer scorer) { + super(scorer); + subintervals = new PositionInterval[numPostings]; + for (int i = 0; i < numPostings; i++) { + subintervals[i] = new PositionInterval(Integer.MIN_VALUE, Integer.MIN_VALUE, -1, -1); + } + } + + @Override + public int advanceTo(int docId) throws IOException { + int currentdoc = -1; + for (PhrasePositions pp=min,prev=null; prev!=max; pp=(prev=pp).next) { + currentdoc = pp.postings.advance(docId); + } + initPhrasePositions(); + return currentdoc; + } + + @Override + public PositionInterval next() throws IOException { + if (!ppIt.next()) + return null; + fillInterval(interval, ppIt, max); + fillInterval(subintervals[0], ppIt); + int i = 1; + for (PhrasePositions pp=min.next,prev=null; prev != max; pp=(prev=pp).next, i++) { + fillInterval(subintervals[i], pp); + } + return interval; + } + + @Override + public void collect() { + collector.collectComposite(scorer, interval, currentDoc); + int i = 0; + for (PhrasePositions pp=min,prev=null; prev != max; pp=(prev=pp).next, i++) { + collector.collectLeafPosition(scorer, subintervals[i], currentDoc); + } + } + + @Override + public PositionIntervalIterator[] subs(boolean inOrder) { + return EMPTY; + } + + private void fillInterval(PositionInterval i, PhrasePositionsIterator it) { + i.begin = it.headPosition; + i.end = it.headPosition; + i.offsetBegin = it.headStartOffset; + i.offsetEnd = it.headEndOffset; + } + + private void fillInterval(PositionInterval i, PhrasePositionsIterator it, PhrasePositions end) throws IOException { + i.begin = it.headPosition; + i.end = end.position + end.offset; + i.offsetBegin = it.headStartOffset; + i.offsetEnd = end.postings.endOffset(); + } + + private void fillInterval(PositionInterval i, PhrasePositions pp) throws IOException { + i.end = i.begin = pp.offset + pp.position; + i.offsetBegin = pp.postings.startOffset(); + i.offsetEnd = pp.postings.endOffset(); + } } // private void printQueue(PrintStream ps, PhrasePositions ext, String title) { diff --git a/lucene/core/src/java/org/apache/lucene/search/Sort.java b/lucene/core/src/java/org/apache/lucene/search/Sort.java index ac324db..bc17e99 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Sort.java +++ b/lucene/core/src/java/org/apache/lucene/search/Sort.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/SortField.java b/lucene/core/src/java/org/apache/lucene/search/SortField.java index 74d6b92..35e07c4 100644 --- a/lucene/core/src/java/org/apache/lucene/search/SortField.java +++ b/lucene/core/src/java/org/apache/lucene/search/SortField.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/TermCollectingRewrite.java b/lucene/core/src/java/org/apache/lucene/search/TermCollectingRewrite.java index 432286f..a237d07 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermCollectingRewrite.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermCollectingRewrite.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -25,11 +25,11 @@ import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.TermContext; abstract class TermCollectingRewrite extends MultiTermQuery.RewriteMethod { @@ -48,8 +48,7 @@ abstract class TermCollectingRewrite extends MultiTermQuery.Rew final void collectTerms(IndexReader reader, MultiTermQuery query, TermCollector collector) throws IOException { IndexReaderContext topReaderContext = reader.getTopReaderContext(); Comparator lastTermComp = null; - final AtomicReaderContext[] leaves = topReaderContext.leaves(); - for (AtomicReaderContext context : leaves) { + for (AtomicReaderContext context : topReaderContext.leaves()) { final Fields fields = context.reader().fields(); if (fields == null) { // reader has no fields diff --git a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java index ebc4458..278c19d 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,29 +17,21 @@ package org.apache.lucene.search; * limitations under the License. */ -import java.io.IOException; -import java.util.Set; - -import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.index.DocsAndPositionsEnum; -import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.index.AtomicReader; -import org.apache.lucene.index.IndexReaderContext; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermState; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.similarities.Similarity.ExactSimScorer; +import org.apache.lucene.index.*; import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.Similarity.ExactSimScorer; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.ReaderUtil; -import org.apache.lucene.util.TermContext; import org.apache.lucene.util.ToStringUtils; +import java.io.IOException; +import java.util.Set; + /** * A Query that matches documents containing a term. This may be combined with * other terms with a {@link BooleanQuery}. */ + public class TermQuery extends Query { private final Term term; private final int docFreq; @@ -94,7 +86,7 @@ public class TermQuery extends Query { } DocsEnum docs = termsEnum.docs(acceptDocs, null, true); if (docs != null) { - return new TermScorer(this, new TermDocsEnumFactory(termsEnum, docs, docs, acceptDocs), createDocScorer(context)); + return new TermScorer(this, docs, new TermDocsEnumFactory(termsEnum, acceptDocs), createDocScorer(context)); } else { // Index does not store freq info docs = termsEnum.docs(acceptDocs, null, false); @@ -248,43 +240,41 @@ public class TermQuery extends Query { } static class TermDocsEnumFactory { - private final TermsEnum termsEnum; - private final Bits liveDocs; - private final DocsEnum docs; - private final DocsEnum docsAndFreqs; - private final TermState state; - private BytesRef term; + protected final TermsEnum termsEnum; + protected final Bits liveDocs; + //private final DocsEnum docs; + //private final DocsEnum docsAndFreqs; + //private final TermState state; + protected BytesRef term; - TermDocsEnumFactory(TermsEnum termsEnum, DocsEnum docs, DocsEnum docsAndFreqs, Bits liveDocs) { - this(null, null, termsEnum, docs, docsAndFreqs, liveDocs); + TermDocsEnumFactory(TermsEnum termsEnum, Bits liveDocs) { + this(null, termsEnum, liveDocs); } - TermDocsEnumFactory(BytesRef term, TermState state, TermsEnum termsEnum, - DocsEnum docs, DocsEnum docsAndFreqs, Bits liveDocs) { + TermDocsEnumFactory(BytesRef term, TermsEnum termsEnum, Bits liveDocs) { this.termsEnum = termsEnum; this.liveDocs = liveDocs; - this.docs = docs; - this.docsAndFreqs = docsAndFreqs; - this.state = state; + //this.docs = docs; + //this.docsAndFreqs = docsAndFreqs; + //this.state = state; this.term = term; } + /* public DocsEnum docsEnum() throws IOException { return docs; } + */ public DocsAndPositionsEnum docsAndPositionsEnum(boolean offsets) throws IOException { - if (state != null) { + if (term != null) { assert term != null; - termsEnum.seekExact(term, state); + termsEnum.seekExact(term, false); } return termsEnum.docsAndPositions(liveDocs, null, offsets); } - - public DocsEnum docsAndFreqsEnum() throws IOException{ - return docsAndFreqs; - } + } } diff --git a/lucene/core/src/java/org/apache/lucene/search/TermRangeFilter.java b/lucene/core/src/java/org/apache/lucene/search/TermRangeFilter.java index 6b05122..c41e839 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermRangeFilter.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermRangeFilter.java @@ -2,7 +2,7 @@ package org.apache.lucene.search; import org.apache.lucene.util.BytesRef; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/TermRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermRangeQuery.java index c55a433..895dd00 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermRangeQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/TermRangeTermsEnum.java b/lucene/core/src/java/org/apache/lucene/search/TermRangeTermsEnum.java index ef4750b..c6b0a20 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermRangeTermsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermRangeTermsEnum.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -58,11 +58,9 @@ public class TermRangeTermsEnum extends FilteredTermsEnum { * If true, the lowerTerm is included in the range. * @param includeUpper * If true, the upperTerm is included in the range. - * - * @throws IOException */ public TermRangeTermsEnum(TermsEnum tenum, BytesRef lowerTerm, BytesRef upperTerm, - boolean includeLower, boolean includeUpper) throws IOException { + boolean includeLower, boolean includeUpper) { super(tenum); // do a little bit of normalization... diff --git a/lucene/core/src/java/org/apache/lucene/search/TermScorer.java b/lucene/core/src/java/org/apache/lucene/search/TermScorer.java index 16aa231..309c5b3 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermScorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermScorer.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -45,10 +45,11 @@ final class TermScorer extends Scorer { * The Similarity.ExactSimScorer implementation * to be used for score computations. */ - TermScorer(Weight weight, TermDocsEnumFactory factory, Similarity.ExactSimScorer docScorer) throws IOException { + + TermScorer(Weight weight, DocsEnum docsEnum, TermDocsEnumFactory factory, Similarity.ExactSimScorer docScorer) throws IOException { super(weight); this.docScorer = docScorer; - this.docsEnum = factory.docsAndFreqsEnum(); + this.docsEnum = docsEnum; this.factory = factory; } diff --git a/lucene/core/src/java/org/apache/lucene/search/TermStatistics.java b/lucene/core/src/java/org/apache/lucene/search/TermStatistics.java index 3c199a9..a17d50f 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermStatistics.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermStatistics.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/TimeLimitingCollector.java b/lucene/core/src/java/org/apache/lucene/search/TimeLimitingCollector.java index 0a56b86..1f8c472 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TimeLimitingCollector.java +++ b/lucene/core/src/java/org/apache/lucene/search/TimeLimitingCollector.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/TopDocs.java b/lucene/core/src/java/org/apache/lucene/search/TopDocs.java index e8af0f8..46e8b81 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopDocs.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopDocs.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/TopDocsCollector.java b/lucene/core/src/java/org/apache/lucene/search/TopDocsCollector.java index 75433ef..afbae97 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopDocsCollector.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopDocsCollector.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java index f14cabd..942c723 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -51,7 +51,7 @@ public abstract class TopFieldCollector extends TopDocsCollector { final FieldValueHitQueue queue; public OneComparatorNonScoringCollector(FieldValueHitQueue queue, - int numHits, boolean fillFields) throws IOException { + int numHits, boolean fillFields) { super(queue, numHits, fillFields); this.queue = queue; comparator = queue.getComparators()[0]; @@ -114,7 +114,7 @@ public abstract class TopFieldCollector extends TopDocsCollector { OneComparatorNonScoringCollector { public OutOfOrderOneComparatorNonScoringCollector(FieldValueHitQueue queue, - int numHits, boolean fillFields) throws IOException { + int numHits, boolean fillFields) { super(queue, numHits, fillFields); } @@ -161,7 +161,7 @@ public abstract class TopFieldCollector extends TopDocsCollector { Scorer scorer; public OneComparatorScoringNoMaxScoreCollector(FieldValueHitQueue queue, - int numHits, boolean fillFields) throws IOException { + int numHits, boolean fillFields) { super(queue, numHits, fillFields); } @@ -221,8 +221,7 @@ public abstract class TopFieldCollector extends TopDocsCollector { OneComparatorScoringNoMaxScoreCollector { public OutOfOrderOneComparatorScoringNoMaxScoreCollector( - FieldValueHitQueue queue, int numHits, boolean fillFields) - throws IOException { + FieldValueHitQueue queue, int numHits, boolean fillFields) { super(queue, numHits, fillFields); } @@ -275,7 +274,7 @@ public abstract class TopFieldCollector extends TopDocsCollector { Scorer scorer; public OneComparatorScoringMaxScoreCollector(FieldValueHitQueue queue, - int numHits, boolean fillFields) throws IOException { + int numHits, boolean fillFields) { super(queue, numHits, fillFields); // Must set maxScore to NEG_INF, or otherwise Math.max always returns NaN. maxScore = Float.NEGATIVE_INFINITY; @@ -335,7 +334,7 @@ public abstract class TopFieldCollector extends TopDocsCollector { OneComparatorScoringMaxScoreCollector { public OutOfOrderOneComparatorScoringMaxScoreCollector(FieldValueHitQueue queue, - int numHits, boolean fillFields) throws IOException { + int numHits, boolean fillFields) { super(queue, numHits, fillFields); } @@ -386,7 +385,7 @@ public abstract class TopFieldCollector extends TopDocsCollector { final int[] reverseMul; final FieldValueHitQueue queue; public MultiComparatorNonScoringCollector(FieldValueHitQueue queue, - int numHits, boolean fillFields) throws IOException { + int numHits, boolean fillFields) { super(queue, numHits, fillFields); this.queue = queue; comparators = queue.getComparators(); @@ -472,7 +471,7 @@ public abstract class TopFieldCollector extends TopDocsCollector { MultiComparatorNonScoringCollector { public OutOfOrderMultiComparatorNonScoringCollector(FieldValueHitQueue queue, - int numHits, boolean fillFields) throws IOException { + int numHits, boolean fillFields) { super(queue, numHits, fillFields); } @@ -541,7 +540,7 @@ public abstract class TopFieldCollector extends TopDocsCollector { Scorer scorer; public MultiComparatorScoringMaxScoreCollector(FieldValueHitQueue queue, - int numHits, boolean fillFields) throws IOException { + int numHits, boolean fillFields) { super(queue, numHits, fillFields); // Must set maxScore to NEG_INF, or otherwise Math.max always returns NaN. maxScore = Float.NEGATIVE_INFINITY; @@ -620,7 +619,7 @@ public abstract class TopFieldCollector extends TopDocsCollector { extends MultiComparatorScoringMaxScoreCollector { public OutOfOrderMultiComparatorScoringMaxScoreCollector(FieldValueHitQueue queue, - int numHits, boolean fillFields) throws IOException { + int numHits, boolean fillFields) { super(queue, numHits, fillFields); } @@ -693,7 +692,7 @@ public abstract class TopFieldCollector extends TopDocsCollector { Scorer scorer; public MultiComparatorScoringNoMaxScoreCollector(FieldValueHitQueue queue, - int numHits, boolean fillFields) throws IOException { + int numHits, boolean fillFields) { super(queue, numHits, fillFields); } @@ -771,8 +770,7 @@ public abstract class TopFieldCollector extends TopDocsCollector { extends MultiComparatorScoringNoMaxScoreCollector { public OutOfOrderMultiComparatorScoringNoMaxScoreCollector( - FieldValueHitQueue queue, int numHits, boolean fillFields) - throws IOException { + FieldValueHitQueue queue, int numHits, boolean fillFields) { super(queue, numHits, fillFields); } @@ -860,8 +858,7 @@ public abstract class TopFieldCollector extends TopDocsCollector { public PagingFieldCollector( FieldValueHitQueue queue, FieldDoc after, int numHits, boolean fillFields, - boolean trackDocScores, boolean trackMaxScore) - throws IOException { + boolean trackDocScores, boolean trackMaxScore) { super(queue, numHits, fillFields); this.queue = queue; this.trackDocScores = trackDocScores; @@ -981,7 +978,7 @@ public abstract class TopFieldCollector extends TopDocsCollector { } @Override - public void setScorer(Scorer scorer) throws IOException { + public void setScorer(Scorer scorer) { this.scorer = scorer; for (int i = 0; i < comparators.length; i++) { comparators[i].setScorer(scorer); diff --git a/lucene/core/src/java/org/apache/lucene/search/TopFieldDocs.java b/lucene/core/src/java/org/apache/lucene/search/TopFieldDocs.java index aa17421..42b3f84 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopFieldDocs.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopFieldDocs.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java index bdef2c3..9c684c5 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/TopTermsRewrite.java b/lucene/core/src/java/org/apache/lucene/search/TopTermsRewrite.java index e84b7e7..15205ae 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopTermsRewrite.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopTermsRewrite.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -25,11 +25,11 @@ import java.util.Comparator; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.TermContext; /** * Base rewrite method for collecting only the top terms @@ -75,7 +75,7 @@ public abstract class TopTermsRewrite extends TermCollectingRew private ScoreTerm st; @Override - public void setNextEnum(TermsEnum termsEnum) throws IOException { + public void setNextEnum(TermsEnum termsEnum) { this.termsEnum = termsEnum; this.termComp = termsEnum.getComparator(); @@ -89,7 +89,7 @@ public abstract class TopTermsRewrite extends TermCollectingRew // for assert: private BytesRef lastTerm; - private boolean compareToLastTerm(BytesRef t) throws IOException { + private boolean compareToLastTerm(BytesRef t) { if (lastTerm == null && t != null) { lastTerm = BytesRef.deepCopyOf(t); } else if (t == null) { diff --git a/lucene/core/src/java/org/apache/lucene/search/TotalHitCountCollector.java b/lucene/core/src/java/org/apache/lucene/search/TotalHitCountCollector.java index f8d3d83..1704d8b 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TotalHitCountCollector.java +++ b/lucene/core/src/java/org/apache/lucene/search/TotalHitCountCollector.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/Weight.java b/lucene/core/src/java/org/apache/lucene/search/Weight.java index aba9ac8..051d705 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Weight.java +++ b/lucene/core/src/java/org/apache/lucene/search/Weight.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java b/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java index b424246..1a041eb 100644 --- a/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java b/lucene/core/src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java index 31bbc88..1f5e8a3 100644 --- a/lucene/core/src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java +++ b/lucene/core/src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java @@ -1,7 +1,7 @@ package org.apache.lucene.search.payloads; import org.apache.lucene.search.Explanation; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java b/lucene/core/src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java index 34ea95e..c113fbc 100644 --- a/lucene/core/src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java +++ b/lucene/core/src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java @@ -1,7 +1,7 @@ package org.apache.lucene.search.payloads; import org.apache.lucene.search.Explanation; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java b/lucene/core/src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java index 24ee1bc..40cbbb0 100644 --- a/lucene/core/src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java +++ b/lucene/core/src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java @@ -1,7 +1,7 @@ package org.apache.lucene.search.payloads; import org.apache.lucene.search.Explanation; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadFunction.java b/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadFunction.java index b6c6035..cb34a0b 100644 --- a/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadFunction.java +++ b/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadFunction.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.payloads; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java b/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java index f892d5f..fe1433b 100644 --- a/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.payloads; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java b/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java index 28e2c08..6bae19b 100644 --- a/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java +++ b/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.payloads; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -30,6 +30,7 @@ import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.DisjunctionMaxQuery; @@ -43,7 +44,6 @@ import org.apache.lucene.search.spans.SpanOrQuery; import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.search.spans.Spans; -import org.apache.lucene.util.TermContext; /** * Experimental class to get set of payloads for most standard Lucene queries. @@ -184,8 +184,7 @@ public class PayloadSpanUtil { for (Term term : terms) { termContexts.put(term, TermContext.build(context, term, true)); } - final AtomicReaderContext[] leaves = context.leaves(); - for (AtomicReaderContext atomicReaderContext : leaves) { + for (AtomicReaderContext atomicReaderContext : context.leaves()) { final Spans spans = query.getSpans(atomicReaderContext, atomicReaderContext.reader().getLiveDocs(), termContexts); while (spans.next() == true) { if (spans.isPayloadAvailable()) { diff --git a/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java b/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java index 1ca1149..54e4ba1 100644 --- a/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.payloads; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/AfterEffect.java b/lucene/core/src/java/org/apache/lucene/search/similarities/AfterEffect.java index 4610f2b..0566947 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/AfterEffect.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/AfterEffect.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/AfterEffectB.java b/lucene/core/src/java/org/apache/lucene/search/similarities/AfterEffectB.java index 49ddbc0..149cace 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/AfterEffectB.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/AfterEffectB.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/AfterEffectL.java b/lucene/core/src/java/org/apache/lucene/search/similarities/AfterEffectL.java index 5479830..8fd4e0d 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/AfterEffectL.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/AfterEffectL.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java index a0a4a9f..f9c51ed 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -73,8 +73,8 @@ public class BM25Similarity extends Similarity { } /** The default implementation computes the average as sumTotalTermFreq / maxDoc, - * or returns 1 if the index does not store sumTotalTermFreq (Lucene 3.x indexes - * or any field that omits frequency information). */ + * or returns 1 if the index does not store sumTotalTermFreq: + * any field that omits frequency information). */ protected float avgFieldLength(CollectionStatistics collectionStats) { final long sumTotalTermFreq = collectionStats.sumTotalTermFreq(); if (sumTotalTermFreq <= 0) { diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModel.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModel.java index ff4d12d..13ef257 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModel.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModel.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelBE.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelBE.java index a8b218e..c1513d6 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelBE.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelBE.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelD.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelD.java index b32356f..982afe4 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelD.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelD.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelG.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelG.java index 3c3a692..674e018 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelG.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelG.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelIF.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelIF.java index fda7471..f97be3e 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelIF.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelIF.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelIn.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelIn.java index 3f8afaf..7bd3ab0 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelIn.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelIn.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelIne.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelIne.java index a2262b6..1377680 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelIne.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelIne.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelP.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelP.java index d78f7ad..e981764 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelP.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BasicModelP.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BasicStats.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BasicStats.java index e438c26..848f64c 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/BasicStats.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BasicStats.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java index bd2010e..ff45f21 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java index 3d8ef7e..79514a5 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java @@ -4,7 +4,7 @@ import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.Norm; import org.apache.lucene.util.BytesRef; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/Distribution.java b/lucene/core/src/java/org/apache/lucene/search/similarities/Distribution.java index c31164e..98d73ed 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/Distribution.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/Distribution.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/DistributionLL.java b/lucene/core/src/java/org/apache/lucene/search/similarities/DistributionLL.java index b9e0913..da4e817 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/DistributionLL.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/DistributionLL.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/DistributionSPL.java b/lucene/core/src/java/org/apache/lucene/search/similarities/DistributionSPL.java index 13baf52..54cf065 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/DistributionSPL.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/DistributionSPL.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/IBSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/IBSimilarity.java index aec8975..bdbb499 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/IBSimilarity.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/IBSimilarity.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/LMDirichletSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/LMDirichletSimilarity.java index ad7e309..aac9f99 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/LMDirichletSimilarity.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/LMDirichletSimilarity.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java index 910e769..f6a4733 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/LMSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/LMSimilarity.java index db7607e..52f18f9 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/LMSimilarity.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/LMSimilarity.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/Lambda.java b/lucene/core/src/java/org/apache/lucene/search/similarities/Lambda.java index 64b8c34..cefdb2c 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/Lambda.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/Lambda.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/LambdaDF.java b/lucene/core/src/java/org/apache/lucene/search/similarities/LambdaDF.java index 73ae600..862a22b 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/LambdaDF.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/LambdaDF.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/LambdaTTF.java b/lucene/core/src/java/org/apache/lucene/search/similarities/LambdaTTF.java index 7024095..58bc0ba 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/LambdaTTF.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/LambdaTTF.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java index b15e87c..67c98ed 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/Normalization.java b/lucene/core/src/java/org/apache/lucene/search/similarities/Normalization.java index f635baa..29bd4c8 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/Normalization.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/Normalization.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/NormalizationH1.java b/lucene/core/src/java/org/apache/lucene/search/similarities/NormalizationH1.java index c667d78..29c3813 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/NormalizationH1.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/NormalizationH1.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/NormalizationH2.java b/lucene/core/src/java/org/apache/lucene/search/similarities/NormalizationH2.java index 12890df..da5221e 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/NormalizationH2.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/NormalizationH2.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/NormalizationH3.java b/lucene/core/src/java/org/apache/lucene/search/similarities/NormalizationH3.java index dc5cd35..903bdf3 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/NormalizationH3.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/NormalizationH3.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/NormalizationZ.java b/lucene/core/src/java/org/apache/lucene/search/similarities/NormalizationZ.java index 224dea8..61ebdf1 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/NormalizationZ.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/NormalizationZ.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java b/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java index c2d036f..117787a 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java index 05a28f9..198ef8c 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -99,7 +99,7 @@ import org.apache.lucene.util.SmallFloat; // javadoc * *

    * - * When {@link IndexSearcher#explain(Query, int)} is called, queries consult the Similarity's DocScorer for an + * When {@link IndexSearcher#explain(org.apache.lucene.search.Query, int)} is called, queries consult the Similarity's DocScorer for an * explanation of how it computed its score. The query passes in a the document id and an explanation of how the frequency * was computed. * diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java b/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java index 8efb66a..798a6ef 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java index a2645e2..cc6a1a5 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -611,7 +611,6 @@ public abstract class TFIDFSimilarity extends Similarity { * @param termStats term-level statistics for the term * @return an Explain object that includes both an idf score factor and an explanation for the term. - * @throws IOException */ public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats) { final long df = termStats.docFreq(); @@ -632,7 +631,6 @@ public abstract class TFIDFSimilarity extends Similarity { * @return an Explain object that includes both an idf * score factor for the phrase and an explanation * for each term. - * @throws IOException */ public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats[]) { final long max = collectionStats.maxDoc(); diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java index eb15b34..5f99458 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -24,11 +24,11 @@ import java.util.Set; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; import org.apache.lucene.search.Query; import org.apache.lucene.search.Weight; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.TermContext; import org.apache.lucene.util.ToStringUtils; /** diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/MockSpanQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/MockSpanQuery.java index a9ed375..6ccfb27 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/MockSpanQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/MockSpanQuery.java @@ -22,7 +22,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.*; import org.apache.lucene.search.positions.PositionIntervalIterator.PositionIntervalFilter; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.TermContext; +import org.apache.lucene.index.TermContext; import java.io.IOException; import java.util.Collection; diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java b/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java index 9907375..a27c0cc 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,9 +19,9 @@ package org.apache.lucene.search.spans; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.TermContext; import java.io.IOException; import java.util.ArrayList; diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java b/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java index 0ec6fb4..f59e4c0 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,9 +19,9 @@ package org.apache.lucene.search.spans; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; import org.apache.lucene.util.Bits; import org.apache.lucene.util.PriorityQueue; -import org.apache.lucene.util.TermContext; import java.io.IOException; import java.util.ArrayList; @@ -285,7 +285,7 @@ public class NearSpansUnordered extends Spans { } } - private void addToList(SpansCell cell) throws IOException { + private void addToList(SpansCell cell) { if (last != null) { // add next to end of list last.next = cell; } else @@ -301,7 +301,7 @@ public class NearSpansUnordered extends Spans { last.next = null; } - private void queueToList() throws IOException { + private void queueToList() { last = first = null; while (queue.top() != null) { addToList(queue.pop()); diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanFirstQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanFirstQuery.java index 01e582c..62f98a2 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanFirstQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanFirstQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java index 0a69a6d..22bf5a2 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,13 +23,13 @@ import java.util.Map; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TopTermsRewrite; import org.apache.lucene.search.ScoringRewrite; import org.apache.lucene.search.BooleanClause.Occur; // javadocs only import org.apache.lucene.util.Bits; -import org.apache.lucene.util.TermContext; /** * Wraps any {@link MultiTermQuery} as a {@link SpanQuery}, diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearPayloadCheckQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearPayloadCheckQuery.java index 4ec8882..ff761e8 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearPayloadCheckQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearPayloadCheckQuery.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java index 43d40ba..68aff02 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -30,9 +30,9 @@ import java.util.Set; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; import org.apache.lucene.search.Query; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.TermContext; import org.apache.lucene.util.ToStringUtils; /** Matches spans which are near one another. One can specify slop, the diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java index f1396b3..0258d13 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,9 +20,9 @@ package org.apache.lucene.search.spans; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; import org.apache.lucene.search.Query; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.TermContext; import org.apache.lucene.util.ToStringUtils; import java.io.IOException; diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java index fc8a130..3f00fc5 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -29,9 +29,9 @@ import java.util.Set; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; import org.apache.lucene.util.Bits; import org.apache.lucene.util.PriorityQueue; -import org.apache.lucene.util.TermContext; import org.apache.lucene.util.ToStringUtils; import org.apache.lucene.search.Query; diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanPayloadCheckQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanPayloadCheckQuery.java index b7fa763..9abd62e 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanPayloadCheckQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanPayloadCheckQuery.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java index d3a1b69..052f5a6 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,9 +20,9 @@ package org.apache.lucene.search.spans; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; import org.apache.lucene.search.Query; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.TermContext; import java.io.IOException; import java.util.ArrayList; diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionRangeQuery.java index ad9e01b..f588d28 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionRangeQuery.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanQuery.java index 3164251..3767279 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,11 +22,11 @@ import java.util.Map; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; import org.apache.lucene.search.Query; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Weight; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.TermContext; /** Base class for span-based queries. */ public abstract class SpanQuery extends Query { diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java index 30b304b..640cea5 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java index ccb0565..975d69c 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,11 +17,11 @@ package org.apache.lucene.search.spans; * limitations under the License. */ -import java.util.Set; - import org.apache.lucene.index.Term; import org.apache.lucene.search.TermQuery; +import java.util.Set; + /** Matches spans containing a term. */ public class SpanTermQuery extends MockSpanQuery { protected Term term; diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java index 3f66603..7292e90 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,11 +20,11 @@ package org.apache.lucene.search.spans; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; import org.apache.lucene.search.*; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.search.similarities.Similarity.SloppySimScorer; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.TermContext; import java.io.IOException; import java.util.HashMap; diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java b/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java index 1462f60..d0af3c4 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java b/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java new file mode 100644 index 0000000..11348fb --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java @@ -0,0 +1,170 @@ +package org.apache.lucene.search.spans; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import org.apache.lucene.index.Term; +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BytesRef; + +import java.io.IOException; +import java.util.Collections; +import java.util.Collection; + +/** + * Expert: + * Public for extension only + */ +public class TermSpans extends Spans { + protected final DocsAndPositionsEnum postings; + protected final Term term; + protected int doc; + protected int freq; + protected int count; + protected int position; + + public TermSpans(DocsAndPositionsEnum postings, Term term) { + this.postings = postings; + this.term = term; + doc = -1; + } + + // only for EmptyTermSpans (below) + TermSpans() { + term = null; + postings = null; + } + + @Override + public boolean next() throws IOException { + if (count == freq) { + if (postings == null) { + return false; + } + doc = postings.nextDoc(); + if (doc == DocIdSetIterator.NO_MORE_DOCS) { + return false; + } + freq = postings.freq(); + count = 0; + } + position = postings.nextPosition(); + count++; + return true; + } + + @Override + public boolean skipTo(int target) throws IOException { + doc = postings.advance(target); + if (doc == DocIdSetIterator.NO_MORE_DOCS) { + return false; + } + + freq = postings.freq(); + count = 0; + position = postings.nextPosition(); + count++; + + return true; + } + + @Override + public int doc() { + return doc; + } + + @Override + public int start() { + return position; + } + + @Override + public int end() { + return position + 1; + } + + // TODO: Remove warning after API has been finalized + @Override + public Collection getPayload() throws IOException { + final BytesRef payload = postings.getPayload(); + final byte[] bytes; + if (payload != null) { + bytes = new byte[payload.length]; + System.arraycopy(payload.bytes, payload.offset, bytes, 0, payload.length); + } else { + bytes = null; + } + return Collections.singletonList(bytes); + } + + // TODO: Remove warning after API has been finalized + @Override + public boolean isPayloadAvailable() { + return postings.hasPayload(); + } + + @Override + public String toString() { + return "spans(" + term.toString() + ")@" + + (doc == -1 ? "START" : (doc == Integer.MAX_VALUE) ? "END" : doc + "-" + position); + } + + public DocsAndPositionsEnum getPostings() { + return postings; + } + + private static final class EmptyTermSpans extends TermSpans { + + @Override + public boolean next() { + return false; + } + + @Override + public boolean skipTo(int target) { + return false; + } + + @Override + public int doc() { + return DocIdSetIterator.NO_MORE_DOCS; + } + + @Override + public int start() { + return -1; + } + + @Override + public int end() { + return -1; + } + + @Override + public Collection getPayload() { + return null; + } + + @Override + public boolean isPayloadAvailable() { + return false; + } + } + + public static final TermSpans EMPTY_TERM_SPANS = new EmptyTermSpans(); +} diff --git a/lucene/core/src/java/org/apache/lucene/store/AlreadyClosedException.java b/lucene/core/src/java/org/apache/lucene/store/AlreadyClosedException.java index 435b9dd..f1a20cc 100644 --- a/lucene/core/src/java/org/apache/lucene/store/AlreadyClosedException.java +++ b/lucene/core/src/java/org/apache/lucene/store/AlreadyClosedException.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java b/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java index 1ff8f34..812def0 100644 --- a/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java +++ b/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/BufferedIndexOutput.java b/lucene/core/src/java/org/apache/lucene/store/BufferedIndexOutput.java index 1dfbcec..bbc47fe 100644 --- a/lucene/core/src/java/org/apache/lucene/store/BufferedIndexOutput.java +++ b/lucene/core/src/java/org/apache/lucene/store/BufferedIndexOutput.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteArrayDataInput.java b/lucene/core/src/java/org/apache/lucene/store/ByteArrayDataInput.java index d19c3bb..e46f89a 100644 --- a/lucene/core/src/java/org/apache/lucene/store/ByteArrayDataInput.java +++ b/lucene/core/src/java/org/apache/lucene/store/ByteArrayDataInput.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteArrayDataOutput.java b/lucene/core/src/java/org/apache/lucene/store/ByteArrayDataOutput.java index 863470b..4a6599f 100644 --- a/lucene/core/src/java/org/apache/lucene/store/ByteArrayDataOutput.java +++ b/lucene/core/src/java/org/apache/lucene/store/ByteArrayDataOutput.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexInput.java b/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexInput.java index 313d34c..224ca0d 100644 --- a/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexInput.java +++ b/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexInput.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexOutput.java b/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexOutput.java index eb4f742..91db830 100644 --- a/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexOutput.java +++ b/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexOutput.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java b/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java index f48ebad..81c3f31 100644 --- a/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,9 +18,10 @@ package org.apache.lucene.store; */ import org.apache.lucene.codecs.Codec; // javadocs +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.LiveDocsFormat; // javadocs -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.store.DataOutput; // javadocs import org.apache.lucene.util.IOUtils; @@ -51,15 +52,22 @@ import java.io.IOException; *

*

Description:

*
    - *
  • Compound (.cfs) --> FileData FileCount
    • - *
  • Compound Entry Table (.cfe) --> Version, FileCount, <FileName, + *
  • Compound (.cfs) --> Header, FileData FileCount
    • + *
  • Compound Entry Table (.cfe) --> Header, FileCount, <FileName, * DataOffset, DataLength> FileCount
    • - *
  • Version --> {@link DataOutput#writeInt Int32}
    • + *
  • Header --> {@link CodecUtil#writeHeader CodecHeader}
    • *
  • FileCount --> {@link DataOutput#writeVInt VInt}
    • *
  • DataOffset,DataLength --> {@link DataOutput#writeLong UInt64}
    • *
  • FileName --> {@link DataOutput#writeString String}
    • *
  • FileData --> raw file data
    • *
+ *

Notes:

+ *
    + *
  • FileCount indicates how many files are contained in this compound file. + * The entry table that follows has that many entries. + *
  • Each directory entry contains a long pointer to the start of this file's data + * section, the files length, and a String with that file's name. + *
* * @lucene.experimental */ @@ -114,105 +122,43 @@ public final class CompoundFileDirectory extends Directory { /** Helper method that reads CFS entries from an input stream */ private static final Map readEntries( IndexInputSlicer handle, Directory dir, String name) throws IOException { - // read the first VInt. If it is negative, it's the version number - // otherwise it's the count (pre-3.1 indexes) - final IndexInput stream = handle.openFullSlice(); - final Map mapping; - boolean success = false; + IOException priorE = null; + IndexInput stream = null, entriesStream = null; try { - final int firstInt = stream.readVInt(); - if (firstInt == CompoundFileWriter.FORMAT_CURRENT) { - IndexInput input = null; - try { - final String entriesFileName = IndexFileNames.segmentFileName( - IndexFileNames.stripExtension(name), "", - IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION); - input = dir.openInput(entriesFileName, IOContext.READONCE); - final int readInt = input.readInt(); // unused right now - assert readInt == CompoundFileWriter.ENTRY_FORMAT_CURRENT; - final int numEntries = input.readVInt(); - mapping = new HashMap( - numEntries); - for (int i = 0; i < numEntries; i++) { - final FileEntry fileEntry = new FileEntry(); - final String id = input.readString(); - assert !mapping.containsKey(id): "id=" + id + " was written multiple times in the CFS"; - mapping.put(id, fileEntry); - fileEntry.offset = input.readLong(); - fileEntry.length = input.readLong(); - } - return mapping; - } finally { - IOUtils.close(input); + stream = handle.openFullSlice(); + final int firstInt = stream.readInt(); + // NOTE: as long as we want to throw indexformattooold (vs corruptindexexception), we need + // to read the magic ourselves. See SegmentInfos which also has this. + if (firstInt == CodecUtil.CODEC_MAGIC) { + CodecUtil.checkHeaderNoMagic(stream, CompoundFileWriter.DATA_CODEC, + CompoundFileWriter.VERSION_START, CompoundFileWriter.VERSION_START); + final String entriesFileName = IndexFileNames.segmentFileName( + IndexFileNames.stripExtension(name), "", + IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION); + entriesStream = dir.openInput(entriesFileName, IOContext.READONCE); + CodecUtil.checkHeader(entriesStream, CompoundFileWriter.ENTRY_CODEC, CompoundFileWriter.VERSION_START, CompoundFileWriter.VERSION_START); + final int numEntries = entriesStream.readVInt(); + final Map mapping = new HashMap(numEntries); + for (int i = 0; i < numEntries; i++) { + final FileEntry fileEntry = new FileEntry(); + final String id = entriesStream.readString(); + assert !mapping.containsKey(id): "id=" + id + " was written multiple times in the CFS"; + mapping.put(id, fileEntry); + fileEntry.offset = entriesStream.readLong(); + fileEntry.length = entriesStream.readLong(); } + return mapping; } else { - // TODO remove once 3.x is not supported anymore - mapping = readLegacyEntries(stream, firstInt); + throw new IndexFormatTooOldException(stream, firstInt, + CodecUtil.CODEC_MAGIC, CodecUtil.CODEC_MAGIC); } - success = true; - return mapping; + } catch (IOException ioe) { + priorE = ioe; } finally { - if (success) { - IOUtils.close(stream); - } else { - IOUtils.closeWhileHandlingException(stream); - } - } - } - - private static Map readLegacyEntries(IndexInput stream, - int firstInt) throws CorruptIndexException, IOException { - final Map entries = new HashMap(); - final int count; - final boolean stripSegmentName; - if (firstInt < CompoundFileWriter.FORMAT_PRE_VERSION) { - if (firstInt < CompoundFileWriter.FORMAT_CURRENT) { - throw new CorruptIndexException("Incompatible format version: " - + firstInt + " expected " + CompoundFileWriter.FORMAT_CURRENT + " (resource: " + stream + ")"); - } - // It's a post-3.1 index, read the count. - count = stream.readVInt(); - stripSegmentName = false; - } else { - count = firstInt; - stripSegmentName = true; - } - - // read the directory and init files - long streamLength = stream.length(); - FileEntry entry = null; - for (int i=0; i streamLength) { - throw new CorruptIndexException("Invalid CFS entry offset: " + offset + " (resource: " + stream + ")"); - } - String id = stream.readString(); - - if (stripSegmentName) { - // Fix the id to not include the segment names. This is relevant for - // pre-3.1 indexes. - id = IndexFileNames.stripSegmentName(id); - } - - if (entry != null) { - // set length of the previous entry - entry.length = offset - entry.offset; - } - - entry = new FileEntry(); - entry.offset = offset; - - assert !entries.containsKey(id); - - entries.put(id, entry); - } - - // set the length of the final entry - if (entry != null) { - entry.length = streamLength - entry.offset; + IOUtils.closeWhileHandlingException(priorE, stream, entriesStream); } - - return entries; + // this is needed until Java 7's real try-with-resources: + throw new AssertionError("impossible to get here"); } public Directory getDirectory() { @@ -260,7 +206,7 @@ public final class CompoundFileDirectory extends Directory { } else { res = entries.keySet().toArray(new String[entries.size()]); // Add the segment name - String seg = fileName.substring(0, fileName.indexOf('.')); + String seg = IndexFileNames.parseSegmentName(fileName); for (int i = 0; i < res.length; i++) { res[i] = seg + res[i]; } @@ -312,7 +258,7 @@ public final class CompoundFileDirectory extends Directory { } @Override - public void sync(Collection names) throws IOException { + public void sync(Collection names) { throw new UnsupportedOperationException(); } @@ -335,7 +281,7 @@ public final class CompoundFileDirectory extends Directory { } return new IndexInputSlicer() { @Override - public void close() throws IOException { + public void close() { } @Override diff --git a/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java b/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java index a5e3d1c..e5d32a5 100644 --- a/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java +++ b/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -29,33 +29,14 @@ import java.util.Queue; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.MergePolicy.MergeAbortedException; import org.apache.lucene.util.IOUtils; /** * Combines multiple files into a single compound file. * - * The file format data file:
- *
    - *
  • VInt Version
    • - *
  • {File Data} fileCount entries with the raw data of the corresponding file - *
    • - *
      - * File format entry table:
      - *
        - *
      • int Version
        • - *
      • VInt fileCount - number of entries with the following structure:
        • - *
          - *
        • String fileName
          • - *
        • long dataOffset
          • - *
        • long dataLength
          • - *
        - *
      The fileCount integer indicates how many files are contained in - * this compound file. The entry table that follows has that many entries. Each - * directory entry contains a long pointer to the start of this file's data - * section, the files length, and a String with that file's name. - * + * @see CompoundFileDirectory * @lucene.internal */ final class CompoundFileWriter implements Closeable{ @@ -70,20 +51,13 @@ final class CompoundFileWriter implements Closeable{ Directory dir; } - // Before versioning started. - static final int FORMAT_PRE_VERSION = 0; - - // Segment name is not written in the file names. - static final int FORMAT_NO_SEGMENT_PREFIX = -1; - static final int FORMAT_APPEND_FILES = -2; + // versioning for the .cfs file + static final String DATA_CODEC = "CompoundFileWriterData"; + static final int VERSION_START = 0; + static final int VERSION_CURRENT = VERSION_START; - static final int ENTRY_FORMAT_CURRENT = -1; - - // NOTE: if you introduce a new format, make it 1 lower - // than the current one, and always change this if you - // switch to a new format! - /** @lucene.internal */ - static final int FORMAT_CURRENT = FORMAT_APPEND_FILES; + // versioning for the .cfe file + static final String ENTRY_CODEC = "CompoundFileWriterEntries"; private final Directory directory; private final Map entries = new HashMap(); @@ -103,7 +77,7 @@ final class CompoundFileWriter implements Closeable{ * @throws NullPointerException * if dir or name is null */ - CompoundFileWriter(Directory dir, String name) throws IOException { + CompoundFileWriter(Directory dir, String name) { if (dir == null) throw new NullPointerException("directory cannot be null"); if (name == null) @@ -121,7 +95,7 @@ final class CompoundFileWriter implements Closeable{ boolean success = false; try { dataOut = directory.createOutput(dataFileName, IOContext.DEFAULT); - dataOut.writeVInt(FORMAT_CURRENT); + CodecUtil.writeHeader(dataOut, DATA_CODEC, VERSION_CURRENT); success = true; } finally { if (!success) { @@ -199,7 +173,7 @@ final class CompoundFileWriter implements Closeable{ * output stream. */ private final long copyFileEntry(IndexOutput dataOut, FileEntry fileEntry) - throws IOException, MergeAbortedException { + throws IOException { final IndexInput is = fileEntry.dir.openInput(fileEntry.file, IOContext.READONCE); boolean success = false; try { @@ -228,7 +202,7 @@ final class CompoundFileWriter implements Closeable{ protected void writeEntryTable(Collection entries, IndexOutput entryOut) throws IOException { - entryOut.writeInt(ENTRY_FORMAT_CURRENT); + CodecUtil.writeHeader(entryOut, ENTRY_CODEC, VERSION_CURRENT); entryOut.writeVInt(entries.size()); for (FileEntry fe : entries) { entryOut.writeString(IndexFileNames.stripSegmentName(fe.file)); diff --git a/lucene/core/src/java/org/apache/lucene/store/DataInput.java b/lucene/core/src/java/org/apache/lucene/store/DataInput.java index 8e2c09e..62d19b3 100644 --- a/lucene/core/src/java/org/apache/lucene/store/DataInput.java +++ b/lucene/core/src/java/org/apache/lucene/store/DataInput.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -204,7 +204,7 @@ public abstract class DataInput implements Cloneable { } /** Reads a Map<String,String> previously written - * with {@link DataOutput#writeStringStringMap}. */ + * with {@link DataOutput#writeStringStringMap(Map)}. */ public Map readStringStringMap() throws IOException { final Map map = new HashMap(); final int count = readInt(); @@ -218,7 +218,7 @@ public abstract class DataInput implements Cloneable { } /** Reads a Set<String> previously written - * with {@link DataOutput#writeStringSet}. */ + * with {@link DataOutput#writeStringSet(Set)}. */ public Set readStringSet() throws IOException { final Set set = new HashSet(); final int count = readInt(); diff --git a/lucene/core/src/java/org/apache/lucene/store/DataOutput.java b/lucene/core/src/java/org/apache/lucene/store/DataOutput.java index 2d92964..6709d70 100644 --- a/lucene/core/src/java/org/apache/lucene/store/DataOutput.java +++ b/lucene/core/src/java/org/apache/lucene/store/DataOutput.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/Directory.java b/lucene/core/src/java/org/apache/lucene/store/Directory.java index 420c7ea..622ebdd 100644 --- a/lucene/core/src/java/org/apache/lucene/store/Directory.java +++ b/lucene/core/src/java/org/apache/lucene/store/Directory.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -230,7 +230,7 @@ public abstract class Directory implements Closeable { base.close(); } @Override - public IndexInput openFullSlice() throws IOException { + public IndexInput openFullSlice() { return (IndexInput) base.clone(); } }; diff --git a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java index c994426..adf3f70 100644 --- a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/FSLockFactory.java b/lucene/core/src/java/org/apache/lucene/store/FSLockFactory.java index 1bca363..62048fd 100644 --- a/lucene/core/src/java/org/apache/lucene/store/FSLockFactory.java +++ b/lucene/core/src/java/org/apache/lucene/store/FSLockFactory.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java index 201d0ce..99bb98b 100644 --- a/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/FlushInfo.java b/lucene/core/src/java/org/apache/lucene/store/FlushInfo.java index d2be85c..100c610 100644 --- a/lucene/core/src/java/org/apache/lucene/store/FlushInfo.java +++ b/lucene/core/src/java/org/apache/lucene/store/FlushInfo.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/IOContext.java b/lucene/core/src/java/org/apache/lucene/store/IOContext.java index 992ff8c..b9c9992 100644 --- a/lucene/core/src/java/org/apache/lucene/store/IOContext.java +++ b/lucene/core/src/java/org/apache/lucene/store/IOContext.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/IndexInput.java b/lucene/core/src/java/org/apache/lucene/store/IndexInput.java index 973ee5a..3f3c609 100644 --- a/lucene/core/src/java/org/apache/lucene/store/IndexInput.java +++ b/lucene/core/src/java/org/apache/lucene/store/IndexInput.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java b/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java index 549e196..2c9e7c1 100644 --- a/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java +++ b/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/InputStreamDataInput.java b/lucene/core/src/java/org/apache/lucene/store/InputStreamDataInput.java index 7516065..79003df 100644 --- a/lucene/core/src/java/org/apache/lucene/store/InputStreamDataInput.java +++ b/lucene/core/src/java/org/apache/lucene/store/InputStreamDataInput.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/Lock.java b/lucene/core/src/java/org/apache/lucene/store/Lock.java index f56dd1d..e460897 100644 --- a/lucene/core/src/java/org/apache/lucene/store/Lock.java +++ b/lucene/core/src/java/org/apache/lucene/store/Lock.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -67,7 +67,7 @@ public abstract class Lock { * out of bounds * @throws IOException if obtain() throws IOException */ - public boolean obtain(long lockWaitTimeout) throws LockObtainFailedException, IOException { + public boolean obtain(long lockWaitTimeout) throws IOException { failureReason = null; boolean locked = obtain(); if (lockWaitTimeout < 0 && lockWaitTimeout != LOCK_OBTAIN_WAIT_FOREVER) @@ -128,7 +128,7 @@ public abstract class Lock { * be obtained * @throws IOException if {@link Lock#obtain} throws IOException */ - public Object run() throws LockObtainFailedException, IOException { + public Object run() throws IOException { boolean locked = false; try { locked = lock.obtain(lockWaitTimeout); diff --git a/lucene/core/src/java/org/apache/lucene/store/LockFactory.java b/lucene/core/src/java/org/apache/lucene/store/LockFactory.java index 22ea6d4..deda3c0 100755 --- a/lucene/core/src/java/org/apache/lucene/store/LockFactory.java +++ b/lucene/core/src/java/org/apache/lucene/store/LockFactory.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/LockObtainFailedException.java b/lucene/core/src/java/org/apache/lucene/store/LockObtainFailedException.java index 0c61caf..ed716f1 100644 --- a/lucene/core/src/java/org/apache/lucene/store/LockObtainFailedException.java +++ b/lucene/core/src/java/org/apache/lucene/store/LockObtainFailedException.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/LockReleaseFailedException.java b/lucene/core/src/java/org/apache/lucene/store/LockReleaseFailedException.java index 8e7d626..a5dd380 100644 --- a/lucene/core/src/java/org/apache/lucene/store/LockReleaseFailedException.java +++ b/lucene/core/src/java/org/apache/lucene/store/LockReleaseFailedException.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/LockStressTest.java b/lucene/core/src/java/org/apache/lucene/store/LockStressTest.java index 13b7b92..d4f2c86 100644 --- a/lucene/core/src/java/org/apache/lucene/store/LockStressTest.java +++ b/lucene/core/src/java/org/apache/lucene/store/LockStressTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/LockVerifyServer.java b/lucene/core/src/java/org/apache/lucene/store/LockVerifyServer.java index acd0259..f090cae 100644 --- a/lucene/core/src/java/org/apache/lucene/store/LockVerifyServer.java +++ b/lucene/core/src/java/org/apache/lucene/store/LockVerifyServer.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java index f985b90..d4e39eb 100644 --- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -27,9 +27,7 @@ import java.nio.channels.ClosedChannelException; // javadoc @link import java.nio.channels.FileChannel; import java.nio.channels.FileChannel.MapMode; -import java.util.Collections; -import java.util.Set; -import java.util.WeakHashMap; +import java.util.Iterator; import java.security.AccessController; import java.security.PrivilegedExceptionAction; @@ -37,6 +35,7 @@ import java.security.PrivilegedActionException; import java.lang.reflect.Method; import org.apache.lucene.util.Constants; +import org.apache.lucene.util.WeakIdentityMap; /** File-based {@link Directory} implementation that uses * mmap for reading, and {@link @@ -261,7 +260,7 @@ public class MMapDirectory extends FSDirectory { private ByteBuffer curBuf; // redundant for speed: buffers[curBufIndex] private boolean isClone = false; - private final Set clones = Collections.newSetFromMap(new WeakHashMap()); + private final WeakIdentityMap clones = WeakIdentityMap.newConcurrentHashMap(); MMapIndexInput(String resourceDescription, RandomAccessFile raf, long offset, long length, int chunkSizePower) throws IOException { super(resourceDescription); @@ -431,9 +430,7 @@ public class MMapDirectory extends FSDirectory { } // register the new clone in our clone list to clean it up on closing: - synchronized(this.clones) { - this.clones.add(clone); - } + this.clones.put(clone, Boolean.TRUE); return clone; } @@ -449,35 +446,25 @@ public class MMapDirectory extends FSDirectory { try { if (isClone || buffers == null) return; + // make local copy, then un-set early + final ByteBuffer[] bufs = buffers; + unsetBuffers(); + // for extra safety unset also all clones' buffers: - synchronized(this.clones) { - for (final MMapIndexInput clone : this.clones) { - assert clone.isClone; - clone.unsetBuffers(); - } - this.clones.clear(); + for (Iterator it = this.clones.keyIterator(); it.hasNext();) { + final MMapIndexInput clone = it.next(); + assert clone.isClone; + clone.unsetBuffers(); } + this.clones.clear(); - curBuf = null; curBufIndex = 0; // nuke curr pointer early - for (int bufNr = 0; bufNr < buffers.length; bufNr++) { - cleanMapping(buffers[bufNr]); + for (final ByteBuffer b : bufs) { + cleanMapping(b); } } finally { unsetBuffers(); } } - - // make sure we have identity on equals/hashCode for WeakHashMap - @Override - public int hashCode() { - return System.identityHashCode(this); - } - - // make sure we have identity on equals/hashCode for WeakHashMap - @Override - public boolean equals(Object obj) { - return obj == this; - } } } diff --git a/lucene/core/src/java/org/apache/lucene/store/MergeInfo.java b/lucene/core/src/java/org/apache/lucene/store/MergeInfo.java index e7bb1bb..b3aea9b 100644 --- a/lucene/core/src/java/org/apache/lucene/store/MergeInfo.java +++ b/lucene/core/src/java/org/apache/lucene/store/MergeInfo.java @@ -1,5 +1,5 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java b/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java index abbbf18..5f483fc 100644 --- a/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF @@ -94,13 +94,13 @@ public class NIOFSDirectory extends FSDirectory { } @Override - public IndexInput openSlice(String sliceDescription, long offset, long length) throws IOException { + public IndexInput openSlice(String sliceDescription, long offset, long length) { return new NIOFSIndexInput(sliceDescription, path, descriptor, descriptor.getChannel(), offset, length, BufferedIndexInput.bufferSize(context), getReadChunkSize()); } @Override - public IndexInput openFullSlice() throws IOException { + public IndexInput openFullSlice() { return openSlice("full-slice", 0, descriptor.length); } }; @@ -117,7 +117,7 @@ public class NIOFSDirectory extends FSDirectory { channel = file.getChannel(); } - public NIOFSIndexInput(String sliceDescription, File path, Descriptor file, FileChannel fc, long off, long length, int bufferSize, int chunkSize) throws IOException { + public NIOFSIndexInput(String sliceDescription, File path, Descriptor file, FileChannel fc, long off, long length, int bufferSize, int chunkSize) { super("NIOFSIndexInput(" + sliceDescription + " in path=\"" + path + "\" slice=" + off + ":" + (off+length) + ")", file, off, length, bufferSize, chunkSize); channel = fc; isClone = true; diff --git a/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java b/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java index 337c0f8..242f72f 100644 --- a/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/NativeFSLockFactory.java b/lucene/core/src/java/org/apache/lucene/store/NativeFSLockFactory.java index 85f473e..9d55ee8 100755 --- a/lucene/core/src/java/org/apache/lucene/store/NativeFSLockFactory.java +++ b/lucene/core/src/java/org/apache/lucene/store/NativeFSLockFactory.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -63,7 +63,7 @@ public class NativeFSLockFactory extends FSLockFactory { * directory itself. Be sure to create one instance for each directory * your create! */ - public NativeFSLockFactory() throws IOException { + public NativeFSLockFactory() { this((File) null); } @@ -73,7 +73,7 @@ public class NativeFSLockFactory extends FSLockFactory { * * @param lockDirName where lock files are created. */ - public NativeFSLockFactory(String lockDirName) throws IOException { + public NativeFSLockFactory(String lockDirName) { this(new File(lockDirName)); } @@ -83,7 +83,7 @@ public class NativeFSLockFactory extends FSLockFactory { * * @param lockDir where lock files are created. */ - public NativeFSLockFactory(File lockDir) throws IOException { + public NativeFSLockFactory(File lockDir) { setLockDir(lockDir); } diff --git a/lucene/core/src/java/org/apache/lucene/store/NoLockFactory.java b/lucene/core/src/java/org/apache/lucene/store/NoLockFactory.java index 70c835a..bd1378d 100755 --- a/lucene/core/src/java/org/apache/lucene/store/NoLockFactory.java +++ b/lucene/core/src/java/org/apache/lucene/store/NoLockFactory.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/NoSuchDirectoryException.java b/lucene/core/src/java/org/apache/lucene/store/NoSuchDirectoryException.java index ef46f49..b9879cb 100644 --- a/lucene/core/src/java/org/apache/lucene/store/NoSuchDirectoryException.java +++ b/lucene/core/src/java/org/apache/lucene/store/NoSuchDirectoryException.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/OutputStreamDataOutput.java b/lucene/core/src/java/org/apache/lucene/store/OutputStreamDataOutput.java index 8be6ad8..a8d6c53 100644 --- a/lucene/core/src/java/org/apache/lucene/store/OutputStreamDataOutput.java +++ b/lucene/core/src/java/org/apache/lucene/store/OutputStreamDataOutput.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java b/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java index 8e9a78b..367d6d0 100644 --- a/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/RAMFile.java b/lucene/core/src/java/org/apache/lucene/store/RAMFile.java index 88f6c83..e346107 100644 --- a/lucene/core/src/java/org/apache/lucene/store/RAMFile.java +++ b/lucene/core/src/java/org/apache/lucene/store/RAMFile.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java b/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java index 60c83f6..fa2fd5f 100644 --- a/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java +++ b/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java b/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java index 10b3253..c7eceae 100644 --- a/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java +++ b/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -147,7 +147,7 @@ public class RAMOutputStream extends IndexOutput { } } - private final void switchCurrentBuffer() throws IOException { + private final void switchCurrentBuffer() { if (currentBufferIndex == file.numBuffers()) { currentBuffer = file.addBuffer(BUFFER_SIZE); } else { diff --git a/lucene/core/src/java/org/apache/lucene/store/RateLimiter.java b/lucene/core/src/java/org/apache/lucene/store/RateLimiter.java index 8e09975..fc685f3 100644 --- a/lucene/core/src/java/org/apache/lucene/store/RateLimiter.java +++ b/lucene/core/src/java/org/apache/lucene/store/RateLimiter.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java b/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java index bc58798..e2deb92 100644 --- a/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -73,13 +73,13 @@ public class SimpleFSDirectory extends FSDirectory { } @Override - public IndexInput openSlice(String sliceDescription, long offset, long length) throws IOException { + public IndexInput openSlice(String sliceDescription, long offset, long length) { return new SimpleFSIndexInput("SimpleFSIndexInput(" + sliceDescription + " in path=\"" + file.getPath() + "\" slice=" + offset + ":" + (offset+length) + ")", descriptor, offset, length, BufferedIndexInput.bufferSize(context), getReadChunkSize()); } @Override - public IndexInput openFullSlice() throws IOException { + public IndexInput openFullSlice() { return openSlice("full-slice", 0, descriptor.length); } }; @@ -124,7 +124,7 @@ public class SimpleFSDirectory extends FSDirectory { this.end = file.length; } - public SimpleFSIndexInput(String resourceDesc, Descriptor file, long off, long length, int bufferSize, int chunkSize) throws IOException { + public SimpleFSIndexInput(String resourceDesc, Descriptor file, long off, long length, int bufferSize, int chunkSize) { super(resourceDesc, bufferSize); this.file = file; this.chunkSize = chunkSize; diff --git a/lucene/core/src/java/org/apache/lucene/store/SimpleFSLockFactory.java b/lucene/core/src/java/org/apache/lucene/store/SimpleFSLockFactory.java index e75b7ac..44ac8bf 100755 --- a/lucene/core/src/java/org/apache/lucene/store/SimpleFSLockFactory.java +++ b/lucene/core/src/java/org/apache/lucene/store/SimpleFSLockFactory.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -60,7 +60,7 @@ public class SimpleFSLockFactory extends FSLockFactory { * directory itself. Be sure to create one instance for each directory * your create! */ - public SimpleFSLockFactory() throws IOException { + public SimpleFSLockFactory() { this((File) null); } @@ -68,7 +68,7 @@ public class SimpleFSLockFactory extends FSLockFactory { * Instantiate using the provided directory (as a File instance). * @param lockDir where lock files should be created. */ - public SimpleFSLockFactory(File lockDir) throws IOException { + public SimpleFSLockFactory(File lockDir) { setLockDir(lockDir); } @@ -76,7 +76,7 @@ public class SimpleFSLockFactory extends FSLockFactory { * Instantiate using the provided directory name (String). * @param lockDirName where lock files should be created. */ - public SimpleFSLockFactory(String lockDirName) throws IOException { + public SimpleFSLockFactory(String lockDirName) { setLockDir(new File(lockDirName)); } diff --git a/lucene/core/src/java/org/apache/lucene/store/SingleInstanceLockFactory.java b/lucene/core/src/java/org/apache/lucene/store/SingleInstanceLockFactory.java index 9264a4c..7e56999 100755 --- a/lucene/core/src/java/org/apache/lucene/store/SingleInstanceLockFactory.java +++ b/lucene/core/src/java/org/apache/lucene/store/SingleInstanceLockFactory.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java b/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java index 3ba5301..a7d4f3c 100644 --- a/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java +++ b/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/store/VerifyingLockFactory.java b/lucene/core/src/java/org/apache/lucene/store/VerifyingLockFactory.java index 9abd563..9bc5e32 100644 --- a/lucene/core/src/java/org/apache/lucene/store/VerifyingLockFactory.java +++ b/lucene/core/src/java/org/apache/lucene/store/VerifyingLockFactory.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -69,8 +69,7 @@ public class VerifyingLockFactory extends LockFactory { } @Override - public synchronized boolean obtain(long lockWaitTimeout) - throws LockObtainFailedException, IOException { + public synchronized boolean obtain(long lockWaitTimeout) throws IOException { boolean obtained = lock.obtain(lockWaitTimeout); if (obtained) verify((byte) 1); @@ -78,8 +77,7 @@ public class VerifyingLockFactory extends LockFactory { } @Override - public synchronized boolean obtain() - throws LockObtainFailedException, IOException { + public synchronized boolean obtain() throws IOException { return lock.obtain(); } @@ -105,7 +103,7 @@ public class VerifyingLockFactory extends LockFactory { * @param port the port {@link LockVerifyServer} is listening on */ - public VerifyingLockFactory(byte id, LockFactory lf, String host, int port) throws IOException { + public VerifyingLockFactory(byte id, LockFactory lf, String host, int port) { this.id = id; this.lf = lf; this.host = host; diff --git a/lucene/core/src/java/org/apache/lucene/util/ArrayUtil.java b/lucene/core/src/java/org/apache/lucene/util/ArrayUtil.java index 4eca482..9b722f9 100644 --- a/lucene/core/src/java/org/apache/lucene/util/ArrayUtil.java +++ b/lucene/core/src/java/org/apache/lucene/util/ArrayUtil.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/Attribute.java b/lucene/core/src/java/org/apache/lucene/util/Attribute.java index fcf5c0c..e78d6f2 100644 --- a/lucene/core/src/java/org/apache/lucene/util/Attribute.java +++ b/lucene/core/src/java/org/apache/lucene/util/Attribute.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/AttributeImpl.java b/lucene/core/src/java/org/apache/lucene/util/AttributeImpl.java index 85c66fa..fac0387 100644 --- a/lucene/core/src/java/org/apache/lucene/util/AttributeImpl.java +++ b/lucene/core/src/java/org/apache/lucene/util/AttributeImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/AttributeReflector.java b/lucene/core/src/java/org/apache/lucene/util/AttributeReflector.java index c64d04c..bb04799 100644 --- a/lucene/core/src/java/org/apache/lucene/util/AttributeReflector.java +++ b/lucene/core/src/java/org/apache/lucene/util/AttributeReflector.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java b/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java index f2de36f..65f16c6 100644 --- a/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java +++ b/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/BitUtil.java b/lucene/core/src/java/org/apache/lucene/util/BitUtil.java index 75850d8..e755da1 100644 --- a/lucene/core/src/java/org/apache/lucene/util/BitUtil.java +++ b/lucene/core/src/java/org/apache/lucene/util/BitUtil.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/Bits.java b/lucene/core/src/java/org/apache/lucene/util/Bits.java index 84ea334..6c4f517 100644 --- a/lucene/core/src/java/org/apache/lucene/util/Bits.java +++ b/lucene/core/src/java/org/apache/lucene/util/Bits.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/BitsSlice.java b/lucene/core/src/java/org/apache/lucene/util/BitsSlice.java deleted file mode 100644 index 2c3211c..0000000 --- a/lucene/core/src/java/org/apache/lucene/util/BitsSlice.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.apache.lucene.util; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** Exposes a slice of an existing Bits as a new Bits. */ - -public final class BitsSlice implements Bits { - private final Bits parent; - private final int start; - private final int length; - - // start is inclusive; end is exclusive (length = end-start) - public BitsSlice(Bits parent, ReaderUtil.Slice slice) { - this.parent = parent; - this.start = slice.start; - this.length = slice.length; - assert length >= 0: "length=" + length; - } - - public boolean get(int doc) { - if (doc >= length) { - throw new RuntimeException("doc " + doc + " is out of bounds 0 .. " + (length-1)); - } - assert doc < length: "doc=" + doc + " length=" + length; - return parent.get(doc+start); - } - - public int length() { - return length; - } -} diff --git a/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java b/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java index 03563e5..3378a3e 100644 --- a/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java +++ b/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/BytesRef.java b/lucene/core/src/java/org/apache/lucene/util/BytesRef.java index 900a96f..8fe045f 100644 --- a/lucene/core/src/java/org/apache/lucene/util/BytesRef.java +++ b/lucene/core/src/java/org/apache/lucene/util/BytesRef.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,7 +23,13 @@ import java.util.Comparator; * existing byte[]. The {@link #bytes} member should never be null; * use {@link #EMPTY_BYTES} if necessary. * - * @lucene.experimental */ + *

      Important note: Unless otherwise noted, Lucene uses this class to + * represent terms that are encoded as UTF8 bytes in the index. To + * convert them to a Java {@link String} (which is UTF16), use {@link #utf8ToString}. + * Using code like {@code new String(bytes, offset, length)} to do this + * is wrong, as it does not respect the correct character set + * and may return wrong results (depending on the platform's defaults)! + */ public final class BytesRef implements Comparable,Cloneable { /** An empty byte array for convenience */ public static final byte[] EMPTY_BYTES = new byte[0]; @@ -47,6 +53,9 @@ public final class BytesRef implements Comparable,Cloneable { */ public BytesRef(byte[] bytes, int offset, int length) { assert bytes != null; + assert offset >= 0; + assert length >= 0; + assert bytes.length >= offset + length; this.bytes = bytes; this.offset = offset; this.length = length; @@ -84,8 +93,8 @@ public final class BytesRef implements Comparable,Cloneable { * @param text Must be well-formed unicode text, with no * unpaired surrogates or invalid UTF16 code units. */ - // TODO broken if offset != 0 public void copyChars(CharSequence text) { + assert offset == 0; // TODO broken if offset != 0 UnicodeUtil.UTF16toUTF8(text, 0, text.length(), this); } @@ -180,7 +189,7 @@ public final class BytesRef implements Comparable,Cloneable { * new reference array. */ public void copyBytes(BytesRef other) { - if (bytes.length < other.length) { + if (bytes.length - offset < other.length) { bytes = new byte[other.length]; offset = 0; } @@ -196,7 +205,7 @@ public final class BytesRef implements Comparable,Cloneable { */ public void append(BytesRef other) { int newLen = length + other.length; - if (bytes.length < newLen) { + if (bytes.length - offset < newLen) { byte[] newBytes = new byte[newLen]; System.arraycopy(bytes, offset, newBytes, 0, length); offset = 0; @@ -206,9 +215,13 @@ public final class BytesRef implements Comparable,Cloneable { length = newLen; } - // TODO: stupid if existing offset is non-zero. - /** @lucene.internal */ + /** + * Used to grow the reference array. + * + * In general this should not be used as it does not take the offset into account. + * @lucene.internal */ public void grow(int newLength) { + assert offset == 0; // NOTE: senseless if offset != 0 bytes = ArrayUtil.grow(bytes, newLength); } diff --git a/lucene/core/src/java/org/apache/lucene/util/BytesRefHash.java b/lucene/core/src/java/org/apache/lucene/util/BytesRefHash.java index 8fad50d..45385db 100644 --- a/lucene/core/src/java/org/apache/lucene/util/BytesRefHash.java +++ b/lucene/core/src/java/org/apache/lucene/util/BytesRefHash.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/BytesRefIterator.java b/lucene/core/src/java/org/apache/lucene/util/BytesRefIterator.java index f16d6dc..57d7df7 100644 --- a/lucene/core/src/java/org/apache/lucene/util/BytesRefIterator.java +++ b/lucene/core/src/java/org/apache/lucene/util/BytesRefIterator.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -50,7 +50,7 @@ public interface BytesRefIterator { public static final BytesRefIterator EMPTY = new BytesRefIterator() { @Override - public BytesRef next() throws IOException { + public BytesRef next() { return null; } diff --git a/lucene/core/src/java/org/apache/lucene/util/CharsRef.java b/lucene/core/src/java/org/apache/lucene/util/CharsRef.java index a1e34cd..bc917b0 100644 --- a/lucene/core/src/java/org/apache/lucene/util/CharsRef.java +++ b/lucene/core/src/java/org/apache/lucene/util/CharsRef.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -52,6 +52,8 @@ public final class CharsRef implements Comparable, CharSequence, Clone */ public CharsRef(char[] chars, int offset, int length) { assert chars != null; + assert offset >= 0; + assert length >= 0; assert chars.length >= offset + length; this.chars = chars; this.offset = offset; @@ -138,50 +140,52 @@ public final class CharsRef implements Comparable, CharSequence, Clone } /** - * Copies the given {@link CharsRef} referenced content into this instance - * starting at offset 0. + * Copies the given {@link CharsRef} referenced content into this instance. * * @param other * the {@link CharsRef} to copy */ - // TODO: why does this behave differently/not invoke copyChars(char[], int, int) ??? public void copyChars(CharsRef other) { - if (chars == null) { - chars = new char[other.length]; - } else { - chars = ArrayUtil.grow(chars, other.length); - } - System.arraycopy(other.chars, other.offset, chars, 0, other.length); - length = other.length; - offset = 0; + copyChars(other.chars, other.offset, other.length); } + /** + * Used to grow the reference array. + * + * In general this should not be used as it does not take the offset into account. + * @lucene.internal */ public void grow(int newLength) { + assert offset == 0; if (chars.length < newLength) { chars = ArrayUtil.grow(chars, newLength); } } /** - * Copies the given array into this CharsRef starting at offset 0 + * Copies the given array into this CharsRef. */ public void copyChars(char[] otherChars, int otherOffset, int otherLength) { - grow(otherLength); - System.arraycopy(otherChars, otherOffset, this.chars, 0, - otherLength); - this.offset = 0; - this.length = otherLength; + if (chars.length - offset < otherLength) { + chars = new char[otherLength]; + offset = 0; + } + System.arraycopy(otherChars, otherOffset, chars, offset, otherLength); + length = otherLength; } /** * Appends the given array to this CharsRef */ public void append(char[] otherChars, int otherOffset, int otherLength) { - final int newLength = length + otherLength; - grow(this.offset + newLength); - System.arraycopy(otherChars, otherOffset, this.chars, this.offset+length, - otherLength); - this.length += otherLength; + int newLen = length + otherLength; + if (chars.length - offset < newLen) { + char[] newChars = new char[newLen]; + System.arraycopy(chars, offset, newChars, 0, length); + offset = 0; + chars = newChars; + } + System.arraycopy(otherChars, otherOffset, chars, length+offset, otherLength); + length = newLen; } @Override @@ -194,11 +198,19 @@ public final class CharsRef implements Comparable, CharSequence, Clone } public char charAt(int index) { + // NOTE: must do a real check here to meet the specs of CharSequence + if (index < 0 || index >= length) { + throw new IndexOutOfBoundsException(); + } return chars[offset + index]; } public CharSequence subSequence(int start, int end) { - return new CharsRef(chars, offset + start, offset + end - 1); + // NOTE: must do a real check here to meet the specs of CharSequence + if (start < 0 || end > length || start > end) { + throw new IndexOutOfBoundsException(); + } + return new CharsRef(chars, offset + start, offset + end); } /** @deprecated */ diff --git a/lucene/core/src/java/org/apache/lucene/util/CloseableThreadLocal.java b/lucene/core/src/java/org/apache/lucene/util/CloseableThreadLocal.java index a1792fa..4e521a2 100644 --- a/lucene/core/src/java/org/apache/lucene/util/CloseableThreadLocal.java +++ b/lucene/core/src/java/org/apache/lucene/util/CloseableThreadLocal.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/CodecUtil.java b/lucene/core/src/java/org/apache/lucene/util/CodecUtil.java deleted file mode 100644 index 5e19988..0000000 --- a/lucene/core/src/java/org/apache/lucene/util/CodecUtil.java +++ /dev/null @@ -1,152 +0,0 @@ -package org.apache.lucene.util; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -import java.io.IOException; - -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.IndexFormatTooNewException; -import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.store.DataInput; -import org.apache.lucene.store.DataOutput; - -/** - * Utility class for reading and writing versioned headers. - *

      - * Writing codec headers is useful to ensure that a file is in - * the format you think it is. - * - * @lucene.experimental - */ - -public final class CodecUtil { - private CodecUtil() {} // no instance - - /** - * Constant to identify the start of a codec header. - */ - public final static int CODEC_MAGIC = 0x3fd76c17; - - /** - * Writes a codec header, which records both a string to - * identify the file and a version number. This header can - * be parsed and validated with - * {@link #checkHeader(DataInput, String, int, int) checkHeader()}. - *

      - * CodecHeader --> Magic,CodecName,Version - *

        - *
      • Magic --> {@link DataOutput#writeInt Uint32}. This - * identifies the start of the header. It is always {@value #CODEC_MAGIC}. - *
      • CodecName --> {@link DataOutput#writeString String}. This - * is a string to identify this file. - *
      • Version --> {@link DataOutput#writeInt Uint32}. Records - * the version of the file. - *
      - *

      - * Note that the length of a codec header depends only upon the - * name of the codec, so this length can be computed at any time - * with {@link #headerLength(String)}. - * - * @param out Output stream - * @param codec String to identify this file. It should be simple ASCII, - * less than 128 characters in length. - * @param version Version number - * @throws IOException If there is an I/O error writing to the underlying medium. - */ - public static void writeHeader(DataOutput out, String codec, int version) - throws IOException { - BytesRef bytes = new BytesRef(codec); - if (bytes.length != codec.length() || bytes.length >= 128) { - throw new IllegalArgumentException("codec must be simple ASCII, less than 128 characters in length [got " + codec + "]"); - } - out.writeInt(CODEC_MAGIC); - out.writeString(codec); - out.writeInt(version); - } - - /** - * Computes the length of a codec header. - * - * @param codec Codec name. - * @return length of the entire codec header. - * @see #writeHeader(DataOutput, String, int) - */ - public static int headerLength(String codec) { - return 9+codec.length(); - } - - /** - * Reads and validates a header previously written with - * {@link #writeHeader(DataOutput, String, int)}. - *

      - * When reading a file, supply the expected codec and - * an expected version range (minVersion to maxVersion). - * - * @param in Input stream, positioned at the point where the - * header was previously written. Typically this is located - * at the beginning of the file. - * @param codec The expected codec name. - * @param minVersion The minimum supported expected version number. - * @param maxVersion The maximum supported expected version number. - * @return The actual version found, when a valid header is found - * that matches codec, with an actual version - * where minVersion <= actual <= maxVersion. - * Otherwise an exception is thrown. - * @throws CorruptIndexException If the first four bytes are not - * {@link #CODEC_MAGIC}, or if the actual codec found is - * not codec. - * @throws IndexFormatTooOldException If the actual version is less - * than minVersion. - * @throws IndexFormatTooNewException If the actual version is greater - * than maxVersion. - * @throws IOException If there is an I/O error reading from the underlying medium. - * @see #writeHeader(DataOutput, String, int) - */ - public static int checkHeader(DataInput in, String codec, int minVersion, int maxVersion) - throws IOException { - - // Safety to guard against reading a bogus string: - final int actualHeader = in.readInt(); - if (actualHeader != CODEC_MAGIC) { - throw new CorruptIndexException("codec header mismatch: actual header=" + actualHeader + " vs expected header=" + CODEC_MAGIC + " (resource: " + in + ")"); - } - return checkHeaderNoMagic(in, codec, minVersion, maxVersion); - } - - /** Like {@link - * #checkHeader(DataInput,String,int,int)} except this - * version assumes the first int has already been read - * and validated from the input. */ - public static int checkHeaderNoMagic(DataInput in, String codec, int minVersion, int maxVersion) throws IOException { - final String actualCodec = in.readString(); - if (!actualCodec.equals(codec)) { - throw new CorruptIndexException("codec mismatch: actual codec=" + actualCodec + " vs expected codec=" + codec + " (resource: " + in + ")"); - } - - final int actualVersion = in.readInt(); - if (actualVersion < minVersion) { - throw new IndexFormatTooOldException(in, actualVersion, minVersion, maxVersion); - } - if (actualVersion > maxVersion) { - throw new IndexFormatTooNewException(in, actualVersion, minVersion, maxVersion); - } - - return actualVersion; - } -} diff --git a/lucene/core/src/java/org/apache/lucene/util/CollectionUtil.java b/lucene/core/src/java/org/apache/lucene/util/CollectionUtil.java index d2384cb..2647893 100644 --- a/lucene/core/src/java/org/apache/lucene/util/CollectionUtil.java +++ b/lucene/core/src/java/org/apache/lucene/util/CollectionUtil.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/CommandLineUtil.java b/lucene/core/src/java/org/apache/lucene/util/CommandLineUtil.java index 4f1c38c..c628bc2 100644 --- a/lucene/core/src/java/org/apache/lucene/util/CommandLineUtil.java +++ b/lucene/core/src/java/org/apache/lucene/util/CommandLineUtil.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/Constants.java b/lucene/core/src/java/org/apache/lucene/util/Constants.java index d18feda..353a5be 100644 --- a/lucene/core/src/java/org/apache/lucene/util/Constants.java +++ b/lucene/core/src/java/org/apache/lucene/util/Constants.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -104,8 +104,14 @@ public final class Constants { // NOTE: we track per-segment version as a String with the "X.Y" format, e.g. // "4.0", "3.1", "3.0". Therefore when we change this constant, we should keep // the format. - public static final String LUCENE_MAIN_VERSION = ident("4.0"); + /** + * This is the internal Lucene version, recorded into each segment. + */ + public static final String LUCENE_MAIN_VERSION = ident("5.0"); + /** + * This is the Lucene version for display purposes. + */ public static final String LUCENE_VERSION; static { Package pkg = LucenePackage.get(); diff --git a/lucene/core/src/java/org/apache/lucene/util/Counter.java b/lucene/core/src/java/org/apache/lucene/util/Counter.java index b7642ac..9aa5dec 100644 --- a/lucene/core/src/java/org/apache/lucene/util/Counter.java +++ b/lucene/core/src/java/org/apache/lucene/util/Counter.java @@ -2,7 +2,7 @@ package org.apache.lucene.util; import java.util.concurrent.atomic.AtomicLong; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/DocIdBitSet.java b/lucene/core/src/java/org/apache/lucene/util/DocIdBitSet.java index 679147f..1470de2 100644 --- a/lucene/core/src/java/org/apache/lucene/util/DocIdBitSet.java +++ b/lucene/core/src/java/org/apache/lucene/util/DocIdBitSet.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/DoubleBarrelLRUCache.java b/lucene/core/src/java/org/apache/lucene/util/DoubleBarrelLRUCache.java index d1c06f0..14f2e44 100644 --- a/lucene/core/src/java/org/apache/lucene/util/DoubleBarrelLRUCache.java +++ b/lucene/core/src/java/org/apache/lucene/util/DoubleBarrelLRUCache.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/DummyConcurrentLock.java b/lucene/core/src/java/org/apache/lucene/util/DummyConcurrentLock.java deleted file mode 100644 index de2f46b..0000000 --- a/lucene/core/src/java/org/apache/lucene/util/DummyConcurrentLock.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.apache.lucene.util; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.ReentrantLock; - -/** - * A dummy lock as a replacement for {@link ReentrantLock} to disable locking - * @lucene.internal - */ -public final class DummyConcurrentLock implements Lock { - - /** a default instance, can be always used, as this {@link Lock} is stateless. */ - public static final DummyConcurrentLock INSTANCE = new DummyConcurrentLock(); - - public void lock() {} - - public void lockInterruptibly() {} - - public boolean tryLock() { - return true; - } - - public boolean tryLock(long time, TimeUnit unit) { - return true; - } - - public void unlock() {} - - public Condition newCondition() { - throw new UnsupportedOperationException(); - } - -} diff --git a/lucene/core/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java b/lucene/core/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java index 32e8f2a..6946329 100644 --- a/lucene/core/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java +++ b/lucene/core/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java @@ -280,9 +280,9 @@ public final class FieldCacheSanityChecker { for (int i = 0; i < all.size(); i++) { Object obj = all.get(i); if (obj instanceof CompositeReader) { - IndexReader[] subs = ((CompositeReader)obj).getSequentialSubReaders(); - for (int j = 0; (null != subs) && (j < subs.length); j++) { - all.add(subs[j].getCoreCacheKey()); + List subs = ((CompositeReader)obj).getSequentialSubReaders(); + for (int j = 0; (null != subs) && (j < subs.size()); j++) { + all.add(subs.get(j).getCoreCacheKey()); } } diff --git a/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java b/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java index 10ee140..915dae6 100644 --- a/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java +++ b/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/IOUtils.java b/lucene/core/src/java/org/apache/lucene/util/IOUtils.java index dcd5e6b..45688ab 100644 --- a/lucene/core/src/java/org/apache/lucene/util/IOUtils.java +++ b/lucene/core/src/java/org/apache/lucene/util/IOUtils.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/IndexableBinaryStringTools.java b/lucene/core/src/java/org/apache/lucene/util/IndexableBinaryStringTools.java deleted file mode 100644 index fdb1e71..0000000 --- a/lucene/core/src/java/org/apache/lucene/util/IndexableBinaryStringTools.java +++ /dev/null @@ -1,241 +0,0 @@ -package org.apache.lucene.util; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; // javadoc - -/** - * Provides support for converting byte sequences to Strings and back again. - * The resulting Strings preserve the original byte sequences' sort order. - *

      - * The Strings are constructed using a Base 8000h encoding of the original - * binary data - each char of an encoded String represents a 15-bit chunk - * from the byte sequence. Base 8000h was chosen because it allows for all - * lower 15 bits of char to be used without restriction; the surrogate range - * [U+D8000-U+DFFF] does not represent valid chars, and would require - * complicated handling to avoid them and allow use of char's high bit. - *

      - * Although unset bits are used as padding in the final char, the original - * byte sequence could contain trailing bytes with no set bits (null bytes): - * padding is indistinguishable from valid information. To overcome this - * problem, a char is appended, indicating the number of encoded bytes in the - * final content char. - *

      - * - * @lucene.experimental - * @deprecated Implement {@link TermToBytesRefAttribute} and store bytes directly - * instead. This class will be removed in Lucene 5.0 - */ -@Deprecated -public final class IndexableBinaryStringTools { - - private static final CodingCase[] CODING_CASES = { - // CodingCase(int initialShift, int finalShift) - new CodingCase( 7, 1 ), - // CodingCase(int initialShift, int middleShift, int finalShift) - new CodingCase(14, 6, 2), - new CodingCase(13, 5, 3), - new CodingCase(12, 4, 4), - new CodingCase(11, 3, 5), - new CodingCase(10, 2, 6), - new CodingCase( 9, 1, 7), - new CodingCase( 8, 0 ) - }; - - // Export only static methods - private IndexableBinaryStringTools() {} - - /** - * Returns the number of chars required to encode the given bytes. - * - * @param inputArray byte sequence to be encoded - * @param inputOffset initial offset into inputArray - * @param inputLength number of bytes in inputArray - * @return The number of chars required to encode the number of bytes. - */ - public static int getEncodedLength(byte[] inputArray, int inputOffset, - int inputLength) { - // Use long for intermediaries to protect against overflow - return (int)((8L * inputLength + 14L) / 15L) + 1; - } - - /** - * Returns the number of bytes required to decode the given char sequence. - * - * @param encoded char sequence to be decoded - * @param offset initial offset - * @param length number of characters - * @return The number of bytes required to decode the given char sequence - */ - public static int getDecodedLength(char[] encoded, int offset, int length) { - final int numChars = length - 1; - if (numChars <= 0) { - return 0; - } else { - // Use long for intermediaries to protect against overflow - final long numFullBytesInFinalChar = encoded[offset + length - 1]; - final long numEncodedChars = numChars - 1; - return (int)((numEncodedChars * 15L + 7L) / 8L + numFullBytesInFinalChar); - } - } - - /** - * Encodes the input byte sequence into the output char sequence. Before - * calling this method, ensure that the output array has sufficient - * capacity by calling {@link #getEncodedLength(byte[], int, int)}. - * - * @param inputArray byte sequence to be encoded - * @param inputOffset initial offset into inputArray - * @param inputLength number of bytes in inputArray - * @param outputArray char sequence to store encoded result - * @param outputOffset initial offset into outputArray - * @param outputLength length of output, must be getEncodedLength - */ - public static void encode(byte[] inputArray, int inputOffset, - int inputLength, char[] outputArray, int outputOffset, int outputLength) { - assert (outputLength == getEncodedLength(inputArray, inputOffset, - inputLength)); - if (inputLength > 0) { - int inputByteNum = inputOffset; - int caseNum = 0; - int outputCharNum = outputOffset; - CodingCase codingCase; - for (; inputByteNum + CODING_CASES[caseNum].numBytes <= inputLength; ++outputCharNum) { - codingCase = CODING_CASES[caseNum]; - if (2 == codingCase.numBytes) { - outputArray[outputCharNum] = (char) (((inputArray[inputByteNum] & 0xFF) << codingCase.initialShift) - + (((inputArray[inputByteNum + 1] & 0xFF) >>> codingCase.finalShift) & codingCase.finalMask) & (short) 0x7FFF); - } else { // numBytes is 3 - outputArray[outputCharNum] = (char) (((inputArray[inputByteNum] & 0xFF) << codingCase.initialShift) - + ((inputArray[inputByteNum + 1] & 0xFF) << codingCase.middleShift) - + (((inputArray[inputByteNum + 2] & 0xFF) >>> codingCase.finalShift) & codingCase.finalMask) & (short) 0x7FFF); - } - inputByteNum += codingCase.advanceBytes; - if (++caseNum == CODING_CASES.length) { - caseNum = 0; - } - } - // Produce final char (if any) and trailing count chars. - codingCase = CODING_CASES[caseNum]; - - if (inputByteNum + 1 < inputLength) { // codingCase.numBytes must be 3 - outputArray[outputCharNum++] = (char) ((((inputArray[inputByteNum] & 0xFF) << codingCase.initialShift) + ((inputArray[inputByteNum + 1] & 0xFF) << codingCase.middleShift)) & (short) 0x7FFF); - // Add trailing char containing the number of full bytes in final char - outputArray[outputCharNum++] = (char) 1; - } else if (inputByteNum < inputLength) { - outputArray[outputCharNum++] = (char) (((inputArray[inputByteNum] & 0xFF) << codingCase.initialShift) & (short) 0x7FFF); - // Add trailing char containing the number of full bytes in final char - outputArray[outputCharNum++] = caseNum == 0 ? (char) 1 : (char) 0; - } else { // No left over bits - last char is completely filled. - // Add trailing char containing the number of full bytes in final char - outputArray[outputCharNum++] = (char) 1; - } - } - } - - /** - * Decodes the input char sequence into the output byte sequence. Before - * calling this method, ensure that the output array has sufficient capacity - * by calling {@link #getDecodedLength(char[], int, int)}. - * - * @param inputArray char sequence to be decoded - * @param inputOffset initial offset into inputArray - * @param inputLength number of chars in inputArray - * @param outputArray byte sequence to store encoded result - * @param outputOffset initial offset into outputArray - * @param outputLength length of output, must be - * getDecodedLength(inputArray, inputOffset, inputLength) - */ - public static void decode(char[] inputArray, int inputOffset, - int inputLength, byte[] outputArray, int outputOffset, int outputLength) { - assert (outputLength == getDecodedLength(inputArray, inputOffset, - inputLength)); - final int numInputChars = inputLength - 1; - final int numOutputBytes = outputLength; - - if (numOutputBytes > 0) { - int caseNum = 0; - int outputByteNum = outputOffset; - int inputCharNum = inputOffset; - short inputChar; - CodingCase codingCase; - for (; inputCharNum < numInputChars - 1; ++inputCharNum) { - codingCase = CODING_CASES[caseNum]; - inputChar = (short) inputArray[inputCharNum]; - if (2 == codingCase.numBytes) { - if (0 == caseNum) { - outputArray[outputByteNum] = (byte) (inputChar >>> codingCase.initialShift); - } else { - outputArray[outputByteNum] += (byte) (inputChar >>> codingCase.initialShift); - } - outputArray[outputByteNum + 1] = (byte) ((inputChar & codingCase.finalMask) << codingCase.finalShift); - } else { // numBytes is 3 - outputArray[outputByteNum] += (byte) (inputChar >>> codingCase.initialShift); - outputArray[outputByteNum + 1] = (byte) ((inputChar & codingCase.middleMask) >>> codingCase.middleShift); - outputArray[outputByteNum + 2] = (byte) ((inputChar & codingCase.finalMask) << codingCase.finalShift); - } - outputByteNum += codingCase.advanceBytes; - if (++caseNum == CODING_CASES.length) { - caseNum = 0; - } - } - // Handle final char - inputChar = (short) inputArray[inputCharNum]; - codingCase = CODING_CASES[caseNum]; - if (0 == caseNum) { - outputArray[outputByteNum] = 0; - } - outputArray[outputByteNum] += (byte) (inputChar >>> codingCase.initialShift); - final int bytesLeft = numOutputBytes - outputByteNum; - if (bytesLeft > 1) { - if (2 == codingCase.numBytes) { - outputArray[outputByteNum + 1] = (byte) ((inputChar & codingCase.finalMask) >>> codingCase.finalShift); - } else { // numBytes is 3 - outputArray[outputByteNum + 1] = (byte) ((inputChar & codingCase.middleMask) >>> codingCase.middleShift); - if (bytesLeft > 2) { - outputArray[outputByteNum + 2] = (byte) ((inputChar & codingCase.finalMask) << codingCase.finalShift); - } - } - } - } - } - - static class CodingCase { - int numBytes, initialShift, middleShift, finalShift, advanceBytes = 2; - short middleMask, finalMask; - - CodingCase(int initialShift, int middleShift, int finalShift) { - this.numBytes = 3; - this.initialShift = initialShift; - this.middleShift = middleShift; - this.finalShift = finalShift; - this.finalMask = (short)((short)0xFF >>> finalShift); - this.middleMask = (short)((short)0xFF << middleShift); - } - - CodingCase(int initialShift, int finalShift) { - this.numBytes = 2; - this.initialShift = initialShift; - this.finalShift = finalShift; - this.finalMask = (short)((short)0xFF >>> finalShift); - if (finalShift != 0) { - advanceBytes = 1; - } - } - } -} diff --git a/lucene/core/src/java/org/apache/lucene/util/InfoStream.java b/lucene/core/src/java/org/apache/lucene/util/InfoStream.java index fe5c7a4..7e8a5d3 100644 --- a/lucene/core/src/java/org/apache/lucene/util/InfoStream.java +++ b/lucene/core/src/java/org/apache/lucene/util/InfoStream.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/IntsRef.java b/lucene/core/src/java/org/apache/lucene/util/IntsRef.java index e2a4dc9..6c5fd51 100644 --- a/lucene/core/src/java/org/apache/lucene/util/IntsRef.java +++ b/lucene/core/src/java/org/apache/lucene/util/IntsRef.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -40,6 +40,9 @@ public final class IntsRef implements Comparable, Cloneable { public IntsRef(int[] ints, int offset, int length) { assert ints != null; + assert offset >= 0; + assert length >= 0; + assert ints.length >= offset + length; this.ints = ints; this.offset = offset; this.length = length; @@ -114,17 +117,21 @@ public final class IntsRef implements Comparable, Cloneable { } public void copyInts(IntsRef other) { - if (ints == null) { + if (ints.length - offset < other.length) { ints = new int[other.length]; - } else { - ints = ArrayUtil.grow(ints, other.length); + offset = 0; } - System.arraycopy(other.ints, other.offset, ints, 0, other.length); + System.arraycopy(other.ints, other.offset, ints, offset, other.length); length = other.length; - offset = 0; } + /** + * Used to grow the reference array. + * + * In general this should not be used as it does not take the offset into account. + * @lucene.internal */ public void grow(int newLength) { + assert offset == 0; if (ints.length < newLength) { ints = ArrayUtil.grow(ints, newLength); } diff --git a/lucene/core/src/java/org/apache/lucene/util/LongsRef.java b/lucene/core/src/java/org/apache/lucene/util/LongsRef.java new file mode 100644 index 0000000..54ce73a --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/util/LongsRef.java @@ -0,0 +1,167 @@ +package org.apache.lucene.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Represents long[], as a slice (offset + length) into an + * existing long[]. The {@link #longs} member should never be null; use + * {@link #EMPTY_LONGS} if necessary. + * + * @lucene.internal */ +public final class LongsRef implements Comparable, Cloneable { + + public static final long[] EMPTY_LONGS = new long[0]; + + public long[] longs; + public int offset; + public int length; + + public LongsRef() { + longs = EMPTY_LONGS; + } + + public LongsRef(int capacity) { + longs = new long[capacity]; + } + + public LongsRef(long[] longs, int offset, int length) { + assert longs != null; + assert offset >= 0; + assert length >= 0; + assert longs.length >= offset + length; + this.longs = longs; + this.offset = offset; + this.length = length; + } + + @Override + public LongsRef clone() { + return new LongsRef(longs, offset, length); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 0; + final long end = offset + length; + for(int i = offset; i < end; i++) { + result = prime * result + (int) (longs[i] ^ (longs[i]>>>32)); + } + return result; + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other instanceof LongsRef) { + return this.longsEquals((LongsRef) other); + } + return false; + } + + public boolean longsEquals(LongsRef other) { + if (length == other.length) { + int otherUpto = other.offset; + final long[] otherInts = other.longs; + final long end = offset + length; + for(int upto=offset; upto bInt) { + return 1; + } else if (aInt < bInt) { + return -1; + } + } + + // One is a prefix of the other, or, they are equal: + return this.length - other.length; + } + + public void copyLongs(LongsRef other) { + if (longs.length - offset < other.length) { + longs = new long[other.length]; + offset = 0; + } + System.arraycopy(other.longs, other.offset, longs, offset, other.length); + length = other.length; + } + + /** + * Used to grow the reference array. + * + * In general this should not be used as it does not take the offset into account. + * @lucene.internal */ + public void grow(int newLength) { + assert offset == 0; + if (longs.length < newLength) { + longs = ArrayUtil.grow(longs, newLength); + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append('['); + final long end = offset + length; + for(int i=offset;i offset) { + sb.append(' '); + } + sb.append(Long.toHexString(longs[i])); + } + sb.append(']'); + return sb.toString(); + } + + /** + * Creates a new IntsRef that points to a copy of the longs from + * other + *

      + * The returned IntsRef will have a length of other.length + * and an offset of zero. + */ + public static LongsRef deepCopyOf(LongsRef other) { + LongsRef clone = new LongsRef(); + clone.copyLongs(other); + return clone; + } +} diff --git a/lucene/core/src/java/org/apache/lucene/util/MapOfSets.java b/lucene/core/src/java/org/apache/lucene/util/MapOfSets.java index 9c18925..b08eb76 100644 --- a/lucene/core/src/java/org/apache/lucene/util/MapOfSets.java +++ b/lucene/core/src/java/org/apache/lucene/util/MapOfSets.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/MathUtil.java b/lucene/core/src/java/org/apache/lucene/util/MathUtil.java index a25c637..9c86099 100644 --- a/lucene/core/src/java/org/apache/lucene/util/MathUtil.java +++ b/lucene/core/src/java/org/apache/lucene/util/MathUtil.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -27,9 +27,14 @@ public final class MathUtil { private MathUtil() { } - /** returns x == 0 ? 0 : Math.floor(Math.log(x) / Math.log(base)) */ + /** + * Returns {@code x <= 0 ? 0 : Math.floor(Math.log(x) / Math.log(base))} + * @param base must be {@code > 1} + */ public static int log(long x, int base) { - assert base > 1; + if (base <= 1) { + throw new IllegalArgumentException("base must be > 1"); + } int ret = 0; while (x >= base) { x /= base; diff --git a/lucene/core/src/java/org/apache/lucene/util/MultiBits.java b/lucene/core/src/java/org/apache/lucene/util/MultiBits.java deleted file mode 100644 index b93b1c7..0000000 --- a/lucene/core/src/java/org/apache/lucene/util/MultiBits.java +++ /dev/null @@ -1,119 +0,0 @@ -package org.apache.lucene.util; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.List; - -/** - * Concatenates multiple Bits together, on every lookup. - * - *

      NOTE: This is very costly, as every lookup must - * do a binary search to locate the right sub-reader. - * - * @lucene.experimental - */ - -public final class MultiBits implements Bits { - private final Bits[] subs; - - // length is 1+subs.length (the last entry has the maxDoc): - private final int[] starts; - - private final boolean defaultValue; - - public MultiBits(List bits, List starts, boolean defaultValue) { - assert starts.size() == 1+bits.size(); - this.subs = bits.toArray(Bits.EMPTY_ARRAY); - this.starts = new int[starts.size()]; - for(int i=0;i= offset + length; if (length == 0) { return; diff --git a/lucene/core/src/java/org/apache/lucene/util/PrintStreamInfoStream.java b/lucene/core/src/java/org/apache/lucene/util/PrintStreamInfoStream.java index aa5e689..33413be 100644 --- a/lucene/core/src/java/org/apache/lucene/util/PrintStreamInfoStream.java +++ b/lucene/core/src/java/org/apache/lucene/util/PrintStreamInfoStream.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/PriorityQueue.java b/lucene/core/src/java/org/apache/lucene/util/PriorityQueue.java index 38252bf..6cf6c27 100644 --- a/lucene/core/src/java/org/apache/lucene/util/PriorityQueue.java +++ b/lucene/core/src/java/org/apache/lucene/util/PriorityQueue.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java b/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java index eaf23dd..9e3a793 100644 --- a/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java +++ b/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/ReaderUtil.java b/lucene/core/src/java/org/apache/lucene/util/ReaderUtil.java deleted file mode 100644 index d222090..0000000 --- a/lucene/core/src/java/org/apache/lucene/util/ReaderUtil.java +++ /dev/null @@ -1,185 +0,0 @@ -package org.apache.lucene.util; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.List; -import java.io.IOException; - -import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.index.CompositeReader; -import org.apache.lucene.index.AtomicReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexReaderContext; - -/** - * Common util methods for dealing with {@link IndexReader}s. - * - * @lucene.internal - */ -public final class ReaderUtil { - - private ReaderUtil() {} // no instance - - /** - * Subreader slice from a parent composite reader. - */ - public static class Slice { - public static final Slice[] EMPTY_ARRAY = new Slice[0]; - public final int start; - public final int length; - public final int readerIndex; - - public Slice(int start, int length, int readerIndex) { - this.start = start; - this.length = length; - this.readerIndex = readerIndex; - } - - @Override - public String toString() { - return "slice start=" + start + " length=" + length + " readerIndex=" + readerIndex; - } - } - - /** - * Gathers sub-readers from reader into a List. See - * {@link Gather} for are more general way to gather - * whatever you need to, per reader. - * - * @lucene.experimental - * - * @param allSubReaders - * @param reader - */ - - public static void gatherSubReaders(final List allSubReaders, IndexReader reader) { - try { - new Gather(reader) { - @Override - protected void add(int base, AtomicReader r) { - allSubReaders.add(r); - } - }.run(); - } catch (IOException ioe) { - // won't happen - throw new RuntimeException(ioe); - } - } - - /** Recursively visits all sub-readers of a reader. You - * should subclass this and override the add method to - * gather what you need. - * - * @lucene.experimental */ - public static abstract class Gather { - private final IndexReader topReader; - - public Gather(IndexReader r) { - topReader = r; - } - - public int run() throws IOException { - return run(0, topReader); - } - - public int run(int docBase) throws IOException { - return run(docBase, topReader); - } - - private int run(int base, IndexReader reader) throws IOException { - if (reader instanceof AtomicReader) { - // atomic reader - add(base, (AtomicReader) reader); - base += reader.maxDoc(); - } else { - assert reader instanceof CompositeReader : "must be a composite reader"; - IndexReader[] subReaders = ((CompositeReader) reader).getSequentialSubReaders(); - for (int i = 0; i < subReaders.length; i++) { - base = run(base, subReaders[i]); - } - } - - return base; - } - - protected abstract void add(int base, AtomicReader r) throws IOException; - } - - /** - * Walks up the reader tree and return the given context's top level reader - * context, or in other words the reader tree's root context. - */ - public static IndexReaderContext getTopLevelContext(IndexReaderContext context) { - while (context.parent != null) { - context = context.parent; - } - return context; - } - - /** - * Returns index of the searcher/reader for document n in the - * array used to construct this searcher/reader. - */ - public static int subIndex(int n, int[] docStarts) { // find - // searcher/reader for doc n: - int size = docStarts.length; - int lo = 0; // search starts array - int hi = size - 1; // for first element less than n, return its index - while (hi >= lo) { - int mid = (lo + hi) >>> 1; - int midValue = docStarts[mid]; - if (n < midValue) - hi = mid - 1; - else if (n > midValue) - lo = mid + 1; - else { // found a match - while (mid + 1 < size && docStarts[mid + 1] == midValue) { - mid++; // scan to last match - } - return mid; - } - } - return hi; - } - - /** - * Returns index of the searcher/reader for document n in the - * array used to construct this searcher/reader. - */ - public static int subIndex(int n, AtomicReaderContext[] leaves) { // find - // searcher/reader for doc n: - int size = leaves.length; - int lo = 0; // search starts array - int hi = size - 1; // for first element less than n, return its index - while (hi >= lo) { - int mid = (lo + hi) >>> 1; - int midValue = leaves[mid].docBase; - if (n < midValue) - hi = mid - 1; - else if (n > midValue) - lo = mid + 1; - else { // found a match - while (mid + 1 < size && leaves[mid + 1].docBase == midValue) { - mid++; // scan to last match - } - return mid; - } - } - return hi; - } -} diff --git a/lucene/core/src/java/org/apache/lucene/util/RecyclingByteBlockAllocator.java b/lucene/core/src/java/org/apache/lucene/util/RecyclingByteBlockAllocator.java index f4938ee..6fd2b79 100644 --- a/lucene/core/src/java/org/apache/lucene/util/RecyclingByteBlockAllocator.java +++ b/lucene/core/src/java/org/apache/lucene/util/RecyclingByteBlockAllocator.java @@ -4,7 +4,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.lucene.util.ByteBlockPool.Allocator; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -44,8 +44,6 @@ public final class RecyclingByteBlockAllocator extends ByteBlockPool.Allocator { * maximum number of buffered byte block * @param bytesUsed * {@link AtomicLong} reference counting internally allocated bytes - * - * @see DummyConcurrentLock */ public RecyclingByteBlockAllocator(int blockSize, int maxBufferedBlocks, AtomicLong bytesUsed) { @@ -56,8 +54,7 @@ public final class RecyclingByteBlockAllocator extends ByteBlockPool.Allocator { } /** - * Creates a new {@link RecyclingByteBlockAllocator} with a - * {@link DummyConcurrentLock} instance. + * Creates a new {@link RecyclingByteBlockAllocator}. * * @param blockSize * the block size in bytes @@ -71,8 +68,7 @@ public final class RecyclingByteBlockAllocator extends ByteBlockPool.Allocator { /** * Creates a new {@link RecyclingByteBlockAllocator} with a block size of * {@link ByteBlockPool#BYTE_BLOCK_SIZE}, upper buffered docs limit of - * {@link #DEFAULT_BUFFERED_BLOCKS} ({@value #DEFAULT_BUFFERED_BLOCKS}) and a - * {@link DummyConcurrentLock} instance. + * {@link #DEFAULT_BUFFERED_BLOCKS} ({@value #DEFAULT_BUFFERED_BLOCKS}). * */ public RecyclingByteBlockAllocator() { diff --git a/lucene/core/src/java/org/apache/lucene/util/RollingCharBuffer.java b/lucene/core/src/java/org/apache/lucene/util/RollingCharBuffer.java index 04fc4a4..eaddb17 100644 --- a/lucene/core/src/java/org/apache/lucene/util/RollingCharBuffer.java +++ b/lucene/core/src/java/org/apache/lucene/util/RollingCharBuffer.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/ScorerDocQueue.java b/lucene/core/src/java/org/apache/lucene/util/ScorerDocQueue.java index 952672b..c53aa0b 100755 --- a/lucene/core/src/java/org/apache/lucene/util/ScorerDocQueue.java +++ b/lucene/core/src/java/org/apache/lucene/util/ScorerDocQueue.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/SetOnce.java b/lucene/core/src/java/org/apache/lucene/util/SetOnce.java index f280ee7..ac5cf19 100644 --- a/lucene/core/src/java/org/apache/lucene/util/SetOnce.java +++ b/lucene/core/src/java/org/apache/lucene/util/SetOnce.java @@ -2,7 +2,7 @@ package org.apache.lucene.util; import java.util.concurrent.atomic.AtomicBoolean; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/SortedVIntList.java b/lucene/core/src/java/org/apache/lucene/util/SortedVIntList.java index 1b6957a..135d0ad 100644 --- a/lucene/core/src/java/org/apache/lucene/util/SortedVIntList.java +++ b/lucene/core/src/java/org/apache/lucene/util/SortedVIntList.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/SorterTemplate.java b/lucene/core/src/java/org/apache/lucene/util/SorterTemplate.java index 1ce4619..0ab243e 100644 --- a/lucene/core/src/java/org/apache/lucene/util/SorterTemplate.java +++ b/lucene/core/src/java/org/apache/lucene/util/SorterTemplate.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/StringHelper.java b/lucene/core/src/java/org/apache/lucene/util/StringHelper.java index f43bd49..b1e1718 100644 --- a/lucene/core/src/java/org/apache/lucene/util/StringHelper.java +++ b/lucene/core/src/java/org/apache/lucene/util/StringHelper.java @@ -3,7 +3,7 @@ package org.apache.lucene.util; import java.util.Comparator; import java.util.StringTokenizer; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/TermContext.java b/lucene/core/src/java/org/apache/lucene/util/TermContext.java deleted file mode 100644 index 5a67772..0000000 --- a/lucene/core/src/java/org/apache/lucene/util/TermContext.java +++ /dev/null @@ -1,173 +0,0 @@ -package org.apache.lucene.util; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Arrays; - -import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.index.Fields; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexReaderContext; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermState; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; - -/** - * Maintains a {@link IndexReader} {@link TermState} view over - * {@link IndexReader} instances containing a single term. The - * {@link TermContext} doesn't track if the given {@link TermState} - * objects are valid, neither if the {@link TermState} instances refer to the - * same terms in the associated readers. - * - * @lucene.experimental - */ -public final class TermContext { - public final IndexReaderContext topReaderContext; // for asserting! - private final TermState[] states; - private int docFreq; - private long totalTermFreq; - - //public static boolean DEBUG = BlockTreeTermsWriter.DEBUG; - - /** - * Creates an empty {@link TermContext} from a {@link IndexReaderContext} - */ - public TermContext(IndexReaderContext context) { - assert context != null && context.isTopLevel; - topReaderContext = context; - docFreq = 0; - final int len; - if (context.leaves() == null) { - len = 1; - } else { - len = context.leaves().length; - } - states = new TermState[len]; - } - - /** - * Creates a {@link TermContext} with an initial {@link TermState}, - * {@link IndexReader} pair. - */ - public TermContext(IndexReaderContext context, TermState state, int ord, int docFreq, long totalTermFreq) { - this(context); - register(state, ord, docFreq, totalTermFreq); - } - - /** - * Creates a {@link TermContext} from a top-level {@link IndexReaderContext} and the - * given {@link Term}. This method will lookup the given term in all context's leaf readers - * and register each of the readers containing the term in the returned {@link TermContext} - * using the leaf reader's ordinal. - *

      - * Note: the given context must be a top-level context. - */ - public static TermContext build(IndexReaderContext context, Term term, boolean cache) - throws IOException { - assert context != null && context.isTopLevel; - final String field = term.field(); - final BytesRef bytes = term.bytes(); - final TermContext perReaderTermState = new TermContext(context); - final AtomicReaderContext[] leaves = context.leaves(); - //if (DEBUG) System.out.println("prts.build term=" + term); - for (int i = 0; i < leaves.length; i++) { - //if (DEBUG) System.out.println(" r=" + leaves[i].reader); - final Fields fields = leaves[i].reader().fields(); - if (fields != null) { - final Terms terms = fields.terms(field); - if (terms != null) { - final TermsEnum termsEnum = terms.iterator(null); - if (termsEnum.seekExact(bytes, cache)) { - final TermState termState = termsEnum.termState(); - //if (DEBUG) System.out.println(" found"); - perReaderTermState.register(termState, leaves[i].ord, termsEnum.docFreq(), termsEnum.totalTermFreq()); - } - } - } - } - return perReaderTermState; - } - - /** - * Clears the {@link TermContext} internal state and removes all - * registered {@link TermState}s - */ - public void clear() { - docFreq = 0; - Arrays.fill(states, null); - } - - /** - * Registers and associates a {@link TermState} with an leaf ordinal. The leaf ordinal - * should be derived from a {@link IndexReaderContext}'s leaf ord. - */ - public void register(TermState state, final int ord, final int docFreq, final long totalTermFreq) { - assert state != null : "state must not be null"; - assert ord >= 0 && ord < states.length; - assert states[ord] == null : "state for ord: " + ord - + " already registered"; - this.docFreq += docFreq; - if (this.totalTermFreq >= 0 && totalTermFreq >= 0) - this.totalTermFreq += totalTermFreq; - else - this.totalTermFreq = -1; - states[ord] = state; - } - - /** - * Returns the {@link TermState} for an leaf ordinal or null if no - * {@link TermState} for the ordinal was registered. - * - * @param ord - * the readers leaf ordinal to get the {@link TermState} for. - * @return the {@link TermState} for the given readers ord or null if no - * {@link TermState} for the reader was registered - */ - public TermState get(int ord) { - assert ord >= 0 && ord < states.length; - return states[ord]; - } - - /** - * Returns the accumulated document frequency of all {@link TermState} - * instances passed to {@link #register(TermState, int, int, long)}. - * @return the accumulated document frequency of all {@link TermState} - * instances passed to {@link #register(TermState, int, int, long)}. - */ - public int docFreq() { - return docFreq; - } - - /** - * Returns the accumulated term frequency of all {@link TermState} - * instances passed to {@link #register(TermState, int, int, long)}. - * @return the accumulated term frequency of all {@link TermState} - * instances passed to {@link #register(TermState, int, int, long)}. - */ - public long totalTermFreq() { - return totalTermFreq; - } - - /** expert: only available for queries that want to lie about docfreq - * @lucene.internal */ - public void setDocFreq(int docFreq) { - this.docFreq = docFreq; - } -} \ No newline at end of file diff --git a/lucene/core/src/java/org/apache/lucene/util/ThreadInterruptedException.java b/lucene/core/src/java/org/apache/lucene/util/ThreadInterruptedException.java index 51bd333..e97ebe9 100644 --- a/lucene/core/src/java/org/apache/lucene/util/ThreadInterruptedException.java +++ b/lucene/core/src/java/org/apache/lucene/util/ThreadInterruptedException.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/ToStringUtils.java b/lucene/core/src/java/org/apache/lucene/util/ToStringUtils.java index 2f1d06b..9e9330d 100644 --- a/lucene/core/src/java/org/apache/lucene/util/ToStringUtils.java +++ b/lucene/core/src/java/org/apache/lucene/util/ToStringUtils.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/TwoPhaseCommit.java b/lucene/core/src/java/org/apache/lucene/util/TwoPhaseCommit.java deleted file mode 100644 index efd2d69..0000000 --- a/lucene/core/src/java/org/apache/lucene/util/TwoPhaseCommit.java +++ /dev/null @@ -1,77 +0,0 @@ -package org.apache.lucene.util; - -import java.io.IOException; -import java.util.Map; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * An interface for implementations that support 2-phase commit. You can use - * {@link TwoPhaseCommitTool} to execute a 2-phase commit algorithm over several - * {@link TwoPhaseCommit}s. - * - * @lucene.experimental - */ -public interface TwoPhaseCommit { - - /** - * The first stage of a 2-phase commit. Implementations should do as much work - * as possible in this method, but avoid actual committing changes. If the - * 2-phase commit fails, {@link #rollback()} is called to discard all changes - * since last successful commit. - */ - public void prepareCommit() throws IOException; - - /** - * Like {@link #commit()}, but takes an additional commit data to be included - * w/ the commit. - *

      - * NOTE: some implementations may not support any custom data to be - * included w/ the commit and may discard it altogether. Consult the actual - * implementation documentation for verifying if this is supported. - * - * @see #prepareCommit() - */ - public void prepareCommit(Map commitData) throws IOException; - - /** - * The second phase of a 2-phase commit. Implementations should ideally do - * very little work in this method (following {@link #prepareCommit()}, and - * after it returns, the caller can assume that the changes were successfully - * committed to the underlying storage. - */ - public void commit() throws IOException; - - /** - * Like {@link #commit()}, but takes an additional commit data to be included - * w/ the commit. - * - * @see #commit() - * @see #prepareCommit(Map) - */ - public void commit(Map commitData) throws IOException; - - /** - * Discards any changes that have occurred since the last commit. In a 2-phase - * commit algorithm, where one of the objects failed to {@link #commit()} or - * {@link #prepareCommit()}, this method is used to roll all other objects - * back to their previous state. - */ - public void rollback() throws IOException; - -} diff --git a/lucene/core/src/java/org/apache/lucene/util/TwoPhaseCommitTool.java b/lucene/core/src/java/org/apache/lucene/util/TwoPhaseCommitTool.java deleted file mode 100644 index 66f3526..0000000 --- a/lucene/core/src/java/org/apache/lucene/util/TwoPhaseCommitTool.java +++ /dev/null @@ -1,162 +0,0 @@ -package org.apache.lucene.util; - -import java.io.IOException; -import java.util.Map; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * A utility for executing 2-phase commit on several objects. - * - * @see TwoPhaseCommit - * @lucene.experimental - */ -public final class TwoPhaseCommitTool { - - /** - * A wrapper of a {@link TwoPhaseCommit}, which delegates all calls to the - * wrapped object, passing the specified commitData. This object is useful for - * use with {@link TwoPhaseCommitTool#execute(TwoPhaseCommit...)} if one would - * like to store commitData as part of the commit. - */ - public static final class TwoPhaseCommitWrapper implements TwoPhaseCommit { - - private final TwoPhaseCommit tpc; - private final Map commitData; - - public TwoPhaseCommitWrapper(TwoPhaseCommit tpc, Map commitData) { - this.tpc = tpc; - this.commitData = commitData; - } - - public void prepareCommit() throws IOException { - prepareCommit(commitData); - } - - public void prepareCommit(Map commitData) throws IOException { - tpc.prepareCommit(this.commitData); - } - - public void commit() throws IOException { - commit(commitData); - } - - public void commit(Map commitData) throws IOException { - tpc.commit(this.commitData); - } - - public void rollback() throws IOException { - tpc.rollback(); - } - } - - /** - * Thrown by {@link TwoPhaseCommitTool#execute(TwoPhaseCommit...)} when an - * object fails to prepareCommit(). - */ - public static class PrepareCommitFailException extends IOException { - - public PrepareCommitFailException(Throwable cause, TwoPhaseCommit obj) { - super("prepareCommit() failed on " + obj); - initCause(cause); - } - - } - - /** - * Thrown by {@link TwoPhaseCommitTool#execute(TwoPhaseCommit...)} when an - * object fails to commit(). - */ - public static class CommitFailException extends IOException { - - public CommitFailException(Throwable cause, TwoPhaseCommit obj) { - super("commit() failed on " + obj); - initCause(cause); - } - - } - - /** rollback all objects, discarding any exceptions that occur. */ - private static void rollback(TwoPhaseCommit... objects) { - for (TwoPhaseCommit tpc : objects) { - // ignore any exception that occurs during rollback - we want to ensure - // all objects are rolled-back. - if (tpc != null) { - try { - tpc.rollback(); - } catch (Throwable t) {} - } - } - } - - /** - * Executes a 2-phase commit algorithm by first - * {@link TwoPhaseCommit#prepareCommit()} all objects and only if all succeed, - * it proceeds with {@link TwoPhaseCommit#commit()}. If any of the objects - * fail on either the preparation or actual commit, it terminates and - * {@link TwoPhaseCommit#rollback()} all of them. - *

      - * NOTE: it may happen that an object fails to commit, after few have - * already successfully committed. This tool will still issue a rollback - * instruction on them as well, but depending on the implementation, it may - * not have any effect. - *

      - * NOTE: if any of the objects are {@code null}, this method simply - * skips over them. - * - * @throws PrepareCommitFailException - * if any of the objects fail to - * {@link TwoPhaseCommit#prepareCommit()} - * @throws CommitFailException - * if any of the objects fail to {@link TwoPhaseCommit#commit()} - */ - public static void execute(TwoPhaseCommit... objects) - throws PrepareCommitFailException, CommitFailException { - TwoPhaseCommit tpc = null; - try { - // first, all should successfully prepareCommit() - for (int i = 0; i < objects.length; i++) { - tpc = objects[i]; - if (tpc != null) { - tpc.prepareCommit(); - } - } - } catch (Throwable t) { - // first object that fails results in rollback all of them and - // throwing an exception. - rollback(objects); - throw new PrepareCommitFailException(t, tpc); - } - - // If all successfully prepareCommit(), attempt the actual commit() - try { - for (int i = 0; i < objects.length; i++) { - tpc = objects[i]; - if (tpc != null) { - tpc.commit(); - } - } - } catch (Throwable t) { - // first object that fails results in rollback all of them and - // throwing an exception. - rollback(objects); - throw new CommitFailException(t, tpc); - } - } - -} diff --git a/lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java b/lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java index e81a607..afbe7c6 100644 --- a/lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java +++ b/lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -588,7 +588,7 @@ public final class UnicodeUtil { out[out_offset++] = (char)(((b&0xf)<<12) + ((utf8[offset]&0x3f)<<6) + (utf8[offset+1]&0x3f)); offset += 2; } else { - assert b < 0xf8: "b=" + b; + assert b < 0xf8: "b = 0x" + Integer.toHexString(b); int ch = ((b&0x7)<<18) + ((utf8[offset]&0x3f)<<12) + ((utf8[offset+1]&0x3f)<<6) + (utf8[offset+2]&0x3f); offset += 3; if (ch < UNI_MAX_BMP) { diff --git a/lucene/core/src/java/org/apache/lucene/util/Version.java b/lucene/core/src/java/org/apache/lucene/util/Version.java index 7b364fe..194d59a 100644 --- a/lucene/core/src/java/org/apache/lucene/util/Version.java +++ b/lucene/core/src/java/org/apache/lucene/util/Version.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -32,62 +32,20 @@ import java.util.Locale; // remove me when java 5 is no longer supported // this is a workaround for a JDK bug that wrongly emits a warning. @SuppressWarnings("dep-ann") -public enum Version { - /** - * Match settings and bugs in Lucene's 3.0 release. - * @deprecated (4.0) Use latest - */ - @Deprecated - LUCENE_30, - - /** - * Match settings and bugs in Lucene's 3.1 release. - * @deprecated (4.0) Use latest - */ - @Deprecated - LUCENE_31, - +public enum Version { /** - * Match settings and bugs in Lucene's 3.2 release. - * @deprecated (4.0) Use latest + * Match settings and bugs in Lucene's 4.0 release. + * @deprecated (5.0) Use latest */ @Deprecated - LUCENE_32, - - /** - * Match settings and bugs in Lucene's 3.3 release. - * @deprecated (4.0) Use latest - */ - @Deprecated - LUCENE_33, - - /** - * Match settings and bugs in Lucene's 3.4 release. - * @deprecated (4.0) Use latest - */ - @Deprecated - LUCENE_34, - - /** - * Match settings and bugs in Lucene's 3.5 release. - * @deprecated (4.0) Use latest - */ - @Deprecated - LUCENE_35, - - /** - * Match settings and bugs in Lucene's 3.6 release. - * @deprecated (4.0) Use latest - */ - @Deprecated - LUCENE_36, + LUCENE_40, - /** Match settings and bugs in Lucene's 4.0 release. + /** Match settings and bugs in Lucene's 5.0 release. *

      * Use this to get the latest & greatest settings, bug * fixes, etc, for Lucene. */ - LUCENE_40, + LUCENE_50, /* Add new constants for later versions **here** to respect order! */ diff --git a/lucene/core/src/java/org/apache/lucene/util/VirtualMethod.java b/lucene/core/src/java/org/apache/lucene/util/VirtualMethod.java index c5590a7..ec55374 100644 --- a/lucene/core/src/java/org/apache/lucene/util/VirtualMethod.java +++ b/lucene/core/src/java/org/apache/lucene/util/VirtualMethod.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/WeakIdentityMap.java b/lucene/core/src/java/org/apache/lucene/util/WeakIdentityMap.java index 56149e4..f332ef0 100644 --- a/lucene/core/src/java/org/apache/lucene/util/WeakIdentityMap.java +++ b/lucene/core/src/java/org/apache/lucene/util/WeakIdentityMap.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,7 +21,9 @@ import java.lang.ref.Reference; import java.lang.ref.ReferenceQueue; import java.lang.ref.WeakReference; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; +import java.util.NoSuchElementException; import java.util.concurrent.ConcurrentHashMap; /** @@ -38,9 +40,10 @@ import java.util.concurrent.ConcurrentHashMap; * *

      This implementation was forked from Apache CXF * but modified to not implement the {@link java.util.Map} interface and - * without any set/iterator views on it, as those are error-prone - * and inefficient, if not implemented carefully. Lucene's implementation also - * supports {@code null} keys, but those are never weak! + * without any set views on it, as those are error-prone and inefficient, + * if not implemented carefully. The map only contains {@link Iterator} implementations + * on the values and not-GCed keys. Lucene's implementation also supports {@code null} + * keys, but those are never weak! * * @lucene.internal */ @@ -97,6 +100,70 @@ public final class WeakIdentityMap { reap(); return backingStore.size(); } + + /** Returns an iterator over all weak keys of this map. + * Keys already garbage collected will not be returned. + * This Iterator does not support removals. */ + public Iterator keyIterator() { + reap(); + final Iterator iterator = backingStore.keySet().iterator(); + return new Iterator() { + // holds strong reference to next element in backing iterator: + private Object next = null; + // the backing iterator was already consumed: + private boolean nextIsSet = false; + + @Override + public boolean hasNext() { + return nextIsSet ? true : setNext(); + } + + @Override @SuppressWarnings("unchecked") + public K next() { + if (nextIsSet || setNext()) { + try { + assert nextIsSet; + return (K) next; + } finally { + // release strong reference and invalidate current value: + nextIsSet = false; + next = null; + } + } + throw new NoSuchElementException(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + private boolean setNext() { + assert !nextIsSet; + while (iterator.hasNext()) { + next = iterator.next().get(); + if (next == null) { + // already garbage collected! + continue; + } + // unfold "null" special value + if (next == NULL) { + next = null; + } + return nextIsSet = true; + } + return false; + } + }; + } + + /** Returns an iterator over all values of this map. + * This iterator may return values whose key is already + * garbage collected while iterator is consumed. */ + public Iterator valueIterator() { + reap(); + return backingStore.values().iterator(); + } private void reap() { Reference zombie; @@ -104,6 +171,9 @@ public final class WeakIdentityMap { backingStore.remove(zombie); } } + + // we keep a hard reference to our NULL key, so map supports null keys that never get GCed: + static final Object NULL = new Object(); private static final class IdentityWeakReference extends WeakReference { private final int hash; @@ -129,9 +199,6 @@ public final class WeakIdentityMap { } return false; } - - // we keep a hard reference to our NULL key, so map supports null keys that never get GCed: - private static final Object NULL = new Object(); } } diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/BasicAutomata.java b/lucene/core/src/java/org/apache/lucene/util/automaton/BasicAutomata.java index 7db891a..128c7d9 100644 --- a/lucene/core/src/java/org/apache/lucene/util/automaton/BasicAutomata.java +++ b/lucene/core/src/java/org/apache/lucene/util/automaton/BasicAutomata.java @@ -29,8 +29,9 @@ package org.apache.lucene.util.automaton; -import java.util.ArrayList; -import java.util.Collection; +import java.util.*; + +import org.apache.lucene.util.BytesRef; /** * Construction of basic automata. @@ -239,4 +240,25 @@ final public class BasicAutomata { a.deterministic = true; return a; } + + /** + * Returns a new (deterministic and minimal) automaton that accepts the union + * of the given collection of {@link BytesRef}s representing UTF-8 encoded + * strings. + * + * @param utf8Strings + * The input strings, UTF-8 encoded. The collection must be in sorted + * order. + * + * @return An {@link Automaton} accepting all input strings. The resulting + * automaton is codepoint based (full unicode codepoints on + * transitions). + */ + public static Automaton makeStringUnion(Collection utf8Strings) { + if (utf8Strings.isEmpty()) { + return makeEmpty(); + } else { + return DaciukMihovAutomatonBuilder.build(utf8Strings); + } + } } diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/ByteRunAutomaton.java b/lucene/core/src/java/org/apache/lucene/util/automaton/ByteRunAutomaton.java index e8191a7..8c8d68a 100644 --- a/lucene/core/src/java/org/apache/lucene/util/automaton/ByteRunAutomaton.java +++ b/lucene/core/src/java/org/apache/lucene/util/automaton/ByteRunAutomaton.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.automaton; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/CharacterRunAutomaton.java b/lucene/core/src/java/org/apache/lucene/util/automaton/CharacterRunAutomaton.java index d7675e4..2dcd922 100644 --- a/lucene/core/src/java/org/apache/lucene/util/automaton/CharacterRunAutomaton.java +++ b/lucene/core/src/java/org/apache/lucene/util/automaton/CharacterRunAutomaton.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.automaton; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java b/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java index 8fa2a6c..d606116 100644 --- a/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java +++ b/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.automaton; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java b/lucene/core/src/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java new file mode 100644 index 0000000..c6be628 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java @@ -0,0 +1,332 @@ +package org.apache.lucene.util.automaton; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.*; + +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CharsRef; +import org.apache.lucene.util.UnicodeUtil; + +/** + * Builds a minimal, deterministic {@link Automaton} that accepts a set of + * strings. The algorithm requires sorted input data, but is very fast + * (nearly linear with the input size). + * + * @see #build(Collection) + * @see BasicAutomata#makeStringUnion(Collection) + */ +final class DaciukMihovAutomatonBuilder { + /** + * DFSA state with char labels on transitions. + */ + private final static class State { + + /** An empty set of labels. */ + private final static int[] NO_LABELS = new int[0]; + + /** An empty set of states. */ + private final static State[] NO_STATES = new State[0]; + + /** + * Labels of outgoing transitions. Indexed identically to {@link #states}. + * Labels must be sorted lexicographically. + */ + int[] labels = NO_LABELS; + + /** + * States reachable from outgoing transitions. Indexed identically to + * {@link #labels}. + */ + State[] states = NO_STATES; + + /** + * true if this state corresponds to the end of at least one + * input sequence. + */ + boolean is_final; + + /** + * Returns the target state of a transition leaving this state and labeled + * with label. If no such transition exists, returns + * null. + */ + State getState(int label) { + final int index = Arrays.binarySearch(labels, label); + return index >= 0 ? states[index] : null; + } + + /** + * Two states are equal if: + *
        + *
      • they have an identical number of outgoing transitions, labeled with + * the same labels
        • + *
      • corresponding outgoing transitions lead to the same states (to states + * with an identical right-language). + *
      + */ + @Override + public boolean equals(Object obj) { + final State other = (State) obj; + return is_final == other.is_final + && Arrays.equals(this.labels, other.labels) + && referenceEquals(this.states, other.states); + } + + /** + * Compute the hash code of the current status of this state. + */ + @Override + public int hashCode() { + int hash = is_final ? 1 : 0; + + hash ^= hash * 31 + this.labels.length; + for (int c : this.labels) + hash ^= hash * 31 + c; + + /* + * Compare the right-language of this state using reference-identity of + * outgoing states. This is possible because states are interned (stored + * in registry) and traversed in post-order, so any outgoing transitions + * are already interned. + */ + for (State s : this.states) { + hash ^= System.identityHashCode(s); + } + + return hash; + } + + /** + * Return true if this state has any children (outgoing + * transitions). + */ + boolean hasChildren() { + return labels.length > 0; + } + + /** + * Create a new outgoing transition labeled label and return + * the newly created target state for this transition. + */ + State newState(int label) { + assert Arrays.binarySearch(labels, label) < 0 : "State already has transition labeled: " + + label; + + labels = Arrays.copyOf(labels, labels.length + 1); + states = Arrays.copyOf(states, states.length + 1); + + labels[labels.length - 1] = label; + return states[states.length - 1] = new State(); + } + + /** + * Return the most recent transitions's target state. + */ + State lastChild() { + assert hasChildren() : "No outgoing transitions."; + return states[states.length - 1]; + } + + /** + * Return the associated state if the most recent transition is labeled with + * label. + */ + State lastChild(int label) { + final int index = labels.length - 1; + State s = null; + if (index >= 0 && labels[index] == label) { + s = states[index]; + } + assert s == getState(label); + return s; + } + + /** + * Replace the last added outgoing transition's target state with the given + * state. + */ + void replaceLastChild(State state) { + assert hasChildren() : "No outgoing transitions."; + states[states.length - 1] = state; + } + + /** + * Compare two lists of objects for reference-equality. + */ + private static boolean referenceEquals(Object[] a1, Object[] a2) { + if (a1.length != a2.length) { + return false; + } + + for (int i = 0; i < a1.length; i++) { + if (a1[i] != a2[i]) { + return false; + } + } + + return true; + } + } + + /** + * A "registry" for state interning. + */ + private HashMap stateRegistry = new HashMap(); + + /** + * Root automaton state. + */ + private State root = new State(); + + /** + * Previous sequence added to the automaton in {@link #add(CharsRef)}. + */ + private CharsRef previous; + + /** + * A comparator used for enforcing sorted UTF8 order, used in assertions only. + */ + @SuppressWarnings("deprecation") + private static final Comparator comparator = CharsRef.getUTF16SortedAsUTF8Comparator(); + + /** + * Add another character sequence to this automaton. The sequence must be + * lexicographically larger or equal compared to any previous sequences added + * to this automaton (the input must be sorted). + */ + public void add(CharsRef current) { + assert stateRegistry != null : "Automaton already built."; + assert previous == null + || comparator.compare(previous, current) <= 0 : "Input must be in sorted UTF-8 order: " + + previous + " >= " + current; + assert setPrevious(current); + + // Descend in the automaton (find matching prefix). + int pos = 0, max = current.length(); + State next, state = root; + while (pos < max && (next = state.lastChild(Character.codePointAt(current, pos))) != null) { + state = next; + // todo, optimize me + pos += Character.charCount(Character.codePointAt(current, pos)); + } + + if (state.hasChildren()) replaceOrRegister(state); + + addSuffix(state, current, pos); + } + + /** + * Finalize the automaton and return the root state. No more strings can be + * added to the builder after this call. + * + * @return Root automaton state. + */ + public State complete() { + if (this.stateRegistry == null) throw new IllegalStateException(); + + if (root.hasChildren()) replaceOrRegister(root); + + stateRegistry = null; + return root; + } + + /** + * Internal recursive traversal for conversion. + */ + private static org.apache.lucene.util.automaton.State convert(State s, + IdentityHashMap visited) { + org.apache.lucene.util.automaton.State converted = visited.get(s); + if (converted != null) return converted; + + converted = new org.apache.lucene.util.automaton.State(); + converted.setAccept(s.is_final); + + visited.put(s, converted); + int i = 0; + int[] labels = s.labels; + for (DaciukMihovAutomatonBuilder.State target : s.states) { + converted.addTransition( + new Transition(labels[i++], convert(target, visited))); + } + + return converted; + } + + /** + * Build a minimal, deterministic automaton from a sorted list of {@link BytesRef} representing + * strings in UTF-8. These strings must be binary-sorted. + */ + public static Automaton build(Collection input) { + final DaciukMihovAutomatonBuilder builder = new DaciukMihovAutomatonBuilder(); + + CharsRef scratch = new CharsRef(); + for (BytesRef b : input) { + UnicodeUtil.UTF8toUTF16(b, scratch); + builder.add(scratch); + } + + Automaton a = new Automaton(); + a.initial = convert( + builder.complete(), + new IdentityHashMap()); + a.deterministic = true; + return a; + } + + /** + * Copy current into an internal buffer. + */ + private boolean setPrevious(CharsRef current) { + // don't need to copy, once we fix https://issues.apache.org/jira/browse/LUCENE-3277 + // still, called only from assert + previous = CharsRef.deepCopyOf(current); + return true; + } + + /** + * Replace last child of state with an already registered state + * or stateRegistry the last child state. + */ + private void replaceOrRegister(State state) { + final State child = state.lastChild(); + + if (child.hasChildren()) replaceOrRegister(child); + + final State registered = stateRegistry.get(child); + if (registered != null) { + state.replaceLastChild(registered); + } else { + stateRegistry.put(child, child); + } + } + + /** + * Add a suffix of current starting at fromIndex + * (inclusive) to state state. + */ + private void addSuffix(State state, CharSequence current, int fromIndex) { + final int len = current.length(); + while (fromIndex < len) { + int cp = Character.codePointAt(current, fromIndex); + state = state.newState(cp); + fromIndex += Character.charCount(cp); + } + state.is_final = true; + } +} diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/Lev1ParametricDescription.java b/lucene/core/src/java/org/apache/lucene/util/automaton/Lev1ParametricDescription.java index 608b464..51e244c 100644 --- a/lucene/core/src/java/org/apache/lucene/util/automaton/Lev1ParametricDescription.java +++ b/lucene/core/src/java/org/apache/lucene/util/automaton/Lev1ParametricDescription.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.automaton; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/Lev1TParametricDescription.java b/lucene/core/src/java/org/apache/lucene/util/automaton/Lev1TParametricDescription.java index 9274c19..49d0abc 100644 --- a/lucene/core/src/java/org/apache/lucene/util/automaton/Lev1TParametricDescription.java +++ b/lucene/core/src/java/org/apache/lucene/util/automaton/Lev1TParametricDescription.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.automaton; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/Lev2ParametricDescription.java b/lucene/core/src/java/org/apache/lucene/util/automaton/Lev2ParametricDescription.java index 83912da..9869cfc 100644 --- a/lucene/core/src/java/org/apache/lucene/util/automaton/Lev2ParametricDescription.java +++ b/lucene/core/src/java/org/apache/lucene/util/automaton/Lev2ParametricDescription.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.automaton; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/Lev2TParametricDescription.java b/lucene/core/src/java/org/apache/lucene/util/automaton/Lev2TParametricDescription.java index 32d7dd5..2792130 100644 --- a/lucene/core/src/java/org/apache/lucene/util/automaton/Lev2TParametricDescription.java +++ b/lucene/core/src/java/org/apache/lucene/util/automaton/Lev2TParametricDescription.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.automaton; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/LevenshteinAutomata.java b/lucene/core/src/java/org/apache/lucene/util/automaton/LevenshteinAutomata.java index e26f71e..9621142 100644 --- a/lucene/core/src/java/org/apache/lucene/util/automaton/LevenshteinAutomata.java +++ b/lucene/core/src/java/org/apache/lucene/util/automaton/LevenshteinAutomata.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.automaton; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/SortedIntSet.java b/lucene/core/src/java/org/apache/lucene/util/automaton/SortedIntSet.java index d1f3e28..0cfeae6 100644 --- a/lucene/core/src/java/org/apache/lucene/util/automaton/SortedIntSet.java +++ b/lucene/core/src/java/org/apache/lucene/util/automaton/SortedIntSet.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.automaton; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/UTF32ToUTF8.java b/lucene/core/src/java/org/apache/lucene/util/automaton/UTF32ToUTF8.java index ce98e2c..6f52a7a 100644 --- a/lucene/core/src/java/org/apache/lucene/util/automaton/UTF32ToUTF8.java +++ b/lucene/core/src/java/org/apache/lucene/util/automaton/UTF32ToUTF8.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.automaton; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/Builder.java b/lucene/core/src/java/org/apache/lucene/util/fst/Builder.java index b4b9d1d..ea9fd45 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/Builder.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/Builder.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,6 +23,7 @@ import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.IntsRef; import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.fst.FST.INPUT_TYPE; // javadoc +import org.apache.lucene.util.packed.PackedInts; /** * Builds a minimal FST (maps an IntsRef term to an arbitrary @@ -83,7 +84,18 @@ public class Builder { * pruning options turned off. */ public Builder(FST.INPUT_TYPE inputType, Outputs outputs) { - this(inputType, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, false); + this(inputType, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, false, PackedInts.COMPACT); + } + + /** + * Instantiates an FST/FSA builder with {@link PackedInts#DEFAULT} + * acceptableOverheadRatio. + */ + public Builder(FST.INPUT_TYPE inputType, int minSuffixCount1, int minSuffixCount2, boolean doShareSuffix, + boolean doShareNonSingletonNodes, int shareMaxTailLength, Outputs outputs, + FreezeTail freezeTail, boolean willPackFST) { + this(inputType, minSuffixCount1, minSuffixCount2, doShareSuffix, doShareNonSingletonNodes, + shareMaxTailLength, outputs, freezeTail, willPackFST, PackedInts.DEFAULT); } /** @@ -126,17 +138,20 @@ public class Builder { * @param willPackFST Pass true if you will pack the FST before saving. This * causes the FST to create additional data structures internally to facilitate packing, but * it means the resulting FST cannot be saved: it must - * first be packed using {@link FST#pack(int, int)}}. + * first be packed using {@link FST#pack(int, int, float)} + * + * @param acceptableOverheadRatio How to trade speed for space when building the FST. This option + * is only relevant when willPackFST is true. @see PackedInts#getMutable(int, int, float) */ public Builder(FST.INPUT_TYPE inputType, int minSuffixCount1, int minSuffixCount2, boolean doShareSuffix, boolean doShareNonSingletonNodes, int shareMaxTailLength, Outputs outputs, - FreezeTail freezeTail, boolean willPackFST) { + FreezeTail freezeTail, boolean willPackFST, float acceptableOverheadRatio) { this.minSuffixCount1 = minSuffixCount1; this.minSuffixCount2 = minSuffixCount2; this.freezeTail = freezeTail; this.doShareNonSingletonNodes = doShareNonSingletonNodes; this.shareMaxTailLength = shareMaxTailLength; - fst = new FST(inputType, outputs, willPackFST); + fst = new FST(inputType, outputs, willPackFST, acceptableOverheadRatio); if (doShareSuffix) { dedupHash = new NodeHash(fst); } else { diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java b/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java index d6598ff..0f8ade4 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/BytesRefFSTEnum.java b/lucene/core/src/java/org/apache/lucene/util/fst/BytesRefFSTEnum.java index e4cc48f..baff0b0 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/BytesRefFSTEnum.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/BytesRefFSTEnum.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,6 +19,7 @@ package org.apache.lucene.util.fst; import java.io.IOException; +import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; /** Enumerates all input (BytesRef) + output pairs in an @@ -110,7 +111,7 @@ public final class BytesRefFSTEnum extends FSTEnum { @Override protected void grow() { - current.grow(upto+1); + current.bytes = ArrayUtil.grow(current.bytes, upto+1); } private InputOutput setResult() { diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java b/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java index f24402e..c4bed38 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java index a81f9f0..259eb4f 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -28,17 +28,18 @@ import java.io.OutputStream; import java.util.HashMap; import java.util.Map; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.store.DataInput; import org.apache.lucene.store.DataOutput; import org.apache.lucene.store.InputStreamDataInput; import org.apache.lucene.store.OutputStreamDataOutput; import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IntsRef; import org.apache.lucene.util.PriorityQueue; -import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.fst.Builder.UnCompiledNode; +import org.apache.lucene.util.packed.GrowableWriter; +import org.apache.lucene.util.packed.PackedInts; // TODO: break this into WritableFST and ReadOnlyFST.. then // we can have subclasses of ReadOnlyFST to handle the @@ -155,7 +156,7 @@ public final class FST { public int arcWithOutputCount; private final boolean packed; - private final int[] nodeRefToAddress; + private PackedInts.Reader nodeRefToAddress; // If arc has this label then that arc is final/accepted public static final int END_LABEL = -1; @@ -246,31 +247,29 @@ public final class FST { } }; - private final static boolean flag(int flags, int bit) { + private static boolean flag(int flags, int bit) { return (flags & bit) != 0; } private final BytesWriter writer; - // TODO: we can save RAM here by using growable packed - // ints...: - private int[] nodeAddress; + private GrowableWriter nodeAddress; // TODO: we could be smarter here, and prune periodically // as we go; high in-count nodes will "usually" become // clear early on: - private int[] inCounts; + private GrowableWriter inCounts; // make a new empty FST, for building; Builder invokes // this ctor - FST(INPUT_TYPE inputType, Outputs outputs, boolean willPackFST) { + FST(INPUT_TYPE inputType, Outputs outputs, boolean willPackFST, float acceptableOverheadRatio) { this.inputType = inputType; this.outputs = outputs; bytes = new byte[128]; NO_OUTPUT = outputs.getNoOutput(); if (willPackFST) { - nodeAddress = new int[8]; - inCounts = new int[8]; + nodeAddress = new GrowableWriter(PackedInts.bitsRequired(bytes.length - 1), 8, acceptableOverheadRatio); + inCounts = new GrowableWriter(1, 8, acceptableOverheadRatio); } else { nodeAddress = null; inCounts = null; @@ -320,11 +319,7 @@ public final class FST { throw new IllegalStateException("invalid input type " + t); } if (packed) { - final int nodeRefCount = in.readVInt(); - nodeRefToAddress = new int[nodeRefCount]; - for(int idx=0;idx { public int sizeInBytes() { int size = bytes.length; if (packed) { - size += nodeRefToAddress.length * RamUsageEstimator.NUM_BYTES_INT; + size += nodeRefToAddress.ramBytesUsed(); } else if (nodeAddress != null) { - size += nodeAddress.length * RamUsageEstimator.NUM_BYTES_INT; - size += inCounts.length * RamUsageEstimator.NUM_BYTES_INT; + size += nodeAddress.ramBytesUsed(); + size += inCounts.ramBytesUsed(); } return size; } @@ -374,7 +369,7 @@ public final class FST { private int getNodeAddress(int node) { if (nodeAddress != null) { // Deref - return nodeAddress[node]; + return (int) nodeAddress.get(node); } else { // Straight return node; @@ -444,6 +439,9 @@ public final class FST { if (nodeAddress != null) { throw new IllegalStateException("cannot save an FST pre-packed FST; it must first be packed"); } + if (packed && !(nodeRefToAddress instanceof PackedInts.Mutable)) { + throw new IllegalStateException("cannot save a FST which has been loaded from disk "); + } CodecUtil.writeHeader(out, FILE_FORMAT_NAME, VERSION_CURRENT); if (packed) { out.writeByte((byte) 1); @@ -469,11 +467,7 @@ public final class FST { } out.writeByte(t); if (packed) { - assert nodeRefToAddress != null; - out.writeVInt(nodeRefToAddress.length); - for(int idx=0;idx { if (!targetHasArcs) { flags += BIT_STOP_NODE; } else if (inCounts != null) { - inCounts[target.node]++; + inCounts.set(target.node, inCounts.get(target.node) + 1); } if (arc.output != NO_OUTPUT) { @@ -715,11 +709,11 @@ public final class FST { final int node; if (nodeAddress != null) { // Nodes are addressed by 1+ord: - if (nodeCount == nodeAddress.length) { - nodeAddress = ArrayUtil.grow(nodeAddress); - inCounts = ArrayUtil.grow(inCounts); + if (nodeCount == nodeAddress.size()) { + nodeAddress = nodeAddress.resize(ArrayUtil.oversize(nodeAddress.size() + 1, nodeAddress.getBitsPerValue())); + inCounts = inCounts.resize(ArrayUtil.oversize(inCounts.size() + 1, inCounts.getBitsPerValue())); } - nodeAddress[nodeCount] = endAddress; + nodeAddress.set(nodeCount, endAddress); // System.out.println(" write nodeAddress[" + nodeCount + "] = " + endAddress); node = nodeCount; } else { @@ -755,7 +749,7 @@ public final class FST { * * @return Returns the second argument * (arc). */ - public Arc readLastTargetArc(Arc follow, Arc arc) throws IOException { + public Arc readLastTargetArc(Arc follow, Arc arc, FST.BytesReader in) throws IOException { //System.out.println("readLast"); if (!targetHasArcs(follow)) { //System.out.println(" end node"); @@ -766,7 +760,7 @@ public final class FST { arc.flags = BIT_LAST_ARC; return arc; } else { - final BytesReader in = getBytesReader(getNodeAddress(follow.target)); + in.pos = getNodeAddress(follow.target); arc.node = follow.target; final byte b = in.readByte(); if (b == ARCS_AS_FIXED_ARRAY) { @@ -822,7 +816,7 @@ public final class FST { * * @return Returns the second argument (arc). */ - public Arc readFirstTargetArc(Arc follow, Arc arc) throws IOException { + public Arc readFirstTargetArc(Arc follow, Arc arc, BytesReader in) throws IOException { //int pos = address; //System.out.println(" readFirstTarget follow.target=" + follow.target + " isFinal=" + follow.isFinal()); if (follow.isFinal()) { @@ -841,7 +835,7 @@ public final class FST { //System.out.println(" insert isFinal; nextArc=" + follow.target + " isLast=" + arc.isLast() + " output=" + outputs.outputToString(arc.output)); return arc; } else { - return readFirstRealTargetArc(follow.target, arc, getBytesReader(0)); + return readFirstRealTargetArc(follow.target, arc, in); } } @@ -881,37 +875,36 @@ public final class FST { * @return Returns true if arc points to a state in an * expanded array format. */ - boolean isExpandedTarget(Arc follow) throws IOException { + boolean isExpandedTarget(Arc follow, FST.BytesReader in) throws IOException { if (!targetHasArcs(follow)) { return false; } else { - final BytesReader in = getBytesReader(getNodeAddress(follow.target)); + in.pos = getNodeAddress(follow.target); return in.readByte() == ARCS_AS_FIXED_ARRAY; } } /** In-place read; returns the arc. */ - public Arc readNextArc(Arc arc) throws IOException { + public Arc readNextArc(Arc arc, BytesReader in) throws IOException { if (arc.label == END_LABEL) { // This was a fake inserted "final" arc if (arc.nextArc <= 0) { throw new IllegalArgumentException("cannot readNextArc when arc.isLast()=true"); } - return readFirstRealTargetArc(arc.nextArc, arc, getBytesReader(0)); + return readFirstRealTargetArc(arc.nextArc, arc, in); } else { - return readNextRealArc(arc, getBytesReader(0)); + return readNextRealArc(arc, in); } } /** Peeks at next arc's label; does not alter arc. Do * not call this if arc.isLast()! */ - public int readNextArcLabel(Arc arc) throws IOException { + public int readNextArcLabel(Arc arc, BytesReader in) throws IOException { assert !arc.isLast(); - final BytesReader in; if (arc.label == END_LABEL) { //System.out.println(" nextArc fake " + arc.nextArc); - in = getBytesReader(getNodeAddress(arc.nextArc)); + in.pos = getNodeAddress(arc.nextArc); final byte b = bytes[in.pos]; if (b == ARCS_AS_FIXED_ARRAY) { //System.out.println(" nextArc fake array"); @@ -927,12 +920,12 @@ public final class FST { if (arc.bytesPerArc != 0) { //System.out.println(" nextArc real array"); // arcs are at fixed entries - in = getBytesReader(arc.posArcsStart); + in.pos = arc.posArcsStart; in.skip((1+arc.arcIdx)*arc.bytesPerArc); } else { // arcs are packed //System.out.println(" nextArc real packed"); - in = getBytesReader(arc.nextArc); + in.pos = arc.nextArc; } } // skip flags @@ -1006,9 +999,9 @@ public final class FST { // Address is delta-coded from current address: arc.target = pos + code; //System.out.println(" delta pos=" + pos + " delta=" + code + " target=" + arc.target); - } else if (code < nodeRefToAddress.length) { + } else if (code < nodeRefToAddress.size()) { // Deref - arc.target = nodeRefToAddress[code]; + arc.target = (int) nodeRefToAddress.get(code); //System.out.println(" deref code=" + code + " target=" + arc.target); } else { // Absolute @@ -1223,7 +1216,7 @@ public final class FST { } } - public final BytesReader getBytesReader(int pos) { + public BytesReader getBytesReader(int pos) { // TODO: maybe re-use via ThreadLocal? if (packed) { return new ForwardBytesReader(bytes, pos); @@ -1421,7 +1414,7 @@ public final class FST { */ // Creates a packed FST - private FST(INPUT_TYPE inputType, int[] nodeRefToAddress, Outputs outputs) { + private FST(INPUT_TYPE inputType, PackedInts.Reader nodeRefToAddress, Outputs outputs) { packed = true; this.inputType = inputType; bytes = new byte[128]; @@ -1433,8 +1426,17 @@ public final class FST { /** Expert: creates an FST by packing this one. This * process requires substantial additional RAM (currently - * ~8 bytes per node), but then should produce a smaller FST. */ - public FST pack(int minInCountDeref, int maxDerefNodes) throws IOException { + * up to ~8 bytes per node depending on + * acceptableOverheadRatio), but then should + * produce a smaller FST. + * + *

      The implementation of this method uses ideas from + * Smaller Representation of Finite State Automata, + * which describes techniques to reduce the size of a FST. + * However, this is not a strict implementation of the + * algorithms described in this paper. + */ + public FST pack(int minInCountDeref, int maxDerefNodes, float acceptableOverheadRatio) throws IOException { // TODO: other things to try // - renumber the nodes to get more next / better locality? @@ -1455,22 +1457,22 @@ public final class FST { final BytesReader r = getBytesReader(0); - final int topN = Math.min(maxDerefNodes, inCounts.length); + final int topN = Math.min(maxDerefNodes, inCounts.size()); // Find top nodes with highest number of incoming arcs: NodeQueue q = new NodeQueue(topN); // TODO: we could use more RAM efficient selection algo here... NodeAndInCount bottom = null; - for(int node=0;node= minInCountDeref) { + for(int node=0; node= minInCountDeref) { if (bottom == null) { - q.add(new NodeAndInCount(node, inCounts[node])); + q.add(new NodeAndInCount(node, (int) inCounts.get(node))); if (q.size() == topN) { bottom = q.top(); } - } else if (inCounts[node] > bottom.count) { - q.insertWithOverflow(new NodeAndInCount(node, inCounts[node])); + } else if (inCounts.get(node) > bottom.count) { + q.insertWithOverflow(new NodeAndInCount(node, (int) inCounts.get(node))); } } } @@ -1485,20 +1487,17 @@ public final class FST { //System.out.println("map node=" + n.node + " inCount=" + n.count + " to newID=" + downTo); } - // TODO: we can use packed ints: - // +1 because node ords start at 1 (0 is reserved as - // stop node): - final int[] nodeRefToAddressIn = new int[topNodeMap.size()]; - - final FST fst = new FST(inputType, nodeRefToAddressIn, outputs); + final FST fst = new FST(inputType, null, outputs); final BytesWriter writer = fst.writer; - - final int[] newNodeAddress = new int[1+nodeCount]; + + // +1 because node ords start at 1 (0 is reserved as stop node): + final GrowableWriter newNodeAddress = new GrowableWriter( + PackedInts.bitsRequired(bytes.length), 1 + nodeCount, acceptableOverheadRatio); // Fill initial coarse guess: for(int node=1;node<=nodeCount;node++) { - newNodeAddress[node] = 1 + bytes.length - nodeAddress[node]; + newNodeAddress.set(node, 1 + bytes.length - nodeAddress.get(node)); } int absCount; @@ -1538,11 +1537,11 @@ public final class FST { fst.nodeCount++; final int address = writer.posWrite; //System.out.println(" node: " + node + " address=" + address); - if (address != newNodeAddress[node]) { - addressError = address - newNodeAddress[node]; + if (address != newNodeAddress.get(node)) { + addressError = address - (int) newNodeAddress.get(node); //System.out.println(" change: " + (address - newNodeAddress[node])); changed = true; - newNodeAddress[node] = address; + newNodeAddress.set(node, address); changedCount++; } @@ -1622,10 +1621,10 @@ public final class FST { if (ptr != null) { absPtr = ptr; } else { - absPtr = topNodeMap.size() + newNodeAddress[arc.target] + addressError; + absPtr = topNodeMap.size() + (int) newNodeAddress.get(arc.target) + addressError; } - int delta = newNodeAddress[arc.target] + addressError - writer.posWrite - 2; + int delta = (int) newNodeAddress.get(arc.target) + addressError - writer.posWrite - 2; if (delta < 0) { //System.out.println("neg: " + delta); anyNegDelta = true; @@ -1655,7 +1654,7 @@ public final class FST { if (doWriteTarget) { - int delta = newNodeAddress[arc.target] + addressError - writer.posWrite; + int delta = (int) newNodeAddress.get(arc.target) + addressError - writer.posWrite; if (delta < 0) { anyNegDelta = true; //System.out.println("neg: " + delta); @@ -1746,11 +1745,20 @@ public final class FST { //System.out.println(" " + changedCount + " of " + fst.nodeCount + " changed; retry"); } + long maxAddress = 0; + for (int key : topNodeMap.keySet()) { + maxAddress = Math.max(maxAddress, newNodeAddress.get(key)); + } + + PackedInts.Mutable nodeRefToAddressIn = PackedInts.getMutable(topNodeMap.size(), + PackedInts.bitsRequired(maxAddress), acceptableOverheadRatio); for(Map.Entry ent : topNodeMap.entrySet()) { - nodeRefToAddressIn[ent.getValue()] = newNodeAddress[ent.getKey()]; + nodeRefToAddressIn.set(ent.getValue(), newNodeAddress.get(ent.getKey())); } + fst.nodeRefToAddress = nodeRefToAddressIn; + - fst.startNode = newNodeAddress[startNode]; + fst.startNode = (int) newNodeAddress.get(startNode); //System.out.println("new startNode=" + fst.startNode + " old startNode=" + startNode); if (emptyOutput != null) { diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java b/lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java index b65f180..d4806fb 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -35,6 +35,7 @@ abstract class FSTEnum { @SuppressWarnings({"rawtypes","unchecked"}) protected T[] output = (T[]) new Object[10]; protected final T NO_OUTPUT; + protected final FST.BytesReader fstReader; protected final FST.Arc scratchArc = new FST.Arc(); protected int upto; @@ -45,6 +46,7 @@ abstract class FSTEnum { * term before target. */ protected FSTEnum(FST fst) { this.fst = fst; + fstReader = fst.getBytesReader(0); NO_OUTPUT = fst.outputs.getNoOutput(); fst.getFirstArc(getArc(0)); output[0] = NO_OUTPUT; @@ -62,7 +64,7 @@ abstract class FSTEnum { if (upto == 0) { //System.out.println(" init"); upto = 1; - fst.readFirstTargetArc(getArc(0), getArc(1)); + fst.readFirstTargetArc(getArc(0), getArc(1), fstReader); return; } //System.out.println(" rewind upto=" + upto + " vs targetLength=" + targetLength); @@ -78,7 +80,7 @@ abstract class FSTEnum { } else if (cmp > 0) { // seek backwards -- reset this arc to the first arc final FST.Arc arc = getArc(upto); - fst.readFirstTargetArc(getArc(upto-1), arc); + fst.readFirstTargetArc(getArc(upto-1), arc, fstReader); //System.out.println(" seek first arc"); break; } @@ -92,7 +94,7 @@ abstract class FSTEnum { if (upto == 0) { //System.out.println(" init"); upto = 1; - fst.readFirstTargetArc(getArc(0), getArc(1)); + fst.readFirstTargetArc(getArc(0), getArc(1), fstReader); } else { // pop //System.out.println(" check pop curArc target=" + arcs[upto].target + " label=" + arcs[upto].label + " isLast?=" + arcs[upto].isLast()); @@ -103,7 +105,7 @@ abstract class FSTEnum { return; } } - fst.readNextArc(arcs[upto]); + fst.readNextArc(arcs[upto], fstReader); } pushFirst(); @@ -180,7 +182,7 @@ abstract class FSTEnum { } setCurrentLabel(arc.label); incr(); - arc = fst.readFirstTargetArc(arc, getArc(upto)); + arc = fst.readFirstTargetArc(arc, getArc(upto), fstReader); targetLabel = getTargetLabel(); continue; } else if (low == arc.numArcs) { @@ -198,7 +200,7 @@ abstract class FSTEnum { final FST.Arc prevArc = getArc(upto); //System.out.println(" rollback upto=" + upto + " arc.label=" + prevArc.label + " isLast?=" + prevArc.isLast()); if (!prevArc.isLast()) { - fst.readNextArc(prevArc); + fst.readNextArc(prevArc, fstReader); pushFirst(); return; } @@ -221,7 +223,7 @@ abstract class FSTEnum { } setCurrentLabel(arc.label); incr(); - arc = fst.readFirstTargetArc(arc, getArc(upto)); + arc = fst.readFirstTargetArc(arc, getArc(upto), fstReader); targetLabel = getTargetLabel(); } else if (arc.label > targetLabel) { pushFirst(); @@ -237,7 +239,7 @@ abstract class FSTEnum { final FST.Arc prevArc = getArc(upto); //System.out.println(" rollback upto=" + upto + " arc.label=" + prevArc.label + " isLast?=" + prevArc.isLast()); if (!prevArc.isLast()) { - fst.readNextArc(prevArc); + fst.readNextArc(prevArc, fstReader); pushFirst(); return; } @@ -246,7 +248,7 @@ abstract class FSTEnum { } else { // keep scanning //System.out.println(" next scan"); - fst.readNextArc(arc); + fst.readNextArc(arc, fstReader); } } } @@ -320,7 +322,7 @@ abstract class FSTEnum { } setCurrentLabel(arc.label); incr(); - arc = fst.readFirstTargetArc(arc, getArc(upto)); + arc = fst.readFirstTargetArc(arc, getArc(upto), fstReader); targetLabel = getTargetLabel(); continue; } else if (high == -1) { @@ -333,12 +335,12 @@ abstract class FSTEnum { while(true) { // First, walk backwards until we find a first arc // that's before our target label: - fst.readFirstTargetArc(getArc(upto-1), arc); + fst.readFirstTargetArc(getArc(upto-1), arc, fstReader); if (arc.label < targetLabel) { // Then, scan forwards to the arc just before // the targetLabel: - while(!arc.isLast() && fst.readNextArcLabel(arc) < targetLabel) { - fst.readNextArc(arc); + while(!arc.isLast() && fst.readNextArcLabel(arc, in) < targetLabel) { + fst.readNextArc(arc, fstReader); } pushLast(); return; @@ -355,7 +357,7 @@ abstract class FSTEnum { arc.arcIdx = (low > high ? high : low)-1; //System.out.println(" hasFloor arcIdx=" + (arc.arcIdx+1)); fst.readNextRealArc(arc, in); - assert arc.isLast() || fst.readNextArcLabel(arc) > targetLabel; + assert arc.isLast() || fst.readNextArcLabel(arc, in) > targetLabel; assert arc.label < targetLabel: "arc.label=" + arc.label + " vs targetLabel=" + targetLabel; pushLast(); return; @@ -370,7 +372,7 @@ abstract class FSTEnum { } setCurrentLabel(arc.label); incr(); - arc = fst.readFirstTargetArc(arc, getArc(upto)); + arc = fst.readFirstTargetArc(arc, getArc(upto), fstReader); targetLabel = getTargetLabel(); } else if (arc.label > targetLabel) { // TODO: if each arc could somehow read the arc just @@ -380,12 +382,12 @@ abstract class FSTEnum { while(true) { // First, walk backwards until we find a first arc // that's before our target label: - fst.readFirstTargetArc(getArc(upto-1), arc); + fst.readFirstTargetArc(getArc(upto-1), arc, fstReader); if (arc.label < targetLabel) { // Then, scan forwards to the arc just before // the targetLabel: - while(!arc.isLast() && fst.readNextArcLabel(arc) < targetLabel) { - fst.readNextArc(arc); + while(!arc.isLast() && fst.readNextArcLabel(arc, fstReader) < targetLabel) { + fst.readNextArc(arc, fstReader); } pushLast(); return; @@ -399,12 +401,12 @@ abstract class FSTEnum { } } else if (!arc.isLast()) { //System.out.println(" check next label=" + fst.readNextArcLabel(arc) + " (" + (char) fst.readNextArcLabel(arc) + ")"); - if (fst.readNextArcLabel(arc) > targetLabel) { + if (fst.readNextArcLabel(arc, fstReader) > targetLabel) { pushLast(); return; } else { // keep scanning - fst.readNextArc(arc); + fst.readNextArc(arc, fstReader); } } else { pushLast(); @@ -441,7 +443,7 @@ abstract class FSTEnum { // short circuit //upto--; //upto = 0; - fst.readFirstTargetArc(arc, getArc(upto)); + fst.readFirstTargetArc(arc, getArc(upto), fstReader); //System.out.println(" no match upto=" + upto); return false; } @@ -493,7 +495,7 @@ abstract class FSTEnum { incr(); final FST.Arc nextArc = getArc(upto); - fst.readFirstTargetArc(arc, nextArc); + fst.readFirstTargetArc(arc, nextArc, fstReader); arc = nextArc; } } @@ -514,7 +516,7 @@ abstract class FSTEnum { } incr(); - arc = fst.readLastTargetArc(arc, getArc(upto)); + arc = fst.readLastTargetArc(arc, getArc(upto), fstReader); } } diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java b/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java index c96651a..919fcea 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/IntsRefFSTEnum.java b/lucene/core/src/java/org/apache/lucene/util/fst/IntsRefFSTEnum.java index ecc24fd..ff4b80a 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/IntsRefFSTEnum.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/IntsRefFSTEnum.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,6 +17,7 @@ package org.apache.lucene.util.fst; * limitations under the License. */ +import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.IntsRef; import java.io.IOException; @@ -110,7 +111,7 @@ public final class IntsRefFSTEnum extends FSTEnum { @Override protected void grow() { - current.grow(upto+1); + current.ints = ArrayUtil.grow(current.ints, upto+1); } private InputOutput setResult() { diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/NoOutputs.java b/lucene/core/src/java/org/apache/lucene/util/fst/NoOutputs.java index 042420e..aa829fb 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/NoOutputs.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/NoOutputs.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java b/lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java index 47e669d..7325835 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java b/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java index 51d7694..7d54c15 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java b/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java index f2339a2..e625ca0 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/PositiveIntOutputs.java b/lucene/core/src/java/org/apache/lucene/util/fst/PositiveIntOutputs.java index 14b3da8..dcd09a3 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/PositiveIntOutputs.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/PositiveIntOutputs.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -109,7 +109,6 @@ public final class PositiveIntOutputs extends Outputs { private boolean valid(Long o) { assert o != null; - assert o instanceof Long; assert o == NO_OUTPUT || o > 0; return true; } diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/UpToTwoPositiveIntOutputs.java b/lucene/core/src/java/org/apache/lucene/util/fst/UpToTwoPositiveIntOutputs.java index 5167862..8da8fd1 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/UpToTwoPositiveIntOutputs.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/UpToTwoPositiveIntOutputs.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/Util.java b/lucene/core/src/java/org/apache/lucene/util/fst/Util.java index a398f87..55823a3 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/Util.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/Util.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -335,6 +335,7 @@ public final class Util { final List> results = new ArrayList>(); + final FST.BytesReader fstReader = fst.getBytesReader(0); final T NO_OUTPUT = fst.outputs.getNoOutput(); // TODO: we could enable FST to sorting arcs by weight @@ -366,7 +367,7 @@ public final class Util { FST.Arc minArc = null; path = new FSTPath(NO_OUTPUT, fromNode, comparator); - fst.readFirstTargetArc(fromNode, path.arc); + fst.readFirstTargetArc(fromNode, path.arc, fstReader); // Bootstrap: find the min starting arc while (true) { @@ -383,7 +384,7 @@ public final class Util { if (path.arc.isLast()) { break; } - fst.readNextArc(path.arc); + fst.readNextArc(path.arc, fstReader); } assert minArc != null; @@ -439,7 +440,7 @@ public final class Util { while (true) { //System.out.println("\n cycle path: " + path); - fst.readFirstTargetArc(path.arc, path.arc); + fst.readFirstTargetArc(path.arc, path.arc, fstReader); // For each arc leaving this node: boolean foundZero = false; @@ -463,7 +464,7 @@ public final class Util { if (path.arc.isLast()) { break; } - fst.readNextArc(path.arc); + fst.readNextArc(path.arc, fstReader); } assert foundZero; @@ -598,12 +599,13 @@ public final class Util { emitDotState(out, "initial", "point", "white", ""); final T NO_OUTPUT = fst.outputs.getNoOutput(); + final FST.BytesReader r = fst.getBytesReader(0); // final FST.Arc scratchArc = new FST.Arc(); { final String stateColor; - if (fst.isExpandedTarget(startArc)) { + if (fst.isExpandedTarget(startArc, r)) { stateColor = expandedNodeColor; } else { stateColor = null; @@ -626,8 +628,6 @@ public final class Util { int level = 0; - final FST.BytesReader r = fst.getBytesReader(0); - while (!nextLevelQueue.isEmpty()) { // we could double buffer here, but it doesn't matter probably. //System.out.println("next level=" + level); @@ -666,7 +666,7 @@ public final class Util { } */ final String stateColor; - if (fst.isExpandedTarget(arc)) { + if (fst.isExpandedTarget(arc, r)) { stateColor = expandedNodeColor; } else { stateColor = null; diff --git a/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValue.java b/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValue.java index 57c966c..b98f37e 100755 --- a/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValue.java +++ b/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValue.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueBool.java b/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueBool.java index fe724a6..529ed40 100644 --- a/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueBool.java +++ b/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueBool.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueDate.java b/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueDate.java index e95ace3..38e043b 100755 --- a/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueDate.java +++ b/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueDate.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueDouble.java b/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueDouble.java index 3d40f39..7eb4316 100755 --- a/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueDouble.java +++ b/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueDouble.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueFloat.java b/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueFloat.java index 91c0614..85c4109 100755 --- a/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueFloat.java +++ b/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueFloat.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueInt.java b/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueInt.java index fc7be61..0a1fbd3 100755 --- a/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueInt.java +++ b/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueInt.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueLong.java b/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueLong.java index cb89050..d93a581 100644 --- a/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueLong.java +++ b/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueLong.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueStr.java b/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueStr.java index 55dfe02..05afded 100755 --- a/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueStr.java +++ b/lucene/core/src/java/org/apache/lucene/util/mutable/MutableValueStr.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperation.java b/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperation.java new file mode 100644 index 0000000..0010ee2 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperation.java @@ -0,0 +1,7802 @@ +// This file has been automatically generated, DO NOT EDIT + +package org.apache.lucene.util.packed; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.EnumMap; + +/** + * Efficient sequential read/write of packed integers. + */ +abstract class BulkOperation { + + static final EnumMap BULK_OPERATIONS = new EnumMap(PackedInts.Format.class); + + public static BulkOperation of(PackedInts.Format format, int bitsPerValue) { + assert bitsPerValue > 0 && bitsPerValue <= 64; + BulkOperation[] ops = BULK_OPERATIONS.get(format); + if (ops == null || ops[bitsPerValue] == null) { + throw new IllegalArgumentException("format: " + format + ", bitsPerValue: " + bitsPerValue); + } + return ops[bitsPerValue]; + } + + /** + * For every number of bits per value, there is a minimum number of + * blocks (b) / values (v) you need to write in order to reach the next block + * boundary: + * - 16 bits per value -> b=1, v=4 + * - 24 bits per value -> b=3, v=8 + * - 50 bits per value -> b=25, v=32 + * - 63 bits per value -> b=63, v = 64 + * - ... + * + * A bulk read consists in copying iterations*v values that are + * contained in iterations*b blocks into a long[] + * (higher values of iterations are likely to yield a better + * throughput) => this requires n * (b + v) longs in memory. + * + * This method computes iterations as + * ramBudget / (8 * (b + v)) (since a long is 8 bytes). + */ + public final int computeIterations(int valueCount, int ramBudget) { + final int iterations = (ramBudget >>> 3) / (blocks() + values()); + if (iterations == 0) { + // at least 1 + return 1; + } else if ((iterations - 1) * blocks() >= valueCount) { + // don't allocate for more than the size of the reader + return (int) Math.ceil((double) valueCount / values()); + } else { + return iterations; + } + } + + /** + * The minimum number of blocks required to perform a bulk get/set. + */ + public abstract int blocks(); + + /** + * The number of values that can be stored in blocks() blocks. + */ + public abstract int values(); + + /** + * Get n * values() values from n * blocks() blocks. + */ + public abstract void get(long[] blocks, int blockIndex, long[] values, int valuesIndex, int iterations); + + /** + * Set n * values() values into n * blocks() blocks. + */ + public abstract void set(long[] blocks, int blockIndex, long[] values, int valuesIndex, int iterations); + + static { + BULK_OPERATIONS.put(PackedInts.Format.PACKED, new BulkOperation[65]); BULK_OPERATIONS.get(PackedInts.Format.PACKED)[1] = new Packed64BulkOperation1(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[2] = new Packed64BulkOperation2(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[3] = new Packed64BulkOperation3(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[4] = new Packed64BulkOperation4(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[5] = new Packed64BulkOperation5(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[6] = new Packed64BulkOperation6(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[7] = new Packed64BulkOperation7(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[8] = new Packed64BulkOperation8(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[9] = new Packed64BulkOperation9(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[10] = new Packed64BulkOperation10(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[11] = new Packed64BulkOperation11(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[12] = new Packed64BulkOperation12(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[13] = new Packed64BulkOperation13(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[14] = new Packed64BulkOperation14(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[15] = new Packed64BulkOperation15(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[16] = new Packed64BulkOperation16(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[17] = new Packed64BulkOperation17(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[18] = new Packed64BulkOperation18(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[19] = new Packed64BulkOperation19(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[20] = new Packed64BulkOperation20(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[21] = new Packed64BulkOperation21(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[22] = new Packed64BulkOperation22(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[23] = new Packed64BulkOperation23(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[24] = new Packed64BulkOperation24(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[25] = new Packed64BulkOperation25(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[26] = new Packed64BulkOperation26(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[27] = new Packed64BulkOperation27(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[28] = new Packed64BulkOperation28(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[29] = new Packed64BulkOperation29(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[30] = new Packed64BulkOperation30(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[31] = new Packed64BulkOperation31(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[32] = new Packed64BulkOperation32(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[33] = new Packed64BulkOperation33(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[34] = new Packed64BulkOperation34(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[35] = new Packed64BulkOperation35(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[36] = new Packed64BulkOperation36(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[37] = new Packed64BulkOperation37(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[38] = new Packed64BulkOperation38(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[39] = new Packed64BulkOperation39(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[40] = new Packed64BulkOperation40(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[41] = new Packed64BulkOperation41(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[42] = new Packed64BulkOperation42(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[43] = new Packed64BulkOperation43(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[44] = new Packed64BulkOperation44(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[45] = new Packed64BulkOperation45(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[46] = new Packed64BulkOperation46(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[47] = new Packed64BulkOperation47(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[48] = new Packed64BulkOperation48(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[49] = new Packed64BulkOperation49(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[50] = new Packed64BulkOperation50(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[51] = new Packed64BulkOperation51(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[52] = new Packed64BulkOperation52(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[53] = new Packed64BulkOperation53(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[54] = new Packed64BulkOperation54(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[55] = new Packed64BulkOperation55(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[56] = new Packed64BulkOperation56(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[57] = new Packed64BulkOperation57(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[58] = new Packed64BulkOperation58(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[59] = new Packed64BulkOperation59(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[60] = new Packed64BulkOperation60(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[61] = new Packed64BulkOperation61(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[62] = new Packed64BulkOperation62(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[63] = new Packed64BulkOperation63(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED)[64] = new Packed64BulkOperation64(); + BULK_OPERATIONS.put(PackedInts.Format.PACKED_SINGLE_BLOCK, new BulkOperation[65]); + BULK_OPERATIONS.get(PackedInts.Format.PACKED_SINGLE_BLOCK)[1] = new Packed64SingleBlockBulkOperation1(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED_SINGLE_BLOCK)[2] = new Packed64SingleBlockBulkOperation2(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED_SINGLE_BLOCK)[3] = new Packed64SingleBlockBulkOperation3(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED_SINGLE_BLOCK)[4] = new Packed64SingleBlockBulkOperation4(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED_SINGLE_BLOCK)[5] = new Packed64SingleBlockBulkOperation5(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED_SINGLE_BLOCK)[6] = new Packed64SingleBlockBulkOperation6(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED_SINGLE_BLOCK)[7] = new Packed64SingleBlockBulkOperation7(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED_SINGLE_BLOCK)[8] = new Packed64SingleBlockBulkOperation8(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED_SINGLE_BLOCK)[9] = new Packed64SingleBlockBulkOperation9(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED_SINGLE_BLOCK)[10] = new Packed64SingleBlockBulkOperation10(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED_SINGLE_BLOCK)[12] = new Packed64SingleBlockBulkOperation12(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED_SINGLE_BLOCK)[16] = new Packed64SingleBlockBulkOperation16(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED_SINGLE_BLOCK)[21] = new Packed64SingleBlockBulkOperation21(); + BULK_OPERATIONS.get(PackedInts.Format.PACKED_SINGLE_BLOCK)[32] = new Packed64SingleBlockBulkOperation32(); + } + static final class Packed64BulkOperation1 extends BulkOperation { + + public int blocks() { + return 1; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 63; + values[vi++] = (block0 >>> 62) & 1L; + values[vi++] = (block0 >>> 61) & 1L; + values[vi++] = (block0 >>> 60) & 1L; + values[vi++] = (block0 >>> 59) & 1L; + values[vi++] = (block0 >>> 58) & 1L; + values[vi++] = (block0 >>> 57) & 1L; + values[vi++] = (block0 >>> 56) & 1L; + values[vi++] = (block0 >>> 55) & 1L; + values[vi++] = (block0 >>> 54) & 1L; + values[vi++] = (block0 >>> 53) & 1L; + values[vi++] = (block0 >>> 52) & 1L; + values[vi++] = (block0 >>> 51) & 1L; + values[vi++] = (block0 >>> 50) & 1L; + values[vi++] = (block0 >>> 49) & 1L; + values[vi++] = (block0 >>> 48) & 1L; + values[vi++] = (block0 >>> 47) & 1L; + values[vi++] = (block0 >>> 46) & 1L; + values[vi++] = (block0 >>> 45) & 1L; + values[vi++] = (block0 >>> 44) & 1L; + values[vi++] = (block0 >>> 43) & 1L; + values[vi++] = (block0 >>> 42) & 1L; + values[vi++] = (block0 >>> 41) & 1L; + values[vi++] = (block0 >>> 40) & 1L; + values[vi++] = (block0 >>> 39) & 1L; + values[vi++] = (block0 >>> 38) & 1L; + values[vi++] = (block0 >>> 37) & 1L; + values[vi++] = (block0 >>> 36) & 1L; + values[vi++] = (block0 >>> 35) & 1L; + values[vi++] = (block0 >>> 34) & 1L; + values[vi++] = (block0 >>> 33) & 1L; + values[vi++] = (block0 >>> 32) & 1L; + values[vi++] = (block0 >>> 31) & 1L; + values[vi++] = (block0 >>> 30) & 1L; + values[vi++] = (block0 >>> 29) & 1L; + values[vi++] = (block0 >>> 28) & 1L; + values[vi++] = (block0 >>> 27) & 1L; + values[vi++] = (block0 >>> 26) & 1L; + values[vi++] = (block0 >>> 25) & 1L; + values[vi++] = (block0 >>> 24) & 1L; + values[vi++] = (block0 >>> 23) & 1L; + values[vi++] = (block0 >>> 22) & 1L; + values[vi++] = (block0 >>> 21) & 1L; + values[vi++] = (block0 >>> 20) & 1L; + values[vi++] = (block0 >>> 19) & 1L; + values[vi++] = (block0 >>> 18) & 1L; + values[vi++] = (block0 >>> 17) & 1L; + values[vi++] = (block0 >>> 16) & 1L; + values[vi++] = (block0 >>> 15) & 1L; + values[vi++] = (block0 >>> 14) & 1L; + values[vi++] = (block0 >>> 13) & 1L; + values[vi++] = (block0 >>> 12) & 1L; + values[vi++] = (block0 >>> 11) & 1L; + values[vi++] = (block0 >>> 10) & 1L; + values[vi++] = (block0 >>> 9) & 1L; + values[vi++] = (block0 >>> 8) & 1L; + values[vi++] = (block0 >>> 7) & 1L; + values[vi++] = (block0 >>> 6) & 1L; + values[vi++] = (block0 >>> 5) & 1L; + values[vi++] = (block0 >>> 4) & 1L; + values[vi++] = (block0 >>> 3) & 1L; + values[vi++] = (block0 >>> 2) & 1L; + values[vi++] = (block0 >>> 1) & 1L; + values[vi++] = block0 & 1L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 62) | (values[vi++] << 61) | (values[vi++] << 60) | (values[vi++] << 59) | (values[vi++] << 58) | (values[vi++] << 57) | (values[vi++] << 56) | (values[vi++] << 55) | (values[vi++] << 54) | (values[vi++] << 53) | (values[vi++] << 52) | (values[vi++] << 51) | (values[vi++] << 50) | (values[vi++] << 49) | (values[vi++] << 48) | (values[vi++] << 47) | (values[vi++] << 46) | (values[vi++] << 45) | (values[vi++] << 44) | (values[vi++] << 43) | (values[vi++] << 42) | (values[vi++] << 41) | (values[vi++] << 40) | (values[vi++] << 39) | (values[vi++] << 38) | (values[vi++] << 37) | (values[vi++] << 36) | (values[vi++] << 35) | (values[vi++] << 34) | (values[vi++] << 33) | (values[vi++] << 32) | (values[vi++] << 31) | (values[vi++] << 30) | (values[vi++] << 29) | (values[vi++] << 28) | (values[vi++] << 27) | (values[vi++] << 26) | (values[vi++] << 25) | (values[vi++] << 24) | (values[vi++] << 23) | (values[vi++] << 22) | (values[vi++] << 21) | (values[vi++] << 20) | (values[vi++] << 19) | (values[vi++] << 18) | (values[vi++] << 17) | (values[vi++] << 16) | (values[vi++] << 15) | (values[vi++] << 14) | (values[vi++] << 13) | (values[vi++] << 12) | (values[vi++] << 11) | (values[vi++] << 10) | (values[vi++] << 9) | (values[vi++] << 8) | (values[vi++] << 7) | (values[vi++] << 6) | (values[vi++] << 5) | (values[vi++] << 4) | (values[vi++] << 3) | (values[vi++] << 2) | (values[vi++] << 1) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation2 extends BulkOperation { + + public int blocks() { + return 1; + } + + public int values() { + return 32; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 62; + values[vi++] = (block0 >>> 60) & 3L; + values[vi++] = (block0 >>> 58) & 3L; + values[vi++] = (block0 >>> 56) & 3L; + values[vi++] = (block0 >>> 54) & 3L; + values[vi++] = (block0 >>> 52) & 3L; + values[vi++] = (block0 >>> 50) & 3L; + values[vi++] = (block0 >>> 48) & 3L; + values[vi++] = (block0 >>> 46) & 3L; + values[vi++] = (block0 >>> 44) & 3L; + values[vi++] = (block0 >>> 42) & 3L; + values[vi++] = (block0 >>> 40) & 3L; + values[vi++] = (block0 >>> 38) & 3L; + values[vi++] = (block0 >>> 36) & 3L; + values[vi++] = (block0 >>> 34) & 3L; + values[vi++] = (block0 >>> 32) & 3L; + values[vi++] = (block0 >>> 30) & 3L; + values[vi++] = (block0 >>> 28) & 3L; + values[vi++] = (block0 >>> 26) & 3L; + values[vi++] = (block0 >>> 24) & 3L; + values[vi++] = (block0 >>> 22) & 3L; + values[vi++] = (block0 >>> 20) & 3L; + values[vi++] = (block0 >>> 18) & 3L; + values[vi++] = (block0 >>> 16) & 3L; + values[vi++] = (block0 >>> 14) & 3L; + values[vi++] = (block0 >>> 12) & 3L; + values[vi++] = (block0 >>> 10) & 3L; + values[vi++] = (block0 >>> 8) & 3L; + values[vi++] = (block0 >>> 6) & 3L; + values[vi++] = (block0 >>> 4) & 3L; + values[vi++] = (block0 >>> 2) & 3L; + values[vi++] = block0 & 3L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 60) | (values[vi++] << 58) | (values[vi++] << 56) | (values[vi++] << 54) | (values[vi++] << 52) | (values[vi++] << 50) | (values[vi++] << 48) | (values[vi++] << 46) | (values[vi++] << 44) | (values[vi++] << 42) | (values[vi++] << 40) | (values[vi++] << 38) | (values[vi++] << 36) | (values[vi++] << 34) | (values[vi++] << 32) | (values[vi++] << 30) | (values[vi++] << 28) | (values[vi++] << 26) | (values[vi++] << 24) | (values[vi++] << 22) | (values[vi++] << 20) | (values[vi++] << 18) | (values[vi++] << 16) | (values[vi++] << 14) | (values[vi++] << 12) | (values[vi++] << 10) | (values[vi++] << 8) | (values[vi++] << 6) | (values[vi++] << 4) | (values[vi++] << 2) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation3 extends BulkOperation { + + public int blocks() { + return 3; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 61; + values[vi++] = (block0 >>> 58) & 7L; + values[vi++] = (block0 >>> 55) & 7L; + values[vi++] = (block0 >>> 52) & 7L; + values[vi++] = (block0 >>> 49) & 7L; + values[vi++] = (block0 >>> 46) & 7L; + values[vi++] = (block0 >>> 43) & 7L; + values[vi++] = (block0 >>> 40) & 7L; + values[vi++] = (block0 >>> 37) & 7L; + values[vi++] = (block0 >>> 34) & 7L; + values[vi++] = (block0 >>> 31) & 7L; + values[vi++] = (block0 >>> 28) & 7L; + values[vi++] = (block0 >>> 25) & 7L; + values[vi++] = (block0 >>> 22) & 7L; + values[vi++] = (block0 >>> 19) & 7L; + values[vi++] = (block0 >>> 16) & 7L; + values[vi++] = (block0 >>> 13) & 7L; + values[vi++] = (block0 >>> 10) & 7L; + values[vi++] = (block0 >>> 7) & 7L; + values[vi++] = (block0 >>> 4) & 7L; + values[vi++] = (block0 >>> 1) & 7L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 1L) << 2) | (block1 >>> 62); + values[vi++] = (block1 >>> 59) & 7L; + values[vi++] = (block1 >>> 56) & 7L; + values[vi++] = (block1 >>> 53) & 7L; + values[vi++] = (block1 >>> 50) & 7L; + values[vi++] = (block1 >>> 47) & 7L; + values[vi++] = (block1 >>> 44) & 7L; + values[vi++] = (block1 >>> 41) & 7L; + values[vi++] = (block1 >>> 38) & 7L; + values[vi++] = (block1 >>> 35) & 7L; + values[vi++] = (block1 >>> 32) & 7L; + values[vi++] = (block1 >>> 29) & 7L; + values[vi++] = (block1 >>> 26) & 7L; + values[vi++] = (block1 >>> 23) & 7L; + values[vi++] = (block1 >>> 20) & 7L; + values[vi++] = (block1 >>> 17) & 7L; + values[vi++] = (block1 >>> 14) & 7L; + values[vi++] = (block1 >>> 11) & 7L; + values[vi++] = (block1 >>> 8) & 7L; + values[vi++] = (block1 >>> 5) & 7L; + values[vi++] = (block1 >>> 2) & 7L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 3L) << 1) | (block2 >>> 63); + values[vi++] = (block2 >>> 60) & 7L; + values[vi++] = (block2 >>> 57) & 7L; + values[vi++] = (block2 >>> 54) & 7L; + values[vi++] = (block2 >>> 51) & 7L; + values[vi++] = (block2 >>> 48) & 7L; + values[vi++] = (block2 >>> 45) & 7L; + values[vi++] = (block2 >>> 42) & 7L; + values[vi++] = (block2 >>> 39) & 7L; + values[vi++] = (block2 >>> 36) & 7L; + values[vi++] = (block2 >>> 33) & 7L; + values[vi++] = (block2 >>> 30) & 7L; + values[vi++] = (block2 >>> 27) & 7L; + values[vi++] = (block2 >>> 24) & 7L; + values[vi++] = (block2 >>> 21) & 7L; + values[vi++] = (block2 >>> 18) & 7L; + values[vi++] = (block2 >>> 15) & 7L; + values[vi++] = (block2 >>> 12) & 7L; + values[vi++] = (block2 >>> 9) & 7L; + values[vi++] = (block2 >>> 6) & 7L; + values[vi++] = (block2 >>> 3) & 7L; + values[vi++] = block2 & 7L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 58) | (values[vi++] << 55) | (values[vi++] << 52) | (values[vi++] << 49) | (values[vi++] << 46) | (values[vi++] << 43) | (values[vi++] << 40) | (values[vi++] << 37) | (values[vi++] << 34) | (values[vi++] << 31) | (values[vi++] << 28) | (values[vi++] << 25) | (values[vi++] << 22) | (values[vi++] << 19) | (values[vi++] << 16) | (values[vi++] << 13) | (values[vi++] << 10) | (values[vi++] << 7) | (values[vi++] << 4) | (values[vi++] << 1) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 59) | (values[vi++] << 56) | (values[vi++] << 53) | (values[vi++] << 50) | (values[vi++] << 47) | (values[vi++] << 44) | (values[vi++] << 41) | (values[vi++] << 38) | (values[vi++] << 35) | (values[vi++] << 32) | (values[vi++] << 29) | (values[vi++] << 26) | (values[vi++] << 23) | (values[vi++] << 20) | (values[vi++] << 17) | (values[vi++] << 14) | (values[vi++] << 11) | (values[vi++] << 8) | (values[vi++] << 5) | (values[vi++] << 2) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 60) | (values[vi++] << 57) | (values[vi++] << 54) | (values[vi++] << 51) | (values[vi++] << 48) | (values[vi++] << 45) | (values[vi++] << 42) | (values[vi++] << 39) | (values[vi++] << 36) | (values[vi++] << 33) | (values[vi++] << 30) | (values[vi++] << 27) | (values[vi++] << 24) | (values[vi++] << 21) | (values[vi++] << 18) | (values[vi++] << 15) | (values[vi++] << 12) | (values[vi++] << 9) | (values[vi++] << 6) | (values[vi++] << 3) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation4 extends BulkOperation { + + public int blocks() { + return 1; + } + + public int values() { + return 16; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 60; + values[vi++] = (block0 >>> 56) & 15L; + values[vi++] = (block0 >>> 52) & 15L; + values[vi++] = (block0 >>> 48) & 15L; + values[vi++] = (block0 >>> 44) & 15L; + values[vi++] = (block0 >>> 40) & 15L; + values[vi++] = (block0 >>> 36) & 15L; + values[vi++] = (block0 >>> 32) & 15L; + values[vi++] = (block0 >>> 28) & 15L; + values[vi++] = (block0 >>> 24) & 15L; + values[vi++] = (block0 >>> 20) & 15L; + values[vi++] = (block0 >>> 16) & 15L; + values[vi++] = (block0 >>> 12) & 15L; + values[vi++] = (block0 >>> 8) & 15L; + values[vi++] = (block0 >>> 4) & 15L; + values[vi++] = block0 & 15L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 56) | (values[vi++] << 52) | (values[vi++] << 48) | (values[vi++] << 44) | (values[vi++] << 40) | (values[vi++] << 36) | (values[vi++] << 32) | (values[vi++] << 28) | (values[vi++] << 24) | (values[vi++] << 20) | (values[vi++] << 16) | (values[vi++] << 12) | (values[vi++] << 8) | (values[vi++] << 4) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation5 extends BulkOperation { + + public int blocks() { + return 5; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 59; + values[vi++] = (block0 >>> 54) & 31L; + values[vi++] = (block0 >>> 49) & 31L; + values[vi++] = (block0 >>> 44) & 31L; + values[vi++] = (block0 >>> 39) & 31L; + values[vi++] = (block0 >>> 34) & 31L; + values[vi++] = (block0 >>> 29) & 31L; + values[vi++] = (block0 >>> 24) & 31L; + values[vi++] = (block0 >>> 19) & 31L; + values[vi++] = (block0 >>> 14) & 31L; + values[vi++] = (block0 >>> 9) & 31L; + values[vi++] = (block0 >>> 4) & 31L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 15L) << 1) | (block1 >>> 63); + values[vi++] = (block1 >>> 58) & 31L; + values[vi++] = (block1 >>> 53) & 31L; + values[vi++] = (block1 >>> 48) & 31L; + values[vi++] = (block1 >>> 43) & 31L; + values[vi++] = (block1 >>> 38) & 31L; + values[vi++] = (block1 >>> 33) & 31L; + values[vi++] = (block1 >>> 28) & 31L; + values[vi++] = (block1 >>> 23) & 31L; + values[vi++] = (block1 >>> 18) & 31L; + values[vi++] = (block1 >>> 13) & 31L; + values[vi++] = (block1 >>> 8) & 31L; + values[vi++] = (block1 >>> 3) & 31L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 7L) << 2) | (block2 >>> 62); + values[vi++] = (block2 >>> 57) & 31L; + values[vi++] = (block2 >>> 52) & 31L; + values[vi++] = (block2 >>> 47) & 31L; + values[vi++] = (block2 >>> 42) & 31L; + values[vi++] = (block2 >>> 37) & 31L; + values[vi++] = (block2 >>> 32) & 31L; + values[vi++] = (block2 >>> 27) & 31L; + values[vi++] = (block2 >>> 22) & 31L; + values[vi++] = (block2 >>> 17) & 31L; + values[vi++] = (block2 >>> 12) & 31L; + values[vi++] = (block2 >>> 7) & 31L; + values[vi++] = (block2 >>> 2) & 31L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 3L) << 3) | (block3 >>> 61); + values[vi++] = (block3 >>> 56) & 31L; + values[vi++] = (block3 >>> 51) & 31L; + values[vi++] = (block3 >>> 46) & 31L; + values[vi++] = (block3 >>> 41) & 31L; + values[vi++] = (block3 >>> 36) & 31L; + values[vi++] = (block3 >>> 31) & 31L; + values[vi++] = (block3 >>> 26) & 31L; + values[vi++] = (block3 >>> 21) & 31L; + values[vi++] = (block3 >>> 16) & 31L; + values[vi++] = (block3 >>> 11) & 31L; + values[vi++] = (block3 >>> 6) & 31L; + values[vi++] = (block3 >>> 1) & 31L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 1L) << 4) | (block4 >>> 60); + values[vi++] = (block4 >>> 55) & 31L; + values[vi++] = (block4 >>> 50) & 31L; + values[vi++] = (block4 >>> 45) & 31L; + values[vi++] = (block4 >>> 40) & 31L; + values[vi++] = (block4 >>> 35) & 31L; + values[vi++] = (block4 >>> 30) & 31L; + values[vi++] = (block4 >>> 25) & 31L; + values[vi++] = (block4 >>> 20) & 31L; + values[vi++] = (block4 >>> 15) & 31L; + values[vi++] = (block4 >>> 10) & 31L; + values[vi++] = (block4 >>> 5) & 31L; + values[vi++] = block4 & 31L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 54) | (values[vi++] << 49) | (values[vi++] << 44) | (values[vi++] << 39) | (values[vi++] << 34) | (values[vi++] << 29) | (values[vi++] << 24) | (values[vi++] << 19) | (values[vi++] << 14) | (values[vi++] << 9) | (values[vi++] << 4) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 58) | (values[vi++] << 53) | (values[vi++] << 48) | (values[vi++] << 43) | (values[vi++] << 38) | (values[vi++] << 33) | (values[vi++] << 28) | (values[vi++] << 23) | (values[vi++] << 18) | (values[vi++] << 13) | (values[vi++] << 8) | (values[vi++] << 3) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 57) | (values[vi++] << 52) | (values[vi++] << 47) | (values[vi++] << 42) | (values[vi++] << 37) | (values[vi++] << 32) | (values[vi++] << 27) | (values[vi++] << 22) | (values[vi++] << 17) | (values[vi++] << 12) | (values[vi++] << 7) | (values[vi++] << 2) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 56) | (values[vi++] << 51) | (values[vi++] << 46) | (values[vi++] << 41) | (values[vi++] << 36) | (values[vi++] << 31) | (values[vi++] << 26) | (values[vi++] << 21) | (values[vi++] << 16) | (values[vi++] << 11) | (values[vi++] << 6) | (values[vi++] << 1) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 55) | (values[vi++] << 50) | (values[vi++] << 45) | (values[vi++] << 40) | (values[vi++] << 35) | (values[vi++] << 30) | (values[vi++] << 25) | (values[vi++] << 20) | (values[vi++] << 15) | (values[vi++] << 10) | (values[vi++] << 5) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation6 extends BulkOperation { + + public int blocks() { + return 3; + } + + public int values() { + return 32; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 58; + values[vi++] = (block0 >>> 52) & 63L; + values[vi++] = (block0 >>> 46) & 63L; + values[vi++] = (block0 >>> 40) & 63L; + values[vi++] = (block0 >>> 34) & 63L; + values[vi++] = (block0 >>> 28) & 63L; + values[vi++] = (block0 >>> 22) & 63L; + values[vi++] = (block0 >>> 16) & 63L; + values[vi++] = (block0 >>> 10) & 63L; + values[vi++] = (block0 >>> 4) & 63L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 15L) << 2) | (block1 >>> 62); + values[vi++] = (block1 >>> 56) & 63L; + values[vi++] = (block1 >>> 50) & 63L; + values[vi++] = (block1 >>> 44) & 63L; + values[vi++] = (block1 >>> 38) & 63L; + values[vi++] = (block1 >>> 32) & 63L; + values[vi++] = (block1 >>> 26) & 63L; + values[vi++] = (block1 >>> 20) & 63L; + values[vi++] = (block1 >>> 14) & 63L; + values[vi++] = (block1 >>> 8) & 63L; + values[vi++] = (block1 >>> 2) & 63L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 3L) << 4) | (block2 >>> 60); + values[vi++] = (block2 >>> 54) & 63L; + values[vi++] = (block2 >>> 48) & 63L; + values[vi++] = (block2 >>> 42) & 63L; + values[vi++] = (block2 >>> 36) & 63L; + values[vi++] = (block2 >>> 30) & 63L; + values[vi++] = (block2 >>> 24) & 63L; + values[vi++] = (block2 >>> 18) & 63L; + values[vi++] = (block2 >>> 12) & 63L; + values[vi++] = (block2 >>> 6) & 63L; + values[vi++] = block2 & 63L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 52) | (values[vi++] << 46) | (values[vi++] << 40) | (values[vi++] << 34) | (values[vi++] << 28) | (values[vi++] << 22) | (values[vi++] << 16) | (values[vi++] << 10) | (values[vi++] << 4) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 56) | (values[vi++] << 50) | (values[vi++] << 44) | (values[vi++] << 38) | (values[vi++] << 32) | (values[vi++] << 26) | (values[vi++] << 20) | (values[vi++] << 14) | (values[vi++] << 8) | (values[vi++] << 2) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 54) | (values[vi++] << 48) | (values[vi++] << 42) | (values[vi++] << 36) | (values[vi++] << 30) | (values[vi++] << 24) | (values[vi++] << 18) | (values[vi++] << 12) | (values[vi++] << 6) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation7 extends BulkOperation { + + public int blocks() { + return 7; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 57; + values[vi++] = (block0 >>> 50) & 127L; + values[vi++] = (block0 >>> 43) & 127L; + values[vi++] = (block0 >>> 36) & 127L; + values[vi++] = (block0 >>> 29) & 127L; + values[vi++] = (block0 >>> 22) & 127L; + values[vi++] = (block0 >>> 15) & 127L; + values[vi++] = (block0 >>> 8) & 127L; + values[vi++] = (block0 >>> 1) & 127L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 1L) << 6) | (block1 >>> 58); + values[vi++] = (block1 >>> 51) & 127L; + values[vi++] = (block1 >>> 44) & 127L; + values[vi++] = (block1 >>> 37) & 127L; + values[vi++] = (block1 >>> 30) & 127L; + values[vi++] = (block1 >>> 23) & 127L; + values[vi++] = (block1 >>> 16) & 127L; + values[vi++] = (block1 >>> 9) & 127L; + values[vi++] = (block1 >>> 2) & 127L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 3L) << 5) | (block2 >>> 59); + values[vi++] = (block2 >>> 52) & 127L; + values[vi++] = (block2 >>> 45) & 127L; + values[vi++] = (block2 >>> 38) & 127L; + values[vi++] = (block2 >>> 31) & 127L; + values[vi++] = (block2 >>> 24) & 127L; + values[vi++] = (block2 >>> 17) & 127L; + values[vi++] = (block2 >>> 10) & 127L; + values[vi++] = (block2 >>> 3) & 127L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 7L) << 4) | (block3 >>> 60); + values[vi++] = (block3 >>> 53) & 127L; + values[vi++] = (block3 >>> 46) & 127L; + values[vi++] = (block3 >>> 39) & 127L; + values[vi++] = (block3 >>> 32) & 127L; + values[vi++] = (block3 >>> 25) & 127L; + values[vi++] = (block3 >>> 18) & 127L; + values[vi++] = (block3 >>> 11) & 127L; + values[vi++] = (block3 >>> 4) & 127L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 15L) << 3) | (block4 >>> 61); + values[vi++] = (block4 >>> 54) & 127L; + values[vi++] = (block4 >>> 47) & 127L; + values[vi++] = (block4 >>> 40) & 127L; + values[vi++] = (block4 >>> 33) & 127L; + values[vi++] = (block4 >>> 26) & 127L; + values[vi++] = (block4 >>> 19) & 127L; + values[vi++] = (block4 >>> 12) & 127L; + values[vi++] = (block4 >>> 5) & 127L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 31L) << 2) | (block5 >>> 62); + values[vi++] = (block5 >>> 55) & 127L; + values[vi++] = (block5 >>> 48) & 127L; + values[vi++] = (block5 >>> 41) & 127L; + values[vi++] = (block5 >>> 34) & 127L; + values[vi++] = (block5 >>> 27) & 127L; + values[vi++] = (block5 >>> 20) & 127L; + values[vi++] = (block5 >>> 13) & 127L; + values[vi++] = (block5 >>> 6) & 127L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 63L) << 1) | (block6 >>> 63); + values[vi++] = (block6 >>> 56) & 127L; + values[vi++] = (block6 >>> 49) & 127L; + values[vi++] = (block6 >>> 42) & 127L; + values[vi++] = (block6 >>> 35) & 127L; + values[vi++] = (block6 >>> 28) & 127L; + values[vi++] = (block6 >>> 21) & 127L; + values[vi++] = (block6 >>> 14) & 127L; + values[vi++] = (block6 >>> 7) & 127L; + values[vi++] = block6 & 127L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 50) | (values[vi++] << 43) | (values[vi++] << 36) | (values[vi++] << 29) | (values[vi++] << 22) | (values[vi++] << 15) | (values[vi++] << 8) | (values[vi++] << 1) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 51) | (values[vi++] << 44) | (values[vi++] << 37) | (values[vi++] << 30) | (values[vi++] << 23) | (values[vi++] << 16) | (values[vi++] << 9) | (values[vi++] << 2) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 52) | (values[vi++] << 45) | (values[vi++] << 38) | (values[vi++] << 31) | (values[vi++] << 24) | (values[vi++] << 17) | (values[vi++] << 10) | (values[vi++] << 3) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 53) | (values[vi++] << 46) | (values[vi++] << 39) | (values[vi++] << 32) | (values[vi++] << 25) | (values[vi++] << 18) | (values[vi++] << 11) | (values[vi++] << 4) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 54) | (values[vi++] << 47) | (values[vi++] << 40) | (values[vi++] << 33) | (values[vi++] << 26) | (values[vi++] << 19) | (values[vi++] << 12) | (values[vi++] << 5) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 55) | (values[vi++] << 48) | (values[vi++] << 41) | (values[vi++] << 34) | (values[vi++] << 27) | (values[vi++] << 20) | (values[vi++] << 13) | (values[vi++] << 6) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 56) | (values[vi++] << 49) | (values[vi++] << 42) | (values[vi++] << 35) | (values[vi++] << 28) | (values[vi++] << 21) | (values[vi++] << 14) | (values[vi++] << 7) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation8 extends BulkOperation { + + public int blocks() { + return 1; + } + + public int values() { + return 8; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 56; + values[vi++] = (block0 >>> 48) & 255L; + values[vi++] = (block0 >>> 40) & 255L; + values[vi++] = (block0 >>> 32) & 255L; + values[vi++] = (block0 >>> 24) & 255L; + values[vi++] = (block0 >>> 16) & 255L; + values[vi++] = (block0 >>> 8) & 255L; + values[vi++] = block0 & 255L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 48) | (values[vi++] << 40) | (values[vi++] << 32) | (values[vi++] << 24) | (values[vi++] << 16) | (values[vi++] << 8) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation9 extends BulkOperation { + + public int blocks() { + return 9; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 55; + values[vi++] = (block0 >>> 46) & 511L; + values[vi++] = (block0 >>> 37) & 511L; + values[vi++] = (block0 >>> 28) & 511L; + values[vi++] = (block0 >>> 19) & 511L; + values[vi++] = (block0 >>> 10) & 511L; + values[vi++] = (block0 >>> 1) & 511L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 1L) << 8) | (block1 >>> 56); + values[vi++] = (block1 >>> 47) & 511L; + values[vi++] = (block1 >>> 38) & 511L; + values[vi++] = (block1 >>> 29) & 511L; + values[vi++] = (block1 >>> 20) & 511L; + values[vi++] = (block1 >>> 11) & 511L; + values[vi++] = (block1 >>> 2) & 511L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 3L) << 7) | (block2 >>> 57); + values[vi++] = (block2 >>> 48) & 511L; + values[vi++] = (block2 >>> 39) & 511L; + values[vi++] = (block2 >>> 30) & 511L; + values[vi++] = (block2 >>> 21) & 511L; + values[vi++] = (block2 >>> 12) & 511L; + values[vi++] = (block2 >>> 3) & 511L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 7L) << 6) | (block3 >>> 58); + values[vi++] = (block3 >>> 49) & 511L; + values[vi++] = (block3 >>> 40) & 511L; + values[vi++] = (block3 >>> 31) & 511L; + values[vi++] = (block3 >>> 22) & 511L; + values[vi++] = (block3 >>> 13) & 511L; + values[vi++] = (block3 >>> 4) & 511L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 15L) << 5) | (block4 >>> 59); + values[vi++] = (block4 >>> 50) & 511L; + values[vi++] = (block4 >>> 41) & 511L; + values[vi++] = (block4 >>> 32) & 511L; + values[vi++] = (block4 >>> 23) & 511L; + values[vi++] = (block4 >>> 14) & 511L; + values[vi++] = (block4 >>> 5) & 511L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 31L) << 4) | (block5 >>> 60); + values[vi++] = (block5 >>> 51) & 511L; + values[vi++] = (block5 >>> 42) & 511L; + values[vi++] = (block5 >>> 33) & 511L; + values[vi++] = (block5 >>> 24) & 511L; + values[vi++] = (block5 >>> 15) & 511L; + values[vi++] = (block5 >>> 6) & 511L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 63L) << 3) | (block6 >>> 61); + values[vi++] = (block6 >>> 52) & 511L; + values[vi++] = (block6 >>> 43) & 511L; + values[vi++] = (block6 >>> 34) & 511L; + values[vi++] = (block6 >>> 25) & 511L; + values[vi++] = (block6 >>> 16) & 511L; + values[vi++] = (block6 >>> 7) & 511L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 127L) << 2) | (block7 >>> 62); + values[vi++] = (block7 >>> 53) & 511L; + values[vi++] = (block7 >>> 44) & 511L; + values[vi++] = (block7 >>> 35) & 511L; + values[vi++] = (block7 >>> 26) & 511L; + values[vi++] = (block7 >>> 17) & 511L; + values[vi++] = (block7 >>> 8) & 511L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 255L) << 1) | (block8 >>> 63); + values[vi++] = (block8 >>> 54) & 511L; + values[vi++] = (block8 >>> 45) & 511L; + values[vi++] = (block8 >>> 36) & 511L; + values[vi++] = (block8 >>> 27) & 511L; + values[vi++] = (block8 >>> 18) & 511L; + values[vi++] = (block8 >>> 9) & 511L; + values[vi++] = block8 & 511L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 46) | (values[vi++] << 37) | (values[vi++] << 28) | (values[vi++] << 19) | (values[vi++] << 10) | (values[vi++] << 1) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 47) | (values[vi++] << 38) | (values[vi++] << 29) | (values[vi++] << 20) | (values[vi++] << 11) | (values[vi++] << 2) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 48) | (values[vi++] << 39) | (values[vi++] << 30) | (values[vi++] << 21) | (values[vi++] << 12) | (values[vi++] << 3) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 49) | (values[vi++] << 40) | (values[vi++] << 31) | (values[vi++] << 22) | (values[vi++] << 13) | (values[vi++] << 4) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 50) | (values[vi++] << 41) | (values[vi++] << 32) | (values[vi++] << 23) | (values[vi++] << 14) | (values[vi++] << 5) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 51) | (values[vi++] << 42) | (values[vi++] << 33) | (values[vi++] << 24) | (values[vi++] << 15) | (values[vi++] << 6) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 52) | (values[vi++] << 43) | (values[vi++] << 34) | (values[vi++] << 25) | (values[vi++] << 16) | (values[vi++] << 7) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 53) | (values[vi++] << 44) | (values[vi++] << 35) | (values[vi++] << 26) | (values[vi++] << 17) | (values[vi++] << 8) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 54) | (values[vi++] << 45) | (values[vi++] << 36) | (values[vi++] << 27) | (values[vi++] << 18) | (values[vi++] << 9) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation10 extends BulkOperation { + + public int blocks() { + return 5; + } + + public int values() { + return 32; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 54; + values[vi++] = (block0 >>> 44) & 1023L; + values[vi++] = (block0 >>> 34) & 1023L; + values[vi++] = (block0 >>> 24) & 1023L; + values[vi++] = (block0 >>> 14) & 1023L; + values[vi++] = (block0 >>> 4) & 1023L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 15L) << 6) | (block1 >>> 58); + values[vi++] = (block1 >>> 48) & 1023L; + values[vi++] = (block1 >>> 38) & 1023L; + values[vi++] = (block1 >>> 28) & 1023L; + values[vi++] = (block1 >>> 18) & 1023L; + values[vi++] = (block1 >>> 8) & 1023L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 255L) << 2) | (block2 >>> 62); + values[vi++] = (block2 >>> 52) & 1023L; + values[vi++] = (block2 >>> 42) & 1023L; + values[vi++] = (block2 >>> 32) & 1023L; + values[vi++] = (block2 >>> 22) & 1023L; + values[vi++] = (block2 >>> 12) & 1023L; + values[vi++] = (block2 >>> 2) & 1023L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 3L) << 8) | (block3 >>> 56); + values[vi++] = (block3 >>> 46) & 1023L; + values[vi++] = (block3 >>> 36) & 1023L; + values[vi++] = (block3 >>> 26) & 1023L; + values[vi++] = (block3 >>> 16) & 1023L; + values[vi++] = (block3 >>> 6) & 1023L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 63L) << 4) | (block4 >>> 60); + values[vi++] = (block4 >>> 50) & 1023L; + values[vi++] = (block4 >>> 40) & 1023L; + values[vi++] = (block4 >>> 30) & 1023L; + values[vi++] = (block4 >>> 20) & 1023L; + values[vi++] = (block4 >>> 10) & 1023L; + values[vi++] = block4 & 1023L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 44) | (values[vi++] << 34) | (values[vi++] << 24) | (values[vi++] << 14) | (values[vi++] << 4) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 48) | (values[vi++] << 38) | (values[vi++] << 28) | (values[vi++] << 18) | (values[vi++] << 8) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 52) | (values[vi++] << 42) | (values[vi++] << 32) | (values[vi++] << 22) | (values[vi++] << 12) | (values[vi++] << 2) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 46) | (values[vi++] << 36) | (values[vi++] << 26) | (values[vi++] << 16) | (values[vi++] << 6) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 50) | (values[vi++] << 40) | (values[vi++] << 30) | (values[vi++] << 20) | (values[vi++] << 10) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation11 extends BulkOperation { + + public int blocks() { + return 11; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 53; + values[vi++] = (block0 >>> 42) & 2047L; + values[vi++] = (block0 >>> 31) & 2047L; + values[vi++] = (block0 >>> 20) & 2047L; + values[vi++] = (block0 >>> 9) & 2047L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 511L) << 2) | (block1 >>> 62); + values[vi++] = (block1 >>> 51) & 2047L; + values[vi++] = (block1 >>> 40) & 2047L; + values[vi++] = (block1 >>> 29) & 2047L; + values[vi++] = (block1 >>> 18) & 2047L; + values[vi++] = (block1 >>> 7) & 2047L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 127L) << 4) | (block2 >>> 60); + values[vi++] = (block2 >>> 49) & 2047L; + values[vi++] = (block2 >>> 38) & 2047L; + values[vi++] = (block2 >>> 27) & 2047L; + values[vi++] = (block2 >>> 16) & 2047L; + values[vi++] = (block2 >>> 5) & 2047L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 31L) << 6) | (block3 >>> 58); + values[vi++] = (block3 >>> 47) & 2047L; + values[vi++] = (block3 >>> 36) & 2047L; + values[vi++] = (block3 >>> 25) & 2047L; + values[vi++] = (block3 >>> 14) & 2047L; + values[vi++] = (block3 >>> 3) & 2047L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 7L) << 8) | (block4 >>> 56); + values[vi++] = (block4 >>> 45) & 2047L; + values[vi++] = (block4 >>> 34) & 2047L; + values[vi++] = (block4 >>> 23) & 2047L; + values[vi++] = (block4 >>> 12) & 2047L; + values[vi++] = (block4 >>> 1) & 2047L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 1L) << 10) | (block5 >>> 54); + values[vi++] = (block5 >>> 43) & 2047L; + values[vi++] = (block5 >>> 32) & 2047L; + values[vi++] = (block5 >>> 21) & 2047L; + values[vi++] = (block5 >>> 10) & 2047L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 1023L) << 1) | (block6 >>> 63); + values[vi++] = (block6 >>> 52) & 2047L; + values[vi++] = (block6 >>> 41) & 2047L; + values[vi++] = (block6 >>> 30) & 2047L; + values[vi++] = (block6 >>> 19) & 2047L; + values[vi++] = (block6 >>> 8) & 2047L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 255L) << 3) | (block7 >>> 61); + values[vi++] = (block7 >>> 50) & 2047L; + values[vi++] = (block7 >>> 39) & 2047L; + values[vi++] = (block7 >>> 28) & 2047L; + values[vi++] = (block7 >>> 17) & 2047L; + values[vi++] = (block7 >>> 6) & 2047L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 63L) << 5) | (block8 >>> 59); + values[vi++] = (block8 >>> 48) & 2047L; + values[vi++] = (block8 >>> 37) & 2047L; + values[vi++] = (block8 >>> 26) & 2047L; + values[vi++] = (block8 >>> 15) & 2047L; + values[vi++] = (block8 >>> 4) & 2047L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 15L) << 7) | (block9 >>> 57); + values[vi++] = (block9 >>> 46) & 2047L; + values[vi++] = (block9 >>> 35) & 2047L; + values[vi++] = (block9 >>> 24) & 2047L; + values[vi++] = (block9 >>> 13) & 2047L; + values[vi++] = (block9 >>> 2) & 2047L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 3L) << 9) | (block10 >>> 55); + values[vi++] = (block10 >>> 44) & 2047L; + values[vi++] = (block10 >>> 33) & 2047L; + values[vi++] = (block10 >>> 22) & 2047L; + values[vi++] = (block10 >>> 11) & 2047L; + values[vi++] = block10 & 2047L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 53) | (values[vi++] << 42) | (values[vi++] << 31) | (values[vi++] << 20) | (values[vi++] << 9) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 51) | (values[vi++] << 40) | (values[vi++] << 29) | (values[vi++] << 18) | (values[vi++] << 7) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 49) | (values[vi++] << 38) | (values[vi++] << 27) | (values[vi++] << 16) | (values[vi++] << 5) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 47) | (values[vi++] << 36) | (values[vi++] << 25) | (values[vi++] << 14) | (values[vi++] << 3) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 45) | (values[vi++] << 34) | (values[vi++] << 23) | (values[vi++] << 12) | (values[vi++] << 1) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 43) | (values[vi++] << 32) | (values[vi++] << 21) | (values[vi++] << 10) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 52) | (values[vi++] << 41) | (values[vi++] << 30) | (values[vi++] << 19) | (values[vi++] << 8) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 50) | (values[vi++] << 39) | (values[vi++] << 28) | (values[vi++] << 17) | (values[vi++] << 6) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 48) | (values[vi++] << 37) | (values[vi++] << 26) | (values[vi++] << 15) | (values[vi++] << 4) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 46) | (values[vi++] << 35) | (values[vi++] << 24) | (values[vi++] << 13) | (values[vi++] << 2) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 44) | (values[vi++] << 33) | (values[vi++] << 22) | (values[vi++] << 11) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation12 extends BulkOperation { + + public int blocks() { + return 3; + } + + public int values() { + return 16; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 52; + values[vi++] = (block0 >>> 40) & 4095L; + values[vi++] = (block0 >>> 28) & 4095L; + values[vi++] = (block0 >>> 16) & 4095L; + values[vi++] = (block0 >>> 4) & 4095L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 15L) << 8) | (block1 >>> 56); + values[vi++] = (block1 >>> 44) & 4095L; + values[vi++] = (block1 >>> 32) & 4095L; + values[vi++] = (block1 >>> 20) & 4095L; + values[vi++] = (block1 >>> 8) & 4095L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 255L) << 4) | (block2 >>> 60); + values[vi++] = (block2 >>> 48) & 4095L; + values[vi++] = (block2 >>> 36) & 4095L; + values[vi++] = (block2 >>> 24) & 4095L; + values[vi++] = (block2 >>> 12) & 4095L; + values[vi++] = block2 & 4095L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 40) | (values[vi++] << 28) | (values[vi++] << 16) | (values[vi++] << 4) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 44) | (values[vi++] << 32) | (values[vi++] << 20) | (values[vi++] << 8) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 48) | (values[vi++] << 36) | (values[vi++] << 24) | (values[vi++] << 12) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation13 extends BulkOperation { + + public int blocks() { + return 13; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 51; + values[vi++] = (block0 >>> 38) & 8191L; + values[vi++] = (block0 >>> 25) & 8191L; + values[vi++] = (block0 >>> 12) & 8191L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 4095L) << 1) | (block1 >>> 63); + values[vi++] = (block1 >>> 50) & 8191L; + values[vi++] = (block1 >>> 37) & 8191L; + values[vi++] = (block1 >>> 24) & 8191L; + values[vi++] = (block1 >>> 11) & 8191L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 2047L) << 2) | (block2 >>> 62); + values[vi++] = (block2 >>> 49) & 8191L; + values[vi++] = (block2 >>> 36) & 8191L; + values[vi++] = (block2 >>> 23) & 8191L; + values[vi++] = (block2 >>> 10) & 8191L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 1023L) << 3) | (block3 >>> 61); + values[vi++] = (block3 >>> 48) & 8191L; + values[vi++] = (block3 >>> 35) & 8191L; + values[vi++] = (block3 >>> 22) & 8191L; + values[vi++] = (block3 >>> 9) & 8191L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 511L) << 4) | (block4 >>> 60); + values[vi++] = (block4 >>> 47) & 8191L; + values[vi++] = (block4 >>> 34) & 8191L; + values[vi++] = (block4 >>> 21) & 8191L; + values[vi++] = (block4 >>> 8) & 8191L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 255L) << 5) | (block5 >>> 59); + values[vi++] = (block5 >>> 46) & 8191L; + values[vi++] = (block5 >>> 33) & 8191L; + values[vi++] = (block5 >>> 20) & 8191L; + values[vi++] = (block5 >>> 7) & 8191L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 127L) << 6) | (block6 >>> 58); + values[vi++] = (block6 >>> 45) & 8191L; + values[vi++] = (block6 >>> 32) & 8191L; + values[vi++] = (block6 >>> 19) & 8191L; + values[vi++] = (block6 >>> 6) & 8191L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 63L) << 7) | (block7 >>> 57); + values[vi++] = (block7 >>> 44) & 8191L; + values[vi++] = (block7 >>> 31) & 8191L; + values[vi++] = (block7 >>> 18) & 8191L; + values[vi++] = (block7 >>> 5) & 8191L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 31L) << 8) | (block8 >>> 56); + values[vi++] = (block8 >>> 43) & 8191L; + values[vi++] = (block8 >>> 30) & 8191L; + values[vi++] = (block8 >>> 17) & 8191L; + values[vi++] = (block8 >>> 4) & 8191L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 15L) << 9) | (block9 >>> 55); + values[vi++] = (block9 >>> 42) & 8191L; + values[vi++] = (block9 >>> 29) & 8191L; + values[vi++] = (block9 >>> 16) & 8191L; + values[vi++] = (block9 >>> 3) & 8191L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 7L) << 10) | (block10 >>> 54); + values[vi++] = (block10 >>> 41) & 8191L; + values[vi++] = (block10 >>> 28) & 8191L; + values[vi++] = (block10 >>> 15) & 8191L; + values[vi++] = (block10 >>> 2) & 8191L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 3L) << 11) | (block11 >>> 53); + values[vi++] = (block11 >>> 40) & 8191L; + values[vi++] = (block11 >>> 27) & 8191L; + values[vi++] = (block11 >>> 14) & 8191L; + values[vi++] = (block11 >>> 1) & 8191L; + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 1L) << 12) | (block12 >>> 52); + values[vi++] = (block12 >>> 39) & 8191L; + values[vi++] = (block12 >>> 26) & 8191L; + values[vi++] = (block12 >>> 13) & 8191L; + values[vi++] = block12 & 8191L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 51) | (values[vi++] << 38) | (values[vi++] << 25) | (values[vi++] << 12) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 50) | (values[vi++] << 37) | (values[vi++] << 24) | (values[vi++] << 11) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 49) | (values[vi++] << 36) | (values[vi++] << 23) | (values[vi++] << 10) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 48) | (values[vi++] << 35) | (values[vi++] << 22) | (values[vi++] << 9) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 47) | (values[vi++] << 34) | (values[vi++] << 21) | (values[vi++] << 8) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 46) | (values[vi++] << 33) | (values[vi++] << 20) | (values[vi++] << 7) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 45) | (values[vi++] << 32) | (values[vi++] << 19) | (values[vi++] << 6) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 44) | (values[vi++] << 31) | (values[vi++] << 18) | (values[vi++] << 5) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 43) | (values[vi++] << 30) | (values[vi++] << 17) | (values[vi++] << 4) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 42) | (values[vi++] << 29) | (values[vi++] << 16) | (values[vi++] << 3) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 41) | (values[vi++] << 28) | (values[vi++] << 15) | (values[vi++] << 2) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi++] << 40) | (values[vi++] << 27) | (values[vi++] << 14) | (values[vi++] << 1) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 39) | (values[vi++] << 26) | (values[vi++] << 13) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation14 extends BulkOperation { + + public int blocks() { + return 7; + } + + public int values() { + return 32; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 50; + values[vi++] = (block0 >>> 36) & 16383L; + values[vi++] = (block0 >>> 22) & 16383L; + values[vi++] = (block0 >>> 8) & 16383L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 255L) << 6) | (block1 >>> 58); + values[vi++] = (block1 >>> 44) & 16383L; + values[vi++] = (block1 >>> 30) & 16383L; + values[vi++] = (block1 >>> 16) & 16383L; + values[vi++] = (block1 >>> 2) & 16383L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 3L) << 12) | (block2 >>> 52); + values[vi++] = (block2 >>> 38) & 16383L; + values[vi++] = (block2 >>> 24) & 16383L; + values[vi++] = (block2 >>> 10) & 16383L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 1023L) << 4) | (block3 >>> 60); + values[vi++] = (block3 >>> 46) & 16383L; + values[vi++] = (block3 >>> 32) & 16383L; + values[vi++] = (block3 >>> 18) & 16383L; + values[vi++] = (block3 >>> 4) & 16383L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 15L) << 10) | (block4 >>> 54); + values[vi++] = (block4 >>> 40) & 16383L; + values[vi++] = (block4 >>> 26) & 16383L; + values[vi++] = (block4 >>> 12) & 16383L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 4095L) << 2) | (block5 >>> 62); + values[vi++] = (block5 >>> 48) & 16383L; + values[vi++] = (block5 >>> 34) & 16383L; + values[vi++] = (block5 >>> 20) & 16383L; + values[vi++] = (block5 >>> 6) & 16383L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 63L) << 8) | (block6 >>> 56); + values[vi++] = (block6 >>> 42) & 16383L; + values[vi++] = (block6 >>> 28) & 16383L; + values[vi++] = (block6 >>> 14) & 16383L; + values[vi++] = block6 & 16383L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 36) | (values[vi++] << 22) | (values[vi++] << 8) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 44) | (values[vi++] << 30) | (values[vi++] << 16) | (values[vi++] << 2) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 38) | (values[vi++] << 24) | (values[vi++] << 10) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 46) | (values[vi++] << 32) | (values[vi++] << 18) | (values[vi++] << 4) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 40) | (values[vi++] << 26) | (values[vi++] << 12) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 48) | (values[vi++] << 34) | (values[vi++] << 20) | (values[vi++] << 6) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 42) | (values[vi++] << 28) | (values[vi++] << 14) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation15 extends BulkOperation { + + public int blocks() { + return 15; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 49; + values[vi++] = (block0 >>> 34) & 32767L; + values[vi++] = (block0 >>> 19) & 32767L; + values[vi++] = (block0 >>> 4) & 32767L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 15L) << 11) | (block1 >>> 53); + values[vi++] = (block1 >>> 38) & 32767L; + values[vi++] = (block1 >>> 23) & 32767L; + values[vi++] = (block1 >>> 8) & 32767L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 255L) << 7) | (block2 >>> 57); + values[vi++] = (block2 >>> 42) & 32767L; + values[vi++] = (block2 >>> 27) & 32767L; + values[vi++] = (block2 >>> 12) & 32767L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 4095L) << 3) | (block3 >>> 61); + values[vi++] = (block3 >>> 46) & 32767L; + values[vi++] = (block3 >>> 31) & 32767L; + values[vi++] = (block3 >>> 16) & 32767L; + values[vi++] = (block3 >>> 1) & 32767L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 1L) << 14) | (block4 >>> 50); + values[vi++] = (block4 >>> 35) & 32767L; + values[vi++] = (block4 >>> 20) & 32767L; + values[vi++] = (block4 >>> 5) & 32767L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 31L) << 10) | (block5 >>> 54); + values[vi++] = (block5 >>> 39) & 32767L; + values[vi++] = (block5 >>> 24) & 32767L; + values[vi++] = (block5 >>> 9) & 32767L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 511L) << 6) | (block6 >>> 58); + values[vi++] = (block6 >>> 43) & 32767L; + values[vi++] = (block6 >>> 28) & 32767L; + values[vi++] = (block6 >>> 13) & 32767L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 8191L) << 2) | (block7 >>> 62); + values[vi++] = (block7 >>> 47) & 32767L; + values[vi++] = (block7 >>> 32) & 32767L; + values[vi++] = (block7 >>> 17) & 32767L; + values[vi++] = (block7 >>> 2) & 32767L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 3L) << 13) | (block8 >>> 51); + values[vi++] = (block8 >>> 36) & 32767L; + values[vi++] = (block8 >>> 21) & 32767L; + values[vi++] = (block8 >>> 6) & 32767L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 63L) << 9) | (block9 >>> 55); + values[vi++] = (block9 >>> 40) & 32767L; + values[vi++] = (block9 >>> 25) & 32767L; + values[vi++] = (block9 >>> 10) & 32767L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 1023L) << 5) | (block10 >>> 59); + values[vi++] = (block10 >>> 44) & 32767L; + values[vi++] = (block10 >>> 29) & 32767L; + values[vi++] = (block10 >>> 14) & 32767L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 16383L) << 1) | (block11 >>> 63); + values[vi++] = (block11 >>> 48) & 32767L; + values[vi++] = (block11 >>> 33) & 32767L; + values[vi++] = (block11 >>> 18) & 32767L; + values[vi++] = (block11 >>> 3) & 32767L; + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 7L) << 12) | (block12 >>> 52); + values[vi++] = (block12 >>> 37) & 32767L; + values[vi++] = (block12 >>> 22) & 32767L; + values[vi++] = (block12 >>> 7) & 32767L; + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 127L) << 8) | (block13 >>> 56); + values[vi++] = (block13 >>> 41) & 32767L; + values[vi++] = (block13 >>> 26) & 32767L; + values[vi++] = (block13 >>> 11) & 32767L; + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 2047L) << 4) | (block14 >>> 60); + values[vi++] = (block14 >>> 45) & 32767L; + values[vi++] = (block14 >>> 30) & 32767L; + values[vi++] = (block14 >>> 15) & 32767L; + values[vi++] = block14 & 32767L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 49) | (values[vi++] << 34) | (values[vi++] << 19) | (values[vi++] << 4) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi++] << 38) | (values[vi++] << 23) | (values[vi++] << 8) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 42) | (values[vi++] << 27) | (values[vi++] << 12) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 46) | (values[vi++] << 31) | (values[vi++] << 16) | (values[vi++] << 1) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 35) | (values[vi++] << 20) | (values[vi++] << 5) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 39) | (values[vi++] << 24) | (values[vi++] << 9) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 43) | (values[vi++] << 28) | (values[vi++] << 13) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 47) | (values[vi++] << 32) | (values[vi++] << 17) | (values[vi++] << 2) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi++] << 36) | (values[vi++] << 21) | (values[vi++] << 6) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 40) | (values[vi++] << 25) | (values[vi++] << 10) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 44) | (values[vi++] << 29) | (values[vi++] << 14) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 48) | (values[vi++] << 33) | (values[vi++] << 18) | (values[vi++] << 3) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 37) | (values[vi++] << 22) | (values[vi++] << 7) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 41) | (values[vi++] << 26) | (values[vi++] << 11) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 45) | (values[vi++] << 30) | (values[vi++] << 15) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation16 extends BulkOperation { + + public int blocks() { + return 1; + } + + public int values() { + return 4; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 48; + values[vi++] = (block0 >>> 32) & 65535L; + values[vi++] = (block0 >>> 16) & 65535L; + values[vi++] = block0 & 65535L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 32) | (values[vi++] << 16) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation17 extends BulkOperation { + + public int blocks() { + return 17; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 47; + values[vi++] = (block0 >>> 30) & 131071L; + values[vi++] = (block0 >>> 13) & 131071L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 8191L) << 4) | (block1 >>> 60); + values[vi++] = (block1 >>> 43) & 131071L; + values[vi++] = (block1 >>> 26) & 131071L; + values[vi++] = (block1 >>> 9) & 131071L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 511L) << 8) | (block2 >>> 56); + values[vi++] = (block2 >>> 39) & 131071L; + values[vi++] = (block2 >>> 22) & 131071L; + values[vi++] = (block2 >>> 5) & 131071L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 31L) << 12) | (block3 >>> 52); + values[vi++] = (block3 >>> 35) & 131071L; + values[vi++] = (block3 >>> 18) & 131071L; + values[vi++] = (block3 >>> 1) & 131071L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 1L) << 16) | (block4 >>> 48); + values[vi++] = (block4 >>> 31) & 131071L; + values[vi++] = (block4 >>> 14) & 131071L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 16383L) << 3) | (block5 >>> 61); + values[vi++] = (block5 >>> 44) & 131071L; + values[vi++] = (block5 >>> 27) & 131071L; + values[vi++] = (block5 >>> 10) & 131071L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 1023L) << 7) | (block6 >>> 57); + values[vi++] = (block6 >>> 40) & 131071L; + values[vi++] = (block6 >>> 23) & 131071L; + values[vi++] = (block6 >>> 6) & 131071L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 63L) << 11) | (block7 >>> 53); + values[vi++] = (block7 >>> 36) & 131071L; + values[vi++] = (block7 >>> 19) & 131071L; + values[vi++] = (block7 >>> 2) & 131071L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 3L) << 15) | (block8 >>> 49); + values[vi++] = (block8 >>> 32) & 131071L; + values[vi++] = (block8 >>> 15) & 131071L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 32767L) << 2) | (block9 >>> 62); + values[vi++] = (block9 >>> 45) & 131071L; + values[vi++] = (block9 >>> 28) & 131071L; + values[vi++] = (block9 >>> 11) & 131071L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 2047L) << 6) | (block10 >>> 58); + values[vi++] = (block10 >>> 41) & 131071L; + values[vi++] = (block10 >>> 24) & 131071L; + values[vi++] = (block10 >>> 7) & 131071L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 127L) << 10) | (block11 >>> 54); + values[vi++] = (block11 >>> 37) & 131071L; + values[vi++] = (block11 >>> 20) & 131071L; + values[vi++] = (block11 >>> 3) & 131071L; + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 7L) << 14) | (block12 >>> 50); + values[vi++] = (block12 >>> 33) & 131071L; + values[vi++] = (block12 >>> 16) & 131071L; + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 65535L) << 1) | (block13 >>> 63); + values[vi++] = (block13 >>> 46) & 131071L; + values[vi++] = (block13 >>> 29) & 131071L; + values[vi++] = (block13 >>> 12) & 131071L; + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 4095L) << 5) | (block14 >>> 59); + values[vi++] = (block14 >>> 42) & 131071L; + values[vi++] = (block14 >>> 25) & 131071L; + values[vi++] = (block14 >>> 8) & 131071L; + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 255L) << 9) | (block15 >>> 55); + values[vi++] = (block15 >>> 38) & 131071L; + values[vi++] = (block15 >>> 21) & 131071L; + values[vi++] = (block15 >>> 4) & 131071L; + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 15L) << 13) | (block16 >>> 51); + values[vi++] = (block16 >>> 34) & 131071L; + values[vi++] = (block16 >>> 17) & 131071L; + values[vi++] = block16 & 131071L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 47) | (values[vi++] << 30) | (values[vi++] << 13) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 43) | (values[vi++] << 26) | (values[vi++] << 9) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 39) | (values[vi++] << 22) | (values[vi++] << 5) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 35) | (values[vi++] << 18) | (values[vi++] << 1) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 31) | (values[vi++] << 14) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 44) | (values[vi++] << 27) | (values[vi++] << 10) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 40) | (values[vi++] << 23) | (values[vi++] << 6) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi++] << 36) | (values[vi++] << 19) | (values[vi++] << 2) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi++] << 32) | (values[vi++] << 15) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 45) | (values[vi++] << 28) | (values[vi++] << 11) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 41) | (values[vi++] << 24) | (values[vi++] << 7) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 37) | (values[vi++] << 20) | (values[vi++] << 3) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 33) | (values[vi++] << 16) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 46) | (values[vi++] << 29) | (values[vi++] << 12) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 42) | (values[vi++] << 25) | (values[vi++] << 8) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 38) | (values[vi++] << 21) | (values[vi++] << 4) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi++] << 34) | (values[vi++] << 17) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation18 extends BulkOperation { + + public int blocks() { + return 9; + } + + public int values() { + return 32; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 46; + values[vi++] = (block0 >>> 28) & 262143L; + values[vi++] = (block0 >>> 10) & 262143L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 1023L) << 8) | (block1 >>> 56); + values[vi++] = (block1 >>> 38) & 262143L; + values[vi++] = (block1 >>> 20) & 262143L; + values[vi++] = (block1 >>> 2) & 262143L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 3L) << 16) | (block2 >>> 48); + values[vi++] = (block2 >>> 30) & 262143L; + values[vi++] = (block2 >>> 12) & 262143L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 4095L) << 6) | (block3 >>> 58); + values[vi++] = (block3 >>> 40) & 262143L; + values[vi++] = (block3 >>> 22) & 262143L; + values[vi++] = (block3 >>> 4) & 262143L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 15L) << 14) | (block4 >>> 50); + values[vi++] = (block4 >>> 32) & 262143L; + values[vi++] = (block4 >>> 14) & 262143L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 16383L) << 4) | (block5 >>> 60); + values[vi++] = (block5 >>> 42) & 262143L; + values[vi++] = (block5 >>> 24) & 262143L; + values[vi++] = (block5 >>> 6) & 262143L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 63L) << 12) | (block6 >>> 52); + values[vi++] = (block6 >>> 34) & 262143L; + values[vi++] = (block6 >>> 16) & 262143L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 65535L) << 2) | (block7 >>> 62); + values[vi++] = (block7 >>> 44) & 262143L; + values[vi++] = (block7 >>> 26) & 262143L; + values[vi++] = (block7 >>> 8) & 262143L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 255L) << 10) | (block8 >>> 54); + values[vi++] = (block8 >>> 36) & 262143L; + values[vi++] = (block8 >>> 18) & 262143L; + values[vi++] = block8 & 262143L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 46) | (values[vi++] << 28) | (values[vi++] << 10) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 38) | (values[vi++] << 20) | (values[vi++] << 2) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 30) | (values[vi++] << 12) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 40) | (values[vi++] << 22) | (values[vi++] << 4) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 32) | (values[vi++] << 14) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 42) | (values[vi++] << 24) | (values[vi++] << 6) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 34) | (values[vi++] << 16) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 44) | (values[vi++] << 26) | (values[vi++] << 8) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 36) | (values[vi++] << 18) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation19 extends BulkOperation { + + public int blocks() { + return 19; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 45; + values[vi++] = (block0 >>> 26) & 524287L; + values[vi++] = (block0 >>> 7) & 524287L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 127L) << 12) | (block1 >>> 52); + values[vi++] = (block1 >>> 33) & 524287L; + values[vi++] = (block1 >>> 14) & 524287L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 16383L) << 5) | (block2 >>> 59); + values[vi++] = (block2 >>> 40) & 524287L; + values[vi++] = (block2 >>> 21) & 524287L; + values[vi++] = (block2 >>> 2) & 524287L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 3L) << 17) | (block3 >>> 47); + values[vi++] = (block3 >>> 28) & 524287L; + values[vi++] = (block3 >>> 9) & 524287L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 511L) << 10) | (block4 >>> 54); + values[vi++] = (block4 >>> 35) & 524287L; + values[vi++] = (block4 >>> 16) & 524287L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 65535L) << 3) | (block5 >>> 61); + values[vi++] = (block5 >>> 42) & 524287L; + values[vi++] = (block5 >>> 23) & 524287L; + values[vi++] = (block5 >>> 4) & 524287L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 15L) << 15) | (block6 >>> 49); + values[vi++] = (block6 >>> 30) & 524287L; + values[vi++] = (block6 >>> 11) & 524287L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 2047L) << 8) | (block7 >>> 56); + values[vi++] = (block7 >>> 37) & 524287L; + values[vi++] = (block7 >>> 18) & 524287L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 262143L) << 1) | (block8 >>> 63); + values[vi++] = (block8 >>> 44) & 524287L; + values[vi++] = (block8 >>> 25) & 524287L; + values[vi++] = (block8 >>> 6) & 524287L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 63L) << 13) | (block9 >>> 51); + values[vi++] = (block9 >>> 32) & 524287L; + values[vi++] = (block9 >>> 13) & 524287L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 8191L) << 6) | (block10 >>> 58); + values[vi++] = (block10 >>> 39) & 524287L; + values[vi++] = (block10 >>> 20) & 524287L; + values[vi++] = (block10 >>> 1) & 524287L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 1L) << 18) | (block11 >>> 46); + values[vi++] = (block11 >>> 27) & 524287L; + values[vi++] = (block11 >>> 8) & 524287L; + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 255L) << 11) | (block12 >>> 53); + values[vi++] = (block12 >>> 34) & 524287L; + values[vi++] = (block12 >>> 15) & 524287L; + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 32767L) << 4) | (block13 >>> 60); + values[vi++] = (block13 >>> 41) & 524287L; + values[vi++] = (block13 >>> 22) & 524287L; + values[vi++] = (block13 >>> 3) & 524287L; + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 7L) << 16) | (block14 >>> 48); + values[vi++] = (block14 >>> 29) & 524287L; + values[vi++] = (block14 >>> 10) & 524287L; + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 1023L) << 9) | (block15 >>> 55); + values[vi++] = (block15 >>> 36) & 524287L; + values[vi++] = (block15 >>> 17) & 524287L; + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 131071L) << 2) | (block16 >>> 62); + values[vi++] = (block16 >>> 43) & 524287L; + values[vi++] = (block16 >>> 24) & 524287L; + values[vi++] = (block16 >>> 5) & 524287L; + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 31L) << 14) | (block17 >>> 50); + values[vi++] = (block17 >>> 31) & 524287L; + values[vi++] = (block17 >>> 12) & 524287L; + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 4095L) << 7) | (block18 >>> 57); + values[vi++] = (block18 >>> 38) & 524287L; + values[vi++] = (block18 >>> 19) & 524287L; + values[vi++] = block18 & 524287L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 45) | (values[vi++] << 26) | (values[vi++] << 7) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 33) | (values[vi++] << 14) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 40) | (values[vi++] << 21) | (values[vi++] << 2) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi++] << 28) | (values[vi++] << 9) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 35) | (values[vi++] << 16) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 42) | (values[vi++] << 23) | (values[vi++] << 4) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi++] << 30) | (values[vi++] << 11) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 37) | (values[vi++] << 18) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 44) | (values[vi++] << 25) | (values[vi++] << 6) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi++] << 32) | (values[vi++] << 13) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 39) | (values[vi++] << 20) | (values[vi++] << 1) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi++] << 27) | (values[vi++] << 8) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi++] << 34) | (values[vi++] << 15) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 41) | (values[vi++] << 22) | (values[vi++] << 3) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 29) | (values[vi++] << 10) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 36) | (values[vi++] << 17) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 43) | (values[vi++] << 24) | (values[vi++] << 5) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 31) | (values[vi++] << 12) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 38) | (values[vi++] << 19) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation20 extends BulkOperation { + + public int blocks() { + return 5; + } + + public int values() { + return 16; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 44; + values[vi++] = (block0 >>> 24) & 1048575L; + values[vi++] = (block0 >>> 4) & 1048575L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 15L) << 16) | (block1 >>> 48); + values[vi++] = (block1 >>> 28) & 1048575L; + values[vi++] = (block1 >>> 8) & 1048575L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 255L) << 12) | (block2 >>> 52); + values[vi++] = (block2 >>> 32) & 1048575L; + values[vi++] = (block2 >>> 12) & 1048575L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 4095L) << 8) | (block3 >>> 56); + values[vi++] = (block3 >>> 36) & 1048575L; + values[vi++] = (block3 >>> 16) & 1048575L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 65535L) << 4) | (block4 >>> 60); + values[vi++] = (block4 >>> 40) & 1048575L; + values[vi++] = (block4 >>> 20) & 1048575L; + values[vi++] = block4 & 1048575L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 44) | (values[vi++] << 24) | (values[vi++] << 4) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 28) | (values[vi++] << 8) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 32) | (values[vi++] << 12) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 36) | (values[vi++] << 16) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 40) | (values[vi++] << 20) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation21 extends BulkOperation { + + public int blocks() { + return 21; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 43; + values[vi++] = (block0 >>> 22) & 2097151L; + values[vi++] = (block0 >>> 1) & 2097151L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 1L) << 20) | (block1 >>> 44); + values[vi++] = (block1 >>> 23) & 2097151L; + values[vi++] = (block1 >>> 2) & 2097151L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 3L) << 19) | (block2 >>> 45); + values[vi++] = (block2 >>> 24) & 2097151L; + values[vi++] = (block2 >>> 3) & 2097151L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 7L) << 18) | (block3 >>> 46); + values[vi++] = (block3 >>> 25) & 2097151L; + values[vi++] = (block3 >>> 4) & 2097151L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 15L) << 17) | (block4 >>> 47); + values[vi++] = (block4 >>> 26) & 2097151L; + values[vi++] = (block4 >>> 5) & 2097151L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 31L) << 16) | (block5 >>> 48); + values[vi++] = (block5 >>> 27) & 2097151L; + values[vi++] = (block5 >>> 6) & 2097151L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 63L) << 15) | (block6 >>> 49); + values[vi++] = (block6 >>> 28) & 2097151L; + values[vi++] = (block6 >>> 7) & 2097151L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 127L) << 14) | (block7 >>> 50); + values[vi++] = (block7 >>> 29) & 2097151L; + values[vi++] = (block7 >>> 8) & 2097151L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 255L) << 13) | (block8 >>> 51); + values[vi++] = (block8 >>> 30) & 2097151L; + values[vi++] = (block8 >>> 9) & 2097151L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 511L) << 12) | (block9 >>> 52); + values[vi++] = (block9 >>> 31) & 2097151L; + values[vi++] = (block9 >>> 10) & 2097151L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 1023L) << 11) | (block10 >>> 53); + values[vi++] = (block10 >>> 32) & 2097151L; + values[vi++] = (block10 >>> 11) & 2097151L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 2047L) << 10) | (block11 >>> 54); + values[vi++] = (block11 >>> 33) & 2097151L; + values[vi++] = (block11 >>> 12) & 2097151L; + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 4095L) << 9) | (block12 >>> 55); + values[vi++] = (block12 >>> 34) & 2097151L; + values[vi++] = (block12 >>> 13) & 2097151L; + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 8191L) << 8) | (block13 >>> 56); + values[vi++] = (block13 >>> 35) & 2097151L; + values[vi++] = (block13 >>> 14) & 2097151L; + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 16383L) << 7) | (block14 >>> 57); + values[vi++] = (block14 >>> 36) & 2097151L; + values[vi++] = (block14 >>> 15) & 2097151L; + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 32767L) << 6) | (block15 >>> 58); + values[vi++] = (block15 >>> 37) & 2097151L; + values[vi++] = (block15 >>> 16) & 2097151L; + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 65535L) << 5) | (block16 >>> 59); + values[vi++] = (block16 >>> 38) & 2097151L; + values[vi++] = (block16 >>> 17) & 2097151L; + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 131071L) << 4) | (block17 >>> 60); + values[vi++] = (block17 >>> 39) & 2097151L; + values[vi++] = (block17 >>> 18) & 2097151L; + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 262143L) << 3) | (block18 >>> 61); + values[vi++] = (block18 >>> 40) & 2097151L; + values[vi++] = (block18 >>> 19) & 2097151L; + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 524287L) << 2) | (block19 >>> 62); + values[vi++] = (block19 >>> 41) & 2097151L; + values[vi++] = (block19 >>> 20) & 2097151L; + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 1048575L) << 1) | (block20 >>> 63); + values[vi++] = (block20 >>> 42) & 2097151L; + values[vi++] = (block20 >>> 21) & 2097151L; + values[vi++] = block20 & 2097151L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 43) | (values[vi++] << 22) | (values[vi++] << 1) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi++] << 23) | (values[vi++] << 2) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | (values[vi++] << 24) | (values[vi++] << 3) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi++] << 25) | (values[vi++] << 4) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi++] << 26) | (values[vi++] << 5) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 27) | (values[vi++] << 6) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi++] << 28) | (values[vi++] << 7) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 29) | (values[vi++] << 8) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi++] << 30) | (values[vi++] << 9) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 31) | (values[vi++] << 10) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi++] << 32) | (values[vi++] << 11) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 33) | (values[vi++] << 12) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 34) | (values[vi++] << 13) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 35) | (values[vi++] << 14) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 36) | (values[vi++] << 15) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 37) | (values[vi++] << 16) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 38) | (values[vi++] << 17) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 39) | (values[vi++] << 18) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 40) | (values[vi++] << 19) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 41) | (values[vi++] << 20) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 42) | (values[vi++] << 21) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation22 extends BulkOperation { + + public int blocks() { + return 11; + } + + public int values() { + return 32; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 42; + values[vi++] = (block0 >>> 20) & 4194303L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 1048575L) << 2) | (block1 >>> 62); + values[vi++] = (block1 >>> 40) & 4194303L; + values[vi++] = (block1 >>> 18) & 4194303L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 262143L) << 4) | (block2 >>> 60); + values[vi++] = (block2 >>> 38) & 4194303L; + values[vi++] = (block2 >>> 16) & 4194303L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 65535L) << 6) | (block3 >>> 58); + values[vi++] = (block3 >>> 36) & 4194303L; + values[vi++] = (block3 >>> 14) & 4194303L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 16383L) << 8) | (block4 >>> 56); + values[vi++] = (block4 >>> 34) & 4194303L; + values[vi++] = (block4 >>> 12) & 4194303L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 4095L) << 10) | (block5 >>> 54); + values[vi++] = (block5 >>> 32) & 4194303L; + values[vi++] = (block5 >>> 10) & 4194303L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 1023L) << 12) | (block6 >>> 52); + values[vi++] = (block6 >>> 30) & 4194303L; + values[vi++] = (block6 >>> 8) & 4194303L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 255L) << 14) | (block7 >>> 50); + values[vi++] = (block7 >>> 28) & 4194303L; + values[vi++] = (block7 >>> 6) & 4194303L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 63L) << 16) | (block8 >>> 48); + values[vi++] = (block8 >>> 26) & 4194303L; + values[vi++] = (block8 >>> 4) & 4194303L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 15L) << 18) | (block9 >>> 46); + values[vi++] = (block9 >>> 24) & 4194303L; + values[vi++] = (block9 >>> 2) & 4194303L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 3L) << 20) | (block10 >>> 44); + values[vi++] = (block10 >>> 22) & 4194303L; + values[vi++] = block10 & 4194303L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 42) | (values[vi++] << 20) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 40) | (values[vi++] << 18) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 38) | (values[vi++] << 16) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 36) | (values[vi++] << 14) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 34) | (values[vi++] << 12) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 32) | (values[vi++] << 10) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 30) | (values[vi++] << 8) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 28) | (values[vi++] << 6) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 26) | (values[vi++] << 4) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi++] << 24) | (values[vi++] << 2) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi++] << 22) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation23 extends BulkOperation { + + public int blocks() { + return 23; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 41; + values[vi++] = (block0 >>> 18) & 8388607L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 262143L) << 5) | (block1 >>> 59); + values[vi++] = (block1 >>> 36) & 8388607L; + values[vi++] = (block1 >>> 13) & 8388607L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 8191L) << 10) | (block2 >>> 54); + values[vi++] = (block2 >>> 31) & 8388607L; + values[vi++] = (block2 >>> 8) & 8388607L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 255L) << 15) | (block3 >>> 49); + values[vi++] = (block3 >>> 26) & 8388607L; + values[vi++] = (block3 >>> 3) & 8388607L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 7L) << 20) | (block4 >>> 44); + values[vi++] = (block4 >>> 21) & 8388607L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 2097151L) << 2) | (block5 >>> 62); + values[vi++] = (block5 >>> 39) & 8388607L; + values[vi++] = (block5 >>> 16) & 8388607L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 65535L) << 7) | (block6 >>> 57); + values[vi++] = (block6 >>> 34) & 8388607L; + values[vi++] = (block6 >>> 11) & 8388607L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 2047L) << 12) | (block7 >>> 52); + values[vi++] = (block7 >>> 29) & 8388607L; + values[vi++] = (block7 >>> 6) & 8388607L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 63L) << 17) | (block8 >>> 47); + values[vi++] = (block8 >>> 24) & 8388607L; + values[vi++] = (block8 >>> 1) & 8388607L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 1L) << 22) | (block9 >>> 42); + values[vi++] = (block9 >>> 19) & 8388607L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 524287L) << 4) | (block10 >>> 60); + values[vi++] = (block10 >>> 37) & 8388607L; + values[vi++] = (block10 >>> 14) & 8388607L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 16383L) << 9) | (block11 >>> 55); + values[vi++] = (block11 >>> 32) & 8388607L; + values[vi++] = (block11 >>> 9) & 8388607L; + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 511L) << 14) | (block12 >>> 50); + values[vi++] = (block12 >>> 27) & 8388607L; + values[vi++] = (block12 >>> 4) & 8388607L; + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 15L) << 19) | (block13 >>> 45); + values[vi++] = (block13 >>> 22) & 8388607L; + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 4194303L) << 1) | (block14 >>> 63); + values[vi++] = (block14 >>> 40) & 8388607L; + values[vi++] = (block14 >>> 17) & 8388607L; + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 131071L) << 6) | (block15 >>> 58); + values[vi++] = (block15 >>> 35) & 8388607L; + values[vi++] = (block15 >>> 12) & 8388607L; + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 4095L) << 11) | (block16 >>> 53); + values[vi++] = (block16 >>> 30) & 8388607L; + values[vi++] = (block16 >>> 7) & 8388607L; + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 127L) << 16) | (block17 >>> 48); + values[vi++] = (block17 >>> 25) & 8388607L; + values[vi++] = (block17 >>> 2) & 8388607L; + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 3L) << 21) | (block18 >>> 43); + values[vi++] = (block18 >>> 20) & 8388607L; + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 1048575L) << 3) | (block19 >>> 61); + values[vi++] = (block19 >>> 38) & 8388607L; + values[vi++] = (block19 >>> 15) & 8388607L; + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 32767L) << 8) | (block20 >>> 56); + values[vi++] = (block20 >>> 33) & 8388607L; + values[vi++] = (block20 >>> 10) & 8388607L; + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 1023L) << 13) | (block21 >>> 51); + values[vi++] = (block21 >>> 28) & 8388607L; + values[vi++] = (block21 >>> 5) & 8388607L; + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 31L) << 18) | (block22 >>> 46); + values[vi++] = (block22 >>> 23) & 8388607L; + values[vi++] = block22 & 8388607L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 41) | (values[vi++] << 18) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 36) | (values[vi++] << 13) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 31) | (values[vi++] << 8) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi++] << 26) | (values[vi++] << 3) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi++] << 21) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 39) | (values[vi++] << 16) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 34) | (values[vi++] << 11) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 29) | (values[vi++] << 6) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi++] << 24) | (values[vi++] << 1) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi++] << 19) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 37) | (values[vi++] << 14) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 32) | (values[vi++] << 9) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 27) | (values[vi++] << 4) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | (values[vi++] << 22) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 40) | (values[vi++] << 17) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 35) | (values[vi++] << 12) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi++] << 30) | (values[vi++] << 7) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 25) | (values[vi++] << 2) | (values[vi] >>> 21); + blocks[bi++] = (values[vi++] << 43) | (values[vi++] << 20) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 38) | (values[vi++] << 15) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 33) | (values[vi++] << 10) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi++] << 28) | (values[vi++] << 5) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi++] << 23) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation24 extends BulkOperation { + + public int blocks() { + return 3; + } + + public int values() { + return 8; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 40; + values[vi++] = (block0 >>> 16) & 16777215L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 65535L) << 8) | (block1 >>> 56); + values[vi++] = (block1 >>> 32) & 16777215L; + values[vi++] = (block1 >>> 8) & 16777215L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 255L) << 16) | (block2 >>> 48); + values[vi++] = (block2 >>> 24) & 16777215L; + values[vi++] = block2 & 16777215L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 40) | (values[vi++] << 16) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 32) | (values[vi++] << 8) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 24) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation25 extends BulkOperation { + + public int blocks() { + return 25; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 39; + values[vi++] = (block0 >>> 14) & 33554431L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 16383L) << 11) | (block1 >>> 53); + values[vi++] = (block1 >>> 28) & 33554431L; + values[vi++] = (block1 >>> 3) & 33554431L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 7L) << 22) | (block2 >>> 42); + values[vi++] = (block2 >>> 17) & 33554431L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 131071L) << 8) | (block3 >>> 56); + values[vi++] = (block3 >>> 31) & 33554431L; + values[vi++] = (block3 >>> 6) & 33554431L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 63L) << 19) | (block4 >>> 45); + values[vi++] = (block4 >>> 20) & 33554431L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 1048575L) << 5) | (block5 >>> 59); + values[vi++] = (block5 >>> 34) & 33554431L; + values[vi++] = (block5 >>> 9) & 33554431L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 511L) << 16) | (block6 >>> 48); + values[vi++] = (block6 >>> 23) & 33554431L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 8388607L) << 2) | (block7 >>> 62); + values[vi++] = (block7 >>> 37) & 33554431L; + values[vi++] = (block7 >>> 12) & 33554431L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 4095L) << 13) | (block8 >>> 51); + values[vi++] = (block8 >>> 26) & 33554431L; + values[vi++] = (block8 >>> 1) & 33554431L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 1L) << 24) | (block9 >>> 40); + values[vi++] = (block9 >>> 15) & 33554431L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 32767L) << 10) | (block10 >>> 54); + values[vi++] = (block10 >>> 29) & 33554431L; + values[vi++] = (block10 >>> 4) & 33554431L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 15L) << 21) | (block11 >>> 43); + values[vi++] = (block11 >>> 18) & 33554431L; + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 262143L) << 7) | (block12 >>> 57); + values[vi++] = (block12 >>> 32) & 33554431L; + values[vi++] = (block12 >>> 7) & 33554431L; + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 127L) << 18) | (block13 >>> 46); + values[vi++] = (block13 >>> 21) & 33554431L; + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 2097151L) << 4) | (block14 >>> 60); + values[vi++] = (block14 >>> 35) & 33554431L; + values[vi++] = (block14 >>> 10) & 33554431L; + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 1023L) << 15) | (block15 >>> 49); + values[vi++] = (block15 >>> 24) & 33554431L; + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 16777215L) << 1) | (block16 >>> 63); + values[vi++] = (block16 >>> 38) & 33554431L; + values[vi++] = (block16 >>> 13) & 33554431L; + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 8191L) << 12) | (block17 >>> 52); + values[vi++] = (block17 >>> 27) & 33554431L; + values[vi++] = (block17 >>> 2) & 33554431L; + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 3L) << 23) | (block18 >>> 41); + values[vi++] = (block18 >>> 16) & 33554431L; + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 65535L) << 9) | (block19 >>> 55); + values[vi++] = (block19 >>> 30) & 33554431L; + values[vi++] = (block19 >>> 5) & 33554431L; + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 31L) << 20) | (block20 >>> 44); + values[vi++] = (block20 >>> 19) & 33554431L; + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 524287L) << 6) | (block21 >>> 58); + values[vi++] = (block21 >>> 33) & 33554431L; + values[vi++] = (block21 >>> 8) & 33554431L; + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 255L) << 17) | (block22 >>> 47); + values[vi++] = (block22 >>> 22) & 33554431L; + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 4194303L) << 3) | (block23 >>> 61); + values[vi++] = (block23 >>> 36) & 33554431L; + values[vi++] = (block23 >>> 11) & 33554431L; + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 2047L) << 14) | (block24 >>> 50); + values[vi++] = (block24 >>> 25) & 33554431L; + values[vi++] = block24 & 33554431L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 39) | (values[vi++] << 14) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi++] << 28) | (values[vi++] << 3) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi++] << 17) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 31) | (values[vi++] << 6) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | (values[vi++] << 20) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 34) | (values[vi++] << 9) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 23) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 37) | (values[vi++] << 12) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi++] << 26) | (values[vi++] << 1) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi++] << 15) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 29) | (values[vi++] << 4) | (values[vi] >>> 21); + blocks[bi++] = (values[vi++] << 43) | (values[vi++] << 18) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 32) | (values[vi++] << 7) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi++] << 21) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 35) | (values[vi++] << 10) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi++] << 24) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 38) | (values[vi++] << 13) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 27) | (values[vi++] << 2) | (values[vi] >>> 23); + blocks[bi++] = (values[vi++] << 41) | (values[vi++] << 16) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 30) | (values[vi++] << 5) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi++] << 19) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 33) | (values[vi++] << 8) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi++] << 22) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 36) | (values[vi++] << 11) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 25) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation26 extends BulkOperation { + + public int blocks() { + return 13; + } + + public int values() { + return 32; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 38; + values[vi++] = (block0 >>> 12) & 67108863L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 4095L) << 14) | (block1 >>> 50); + values[vi++] = (block1 >>> 24) & 67108863L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 16777215L) << 2) | (block2 >>> 62); + values[vi++] = (block2 >>> 36) & 67108863L; + values[vi++] = (block2 >>> 10) & 67108863L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 1023L) << 16) | (block3 >>> 48); + values[vi++] = (block3 >>> 22) & 67108863L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 4194303L) << 4) | (block4 >>> 60); + values[vi++] = (block4 >>> 34) & 67108863L; + values[vi++] = (block4 >>> 8) & 67108863L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 255L) << 18) | (block5 >>> 46); + values[vi++] = (block5 >>> 20) & 67108863L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 1048575L) << 6) | (block6 >>> 58); + values[vi++] = (block6 >>> 32) & 67108863L; + values[vi++] = (block6 >>> 6) & 67108863L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 63L) << 20) | (block7 >>> 44); + values[vi++] = (block7 >>> 18) & 67108863L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 262143L) << 8) | (block8 >>> 56); + values[vi++] = (block8 >>> 30) & 67108863L; + values[vi++] = (block8 >>> 4) & 67108863L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 15L) << 22) | (block9 >>> 42); + values[vi++] = (block9 >>> 16) & 67108863L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 65535L) << 10) | (block10 >>> 54); + values[vi++] = (block10 >>> 28) & 67108863L; + values[vi++] = (block10 >>> 2) & 67108863L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 3L) << 24) | (block11 >>> 40); + values[vi++] = (block11 >>> 14) & 67108863L; + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 16383L) << 12) | (block12 >>> 52); + values[vi++] = (block12 >>> 26) & 67108863L; + values[vi++] = block12 & 67108863L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 38) | (values[vi++] << 12) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 24) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 36) | (values[vi++] << 10) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 22) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 34) | (values[vi++] << 8) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi++] << 20) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 32) | (values[vi++] << 6) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi++] << 18) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 30) | (values[vi++] << 4) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi++] << 16) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 28) | (values[vi++] << 2) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi++] << 14) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 26) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation27 extends BulkOperation { + + public int blocks() { + return 27; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 37; + values[vi++] = (block0 >>> 10) & 134217727L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 1023L) << 17) | (block1 >>> 47); + values[vi++] = (block1 >>> 20) & 134217727L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 1048575L) << 7) | (block2 >>> 57); + values[vi++] = (block2 >>> 30) & 134217727L; + values[vi++] = (block2 >>> 3) & 134217727L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 7L) << 24) | (block3 >>> 40); + values[vi++] = (block3 >>> 13) & 134217727L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 8191L) << 14) | (block4 >>> 50); + values[vi++] = (block4 >>> 23) & 134217727L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 8388607L) << 4) | (block5 >>> 60); + values[vi++] = (block5 >>> 33) & 134217727L; + values[vi++] = (block5 >>> 6) & 134217727L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 63L) << 21) | (block6 >>> 43); + values[vi++] = (block6 >>> 16) & 134217727L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 65535L) << 11) | (block7 >>> 53); + values[vi++] = (block7 >>> 26) & 134217727L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 67108863L) << 1) | (block8 >>> 63); + values[vi++] = (block8 >>> 36) & 134217727L; + values[vi++] = (block8 >>> 9) & 134217727L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 511L) << 18) | (block9 >>> 46); + values[vi++] = (block9 >>> 19) & 134217727L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 524287L) << 8) | (block10 >>> 56); + values[vi++] = (block10 >>> 29) & 134217727L; + values[vi++] = (block10 >>> 2) & 134217727L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 3L) << 25) | (block11 >>> 39); + values[vi++] = (block11 >>> 12) & 134217727L; + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 4095L) << 15) | (block12 >>> 49); + values[vi++] = (block12 >>> 22) & 134217727L; + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 4194303L) << 5) | (block13 >>> 59); + values[vi++] = (block13 >>> 32) & 134217727L; + values[vi++] = (block13 >>> 5) & 134217727L; + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 31L) << 22) | (block14 >>> 42); + values[vi++] = (block14 >>> 15) & 134217727L; + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 32767L) << 12) | (block15 >>> 52); + values[vi++] = (block15 >>> 25) & 134217727L; + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 33554431L) << 2) | (block16 >>> 62); + values[vi++] = (block16 >>> 35) & 134217727L; + values[vi++] = (block16 >>> 8) & 134217727L; + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 255L) << 19) | (block17 >>> 45); + values[vi++] = (block17 >>> 18) & 134217727L; + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 262143L) << 9) | (block18 >>> 55); + values[vi++] = (block18 >>> 28) & 134217727L; + values[vi++] = (block18 >>> 1) & 134217727L; + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 1L) << 26) | (block19 >>> 38); + values[vi++] = (block19 >>> 11) & 134217727L; + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 2047L) << 16) | (block20 >>> 48); + values[vi++] = (block20 >>> 21) & 134217727L; + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 2097151L) << 6) | (block21 >>> 58); + values[vi++] = (block21 >>> 31) & 134217727L; + values[vi++] = (block21 >>> 4) & 134217727L; + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 15L) << 23) | (block22 >>> 41); + values[vi++] = (block22 >>> 14) & 134217727L; + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 16383L) << 13) | (block23 >>> 51); + values[vi++] = (block23 >>> 24) & 134217727L; + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 16777215L) << 3) | (block24 >>> 61); + values[vi++] = (block24 >>> 34) & 134217727L; + values[vi++] = (block24 >>> 7) & 134217727L; + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 127L) << 20) | (block25 >>> 44); + values[vi++] = (block25 >>> 17) & 134217727L; + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 131071L) << 10) | (block26 >>> 54); + values[vi++] = (block26 >>> 27) & 134217727L; + values[vi++] = block26 & 134217727L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 37) | (values[vi++] << 10) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi++] << 20) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 30) | (values[vi++] << 3) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi++] << 13) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 23) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 33) | (values[vi++] << 6) | (values[vi] >>> 21); + blocks[bi++] = (values[vi++] << 43) | (values[vi++] << 16) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi++] << 26) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 36) | (values[vi++] << 9) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi++] << 19) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 29) | (values[vi++] << 2) | (values[vi] >>> 25); + blocks[bi++] = (values[vi++] << 39) | (values[vi++] << 12) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi++] << 22) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 32) | (values[vi++] << 5) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi++] << 15) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 25) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 35) | (values[vi++] << 8) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | (values[vi++] << 18) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 28) | (values[vi++] << 1) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi++] << 11) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 21) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 31) | (values[vi++] << 4) | (values[vi] >>> 23); + blocks[bi++] = (values[vi++] << 41) | (values[vi++] << 14) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi++] << 24) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 34) | (values[vi++] << 7) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi++] << 17) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 27) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation28 extends BulkOperation { + + public int blocks() { + return 7; + } + + public int values() { + return 16; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 36; + values[vi++] = (block0 >>> 8) & 268435455L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 255L) << 20) | (block1 >>> 44); + values[vi++] = (block1 >>> 16) & 268435455L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 65535L) << 12) | (block2 >>> 52); + values[vi++] = (block2 >>> 24) & 268435455L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 16777215L) << 4) | (block3 >>> 60); + values[vi++] = (block3 >>> 32) & 268435455L; + values[vi++] = (block3 >>> 4) & 268435455L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 15L) << 24) | (block4 >>> 40); + values[vi++] = (block4 >>> 12) & 268435455L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 4095L) << 16) | (block5 >>> 48); + values[vi++] = (block5 >>> 20) & 268435455L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 1048575L) << 8) | (block6 >>> 56); + values[vi++] = (block6 >>> 28) & 268435455L; + values[vi++] = block6 & 268435455L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 36) | (values[vi++] << 8) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi++] << 16) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 24) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 32) | (values[vi++] << 4) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi++] << 12) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 20) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 28) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation29 extends BulkOperation { + + public int blocks() { + return 29; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 35; + values[vi++] = (block0 >>> 6) & 536870911L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 63L) << 23) | (block1 >>> 41); + values[vi++] = (block1 >>> 12) & 536870911L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 4095L) << 17) | (block2 >>> 47); + values[vi++] = (block2 >>> 18) & 536870911L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 262143L) << 11) | (block3 >>> 53); + values[vi++] = (block3 >>> 24) & 536870911L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 16777215L) << 5) | (block4 >>> 59); + values[vi++] = (block4 >>> 30) & 536870911L; + values[vi++] = (block4 >>> 1) & 536870911L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 1L) << 28) | (block5 >>> 36); + values[vi++] = (block5 >>> 7) & 536870911L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 127L) << 22) | (block6 >>> 42); + values[vi++] = (block6 >>> 13) & 536870911L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 8191L) << 16) | (block7 >>> 48); + values[vi++] = (block7 >>> 19) & 536870911L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 524287L) << 10) | (block8 >>> 54); + values[vi++] = (block8 >>> 25) & 536870911L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 33554431L) << 4) | (block9 >>> 60); + values[vi++] = (block9 >>> 31) & 536870911L; + values[vi++] = (block9 >>> 2) & 536870911L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 3L) << 27) | (block10 >>> 37); + values[vi++] = (block10 >>> 8) & 536870911L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 255L) << 21) | (block11 >>> 43); + values[vi++] = (block11 >>> 14) & 536870911L; + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 16383L) << 15) | (block12 >>> 49); + values[vi++] = (block12 >>> 20) & 536870911L; + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 1048575L) << 9) | (block13 >>> 55); + values[vi++] = (block13 >>> 26) & 536870911L; + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 67108863L) << 3) | (block14 >>> 61); + values[vi++] = (block14 >>> 32) & 536870911L; + values[vi++] = (block14 >>> 3) & 536870911L; + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 7L) << 26) | (block15 >>> 38); + values[vi++] = (block15 >>> 9) & 536870911L; + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 511L) << 20) | (block16 >>> 44); + values[vi++] = (block16 >>> 15) & 536870911L; + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 32767L) << 14) | (block17 >>> 50); + values[vi++] = (block17 >>> 21) & 536870911L; + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 2097151L) << 8) | (block18 >>> 56); + values[vi++] = (block18 >>> 27) & 536870911L; + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 134217727L) << 2) | (block19 >>> 62); + values[vi++] = (block19 >>> 33) & 536870911L; + values[vi++] = (block19 >>> 4) & 536870911L; + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 15L) << 25) | (block20 >>> 39); + values[vi++] = (block20 >>> 10) & 536870911L; + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 1023L) << 19) | (block21 >>> 45); + values[vi++] = (block21 >>> 16) & 536870911L; + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 65535L) << 13) | (block22 >>> 51); + values[vi++] = (block22 >>> 22) & 536870911L; + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 4194303L) << 7) | (block23 >>> 57); + values[vi++] = (block23 >>> 28) & 536870911L; + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 268435455L) << 1) | (block24 >>> 63); + values[vi++] = (block24 >>> 34) & 536870911L; + values[vi++] = (block24 >>> 5) & 536870911L; + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 31L) << 24) | (block25 >>> 40); + values[vi++] = (block25 >>> 11) & 536870911L; + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 2047L) << 18) | (block26 >>> 46); + values[vi++] = (block26 >>> 17) & 536870911L; + final long block27 = blocks[bi++]; + values[vi++] = ((block26 & 131071L) << 12) | (block27 >>> 52); + values[vi++] = (block27 >>> 23) & 536870911L; + final long block28 = blocks[bi++]; + values[vi++] = ((block27 & 8388607L) << 6) | (block28 >>> 58); + values[vi++] = (block28 >>> 29) & 536870911L; + values[vi++] = block28 & 536870911L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 35) | (values[vi++] << 6) | (values[vi] >>> 23); + blocks[bi++] = (values[vi++] << 41) | (values[vi++] << 12) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi++] << 18) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi++] << 24) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 30) | (values[vi++] << 1) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi++] << 7) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi++] << 13) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 19) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 25) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 31) | (values[vi++] << 2) | (values[vi] >>> 27); + blocks[bi++] = (values[vi++] << 37) | (values[vi++] << 8) | (values[vi] >>> 21); + blocks[bi++] = (values[vi++] << 43) | (values[vi++] << 14) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi++] << 20) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 26) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 32) | (values[vi++] << 3) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi++] << 9) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi++] << 15) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 21) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 27) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 33) | (values[vi++] << 4) | (values[vi] >>> 25); + blocks[bi++] = (values[vi++] << 39) | (values[vi++] << 10) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | (values[vi++] << 16) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi++] << 22) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 28) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 34) | (values[vi++] << 5) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi++] << 11) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi++] << 17) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 23) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 29) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation30 extends BulkOperation { + + public int blocks() { + return 15; + } + + public int values() { + return 32; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 34; + values[vi++] = (block0 >>> 4) & 1073741823L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 15L) << 26) | (block1 >>> 38); + values[vi++] = (block1 >>> 8) & 1073741823L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 255L) << 22) | (block2 >>> 42); + values[vi++] = (block2 >>> 12) & 1073741823L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 4095L) << 18) | (block3 >>> 46); + values[vi++] = (block3 >>> 16) & 1073741823L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 65535L) << 14) | (block4 >>> 50); + values[vi++] = (block4 >>> 20) & 1073741823L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 1048575L) << 10) | (block5 >>> 54); + values[vi++] = (block5 >>> 24) & 1073741823L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 16777215L) << 6) | (block6 >>> 58); + values[vi++] = (block6 >>> 28) & 1073741823L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 268435455L) << 2) | (block7 >>> 62); + values[vi++] = (block7 >>> 32) & 1073741823L; + values[vi++] = (block7 >>> 2) & 1073741823L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 3L) << 28) | (block8 >>> 36); + values[vi++] = (block8 >>> 6) & 1073741823L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 63L) << 24) | (block9 >>> 40); + values[vi++] = (block9 >>> 10) & 1073741823L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 1023L) << 20) | (block10 >>> 44); + values[vi++] = (block10 >>> 14) & 1073741823L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 16383L) << 16) | (block11 >>> 48); + values[vi++] = (block11 >>> 18) & 1073741823L; + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 262143L) << 12) | (block12 >>> 52); + values[vi++] = (block12 >>> 22) & 1073741823L; + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 4194303L) << 8) | (block13 >>> 56); + values[vi++] = (block13 >>> 26) & 1073741823L; + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 67108863L) << 4) | (block14 >>> 60); + values[vi++] = (block14 >>> 30) & 1073741823L; + values[vi++] = block14 & 1073741823L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 34) | (values[vi++] << 4) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi++] << 8) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi++] << 12) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi++] << 16) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 20) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 24) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 28) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 32) | (values[vi++] << 2) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi++] << 6) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi++] << 10) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi++] << 14) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 18) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 22) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 26) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 30) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation31 extends BulkOperation { + + public int blocks() { + return 31; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 33; + values[vi++] = (block0 >>> 2) & 2147483647L; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 3L) << 29) | (block1 >>> 35); + values[vi++] = (block1 >>> 4) & 2147483647L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 15L) << 27) | (block2 >>> 37); + values[vi++] = (block2 >>> 6) & 2147483647L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 63L) << 25) | (block3 >>> 39); + values[vi++] = (block3 >>> 8) & 2147483647L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 255L) << 23) | (block4 >>> 41); + values[vi++] = (block4 >>> 10) & 2147483647L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 1023L) << 21) | (block5 >>> 43); + values[vi++] = (block5 >>> 12) & 2147483647L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 4095L) << 19) | (block6 >>> 45); + values[vi++] = (block6 >>> 14) & 2147483647L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 16383L) << 17) | (block7 >>> 47); + values[vi++] = (block7 >>> 16) & 2147483647L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 65535L) << 15) | (block8 >>> 49); + values[vi++] = (block8 >>> 18) & 2147483647L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 262143L) << 13) | (block9 >>> 51); + values[vi++] = (block9 >>> 20) & 2147483647L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 1048575L) << 11) | (block10 >>> 53); + values[vi++] = (block10 >>> 22) & 2147483647L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 4194303L) << 9) | (block11 >>> 55); + values[vi++] = (block11 >>> 24) & 2147483647L; + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 16777215L) << 7) | (block12 >>> 57); + values[vi++] = (block12 >>> 26) & 2147483647L; + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 67108863L) << 5) | (block13 >>> 59); + values[vi++] = (block13 >>> 28) & 2147483647L; + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 268435455L) << 3) | (block14 >>> 61); + values[vi++] = (block14 >>> 30) & 2147483647L; + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 1073741823L) << 1) | (block15 >>> 63); + values[vi++] = (block15 >>> 32) & 2147483647L; + values[vi++] = (block15 >>> 1) & 2147483647L; + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 1L) << 30) | (block16 >>> 34); + values[vi++] = (block16 >>> 3) & 2147483647L; + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 7L) << 28) | (block17 >>> 36); + values[vi++] = (block17 >>> 5) & 2147483647L; + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 31L) << 26) | (block18 >>> 38); + values[vi++] = (block18 >>> 7) & 2147483647L; + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 127L) << 24) | (block19 >>> 40); + values[vi++] = (block19 >>> 9) & 2147483647L; + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 511L) << 22) | (block20 >>> 42); + values[vi++] = (block20 >>> 11) & 2147483647L; + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 2047L) << 20) | (block21 >>> 44); + values[vi++] = (block21 >>> 13) & 2147483647L; + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 8191L) << 18) | (block22 >>> 46); + values[vi++] = (block22 >>> 15) & 2147483647L; + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 32767L) << 16) | (block23 >>> 48); + values[vi++] = (block23 >>> 17) & 2147483647L; + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 131071L) << 14) | (block24 >>> 50); + values[vi++] = (block24 >>> 19) & 2147483647L; + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 524287L) << 12) | (block25 >>> 52); + values[vi++] = (block25 >>> 21) & 2147483647L; + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 2097151L) << 10) | (block26 >>> 54); + values[vi++] = (block26 >>> 23) & 2147483647L; + final long block27 = blocks[bi++]; + values[vi++] = ((block26 & 8388607L) << 8) | (block27 >>> 56); + values[vi++] = (block27 >>> 25) & 2147483647L; + final long block28 = blocks[bi++]; + values[vi++] = ((block27 & 33554431L) << 6) | (block28 >>> 58); + values[vi++] = (block28 >>> 27) & 2147483647L; + final long block29 = blocks[bi++]; + values[vi++] = ((block28 & 134217727L) << 4) | (block29 >>> 60); + values[vi++] = (block29 >>> 29) & 2147483647L; + final long block30 = blocks[bi++]; + values[vi++] = ((block29 & 536870911L) << 2) | (block30 >>> 62); + values[vi++] = (block30 >>> 31) & 2147483647L; + values[vi++] = block30 & 2147483647L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 33) | (values[vi++] << 2) | (values[vi] >>> 29); + blocks[bi++] = (values[vi++] << 35) | (values[vi++] << 4) | (values[vi] >>> 27); + blocks[bi++] = (values[vi++] << 37) | (values[vi++] << 6) | (values[vi] >>> 25); + blocks[bi++] = (values[vi++] << 39) | (values[vi++] << 8) | (values[vi] >>> 23); + blocks[bi++] = (values[vi++] << 41) | (values[vi++] << 10) | (values[vi] >>> 21); + blocks[bi++] = (values[vi++] << 43) | (values[vi++] << 12) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | (values[vi++] << 14) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi++] << 16) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi++] << 18) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi++] << 20) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi++] << 22) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 24) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 26) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 28) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 30) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 32) | (values[vi++] << 1) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi++] << 3) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi++] << 5) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi++] << 7) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi++] << 9) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi++] << 11) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi++] << 13) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi++] << 15) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 17) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 19) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 21) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 23) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 25) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 27) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 29) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 31) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation32 extends BulkOperation { + + public int blocks() { + return 1; + } + + public int values() { + return 2; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 32; + values[vi++] = block0 & 4294967295L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 32) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation33 extends BulkOperation { + + public int blocks() { + return 33; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 31; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 2147483647L) << 2) | (block1 >>> 62); + values[vi++] = (block1 >>> 29) & 8589934591L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 536870911L) << 4) | (block2 >>> 60); + values[vi++] = (block2 >>> 27) & 8589934591L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 134217727L) << 6) | (block3 >>> 58); + values[vi++] = (block3 >>> 25) & 8589934591L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 33554431L) << 8) | (block4 >>> 56); + values[vi++] = (block4 >>> 23) & 8589934591L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 8388607L) << 10) | (block5 >>> 54); + values[vi++] = (block5 >>> 21) & 8589934591L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 2097151L) << 12) | (block6 >>> 52); + values[vi++] = (block6 >>> 19) & 8589934591L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 524287L) << 14) | (block7 >>> 50); + values[vi++] = (block7 >>> 17) & 8589934591L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 131071L) << 16) | (block8 >>> 48); + values[vi++] = (block8 >>> 15) & 8589934591L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 32767L) << 18) | (block9 >>> 46); + values[vi++] = (block9 >>> 13) & 8589934591L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 8191L) << 20) | (block10 >>> 44); + values[vi++] = (block10 >>> 11) & 8589934591L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 2047L) << 22) | (block11 >>> 42); + values[vi++] = (block11 >>> 9) & 8589934591L; + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 511L) << 24) | (block12 >>> 40); + values[vi++] = (block12 >>> 7) & 8589934591L; + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 127L) << 26) | (block13 >>> 38); + values[vi++] = (block13 >>> 5) & 8589934591L; + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 31L) << 28) | (block14 >>> 36); + values[vi++] = (block14 >>> 3) & 8589934591L; + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 7L) << 30) | (block15 >>> 34); + values[vi++] = (block15 >>> 1) & 8589934591L; + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 1L) << 32) | (block16 >>> 32); + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 4294967295L) << 1) | (block17 >>> 63); + values[vi++] = (block17 >>> 30) & 8589934591L; + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 1073741823L) << 3) | (block18 >>> 61); + values[vi++] = (block18 >>> 28) & 8589934591L; + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 268435455L) << 5) | (block19 >>> 59); + values[vi++] = (block19 >>> 26) & 8589934591L; + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 67108863L) << 7) | (block20 >>> 57); + values[vi++] = (block20 >>> 24) & 8589934591L; + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 16777215L) << 9) | (block21 >>> 55); + values[vi++] = (block21 >>> 22) & 8589934591L; + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 4194303L) << 11) | (block22 >>> 53); + values[vi++] = (block22 >>> 20) & 8589934591L; + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 1048575L) << 13) | (block23 >>> 51); + values[vi++] = (block23 >>> 18) & 8589934591L; + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 262143L) << 15) | (block24 >>> 49); + values[vi++] = (block24 >>> 16) & 8589934591L; + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 65535L) << 17) | (block25 >>> 47); + values[vi++] = (block25 >>> 14) & 8589934591L; + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 16383L) << 19) | (block26 >>> 45); + values[vi++] = (block26 >>> 12) & 8589934591L; + final long block27 = blocks[bi++]; + values[vi++] = ((block26 & 4095L) << 21) | (block27 >>> 43); + values[vi++] = (block27 >>> 10) & 8589934591L; + final long block28 = blocks[bi++]; + values[vi++] = ((block27 & 1023L) << 23) | (block28 >>> 41); + values[vi++] = (block28 >>> 8) & 8589934591L; + final long block29 = blocks[bi++]; + values[vi++] = ((block28 & 255L) << 25) | (block29 >>> 39); + values[vi++] = (block29 >>> 6) & 8589934591L; + final long block30 = blocks[bi++]; + values[vi++] = ((block29 & 63L) << 27) | (block30 >>> 37); + values[vi++] = (block30 >>> 4) & 8589934591L; + final long block31 = blocks[bi++]; + values[vi++] = ((block30 & 15L) << 29) | (block31 >>> 35); + values[vi++] = (block31 >>> 2) & 8589934591L; + final long block32 = blocks[bi++]; + values[vi++] = ((block31 & 3L) << 31) | (block32 >>> 33); + values[vi++] = block32 & 8589934591L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 31) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 29) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 27) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 25) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 23) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 21) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 19) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 17) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 15) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi++] << 13) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi++] << 11) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi++] << 9) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi++] << 7) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi++] << 5) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi++] << 3) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi++] << 1) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 30) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 28) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 26) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 24) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 22) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi++] << 20) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi++] << 18) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi++] << 16) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi++] << 14) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | (values[vi++] << 12) | (values[vi] >>> 21); + blocks[bi++] = (values[vi++] << 43) | (values[vi++] << 10) | (values[vi] >>> 23); + blocks[bi++] = (values[vi++] << 41) | (values[vi++] << 8) | (values[vi] >>> 25); + blocks[bi++] = (values[vi++] << 39) | (values[vi++] << 6) | (values[vi] >>> 27); + blocks[bi++] = (values[vi++] << 37) | (values[vi++] << 4) | (values[vi] >>> 29); + blocks[bi++] = (values[vi++] << 35) | (values[vi++] << 2) | (values[vi] >>> 31); + blocks[bi++] = (values[vi++] << 33) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation34 extends BulkOperation { + + public int blocks() { + return 17; + } + + public int values() { + return 32; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 30; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 1073741823L) << 4) | (block1 >>> 60); + values[vi++] = (block1 >>> 26) & 17179869183L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 67108863L) << 8) | (block2 >>> 56); + values[vi++] = (block2 >>> 22) & 17179869183L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 4194303L) << 12) | (block3 >>> 52); + values[vi++] = (block3 >>> 18) & 17179869183L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 262143L) << 16) | (block4 >>> 48); + values[vi++] = (block4 >>> 14) & 17179869183L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 16383L) << 20) | (block5 >>> 44); + values[vi++] = (block5 >>> 10) & 17179869183L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 1023L) << 24) | (block6 >>> 40); + values[vi++] = (block6 >>> 6) & 17179869183L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 63L) << 28) | (block7 >>> 36); + values[vi++] = (block7 >>> 2) & 17179869183L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 3L) << 32) | (block8 >>> 32); + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 4294967295L) << 2) | (block9 >>> 62); + values[vi++] = (block9 >>> 28) & 17179869183L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 268435455L) << 6) | (block10 >>> 58); + values[vi++] = (block10 >>> 24) & 17179869183L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 16777215L) << 10) | (block11 >>> 54); + values[vi++] = (block11 >>> 20) & 17179869183L; + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 1048575L) << 14) | (block12 >>> 50); + values[vi++] = (block12 >>> 16) & 17179869183L; + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 65535L) << 18) | (block13 >>> 46); + values[vi++] = (block13 >>> 12) & 17179869183L; + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 4095L) << 22) | (block14 >>> 42); + values[vi++] = (block14 >>> 8) & 17179869183L; + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 255L) << 26) | (block15 >>> 38); + values[vi++] = (block15 >>> 4) & 17179869183L; + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 15L) << 30) | (block16 >>> 34); + values[vi++] = block16 & 17179869183L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 26) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 22) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 18) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 14) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi++] << 10) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi++] << 6) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi++] << 2) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 28) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 24) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 20) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 16) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi++] << 12) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi++] << 8) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi++] << 4) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation35 extends BulkOperation { + + public int blocks() { + return 35; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 29; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 536870911L) << 6) | (block1 >>> 58); + values[vi++] = (block1 >>> 23) & 34359738367L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 8388607L) << 12) | (block2 >>> 52); + values[vi++] = (block2 >>> 17) & 34359738367L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 131071L) << 18) | (block3 >>> 46); + values[vi++] = (block3 >>> 11) & 34359738367L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 2047L) << 24) | (block4 >>> 40); + values[vi++] = (block4 >>> 5) & 34359738367L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 31L) << 30) | (block5 >>> 34); + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 17179869183L) << 1) | (block6 >>> 63); + values[vi++] = (block6 >>> 28) & 34359738367L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 268435455L) << 7) | (block7 >>> 57); + values[vi++] = (block7 >>> 22) & 34359738367L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 4194303L) << 13) | (block8 >>> 51); + values[vi++] = (block8 >>> 16) & 34359738367L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 65535L) << 19) | (block9 >>> 45); + values[vi++] = (block9 >>> 10) & 34359738367L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 1023L) << 25) | (block10 >>> 39); + values[vi++] = (block10 >>> 4) & 34359738367L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 15L) << 31) | (block11 >>> 33); + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 8589934591L) << 2) | (block12 >>> 62); + values[vi++] = (block12 >>> 27) & 34359738367L; + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 134217727L) << 8) | (block13 >>> 56); + values[vi++] = (block13 >>> 21) & 34359738367L; + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 2097151L) << 14) | (block14 >>> 50); + values[vi++] = (block14 >>> 15) & 34359738367L; + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 32767L) << 20) | (block15 >>> 44); + values[vi++] = (block15 >>> 9) & 34359738367L; + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 511L) << 26) | (block16 >>> 38); + values[vi++] = (block16 >>> 3) & 34359738367L; + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 7L) << 32) | (block17 >>> 32); + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 4294967295L) << 3) | (block18 >>> 61); + values[vi++] = (block18 >>> 26) & 34359738367L; + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 67108863L) << 9) | (block19 >>> 55); + values[vi++] = (block19 >>> 20) & 34359738367L; + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 1048575L) << 15) | (block20 >>> 49); + values[vi++] = (block20 >>> 14) & 34359738367L; + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 16383L) << 21) | (block21 >>> 43); + values[vi++] = (block21 >>> 8) & 34359738367L; + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 255L) << 27) | (block22 >>> 37); + values[vi++] = (block22 >>> 2) & 34359738367L; + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 3L) << 33) | (block23 >>> 31); + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 2147483647L) << 4) | (block24 >>> 60); + values[vi++] = (block24 >>> 25) & 34359738367L; + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 33554431L) << 10) | (block25 >>> 54); + values[vi++] = (block25 >>> 19) & 34359738367L; + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 524287L) << 16) | (block26 >>> 48); + values[vi++] = (block26 >>> 13) & 34359738367L; + final long block27 = blocks[bi++]; + values[vi++] = ((block26 & 8191L) << 22) | (block27 >>> 42); + values[vi++] = (block27 >>> 7) & 34359738367L; + final long block28 = blocks[bi++]; + values[vi++] = ((block27 & 127L) << 28) | (block28 >>> 36); + values[vi++] = (block28 >>> 1) & 34359738367L; + final long block29 = blocks[bi++]; + values[vi++] = ((block28 & 1L) << 34) | (block29 >>> 30); + final long block30 = blocks[bi++]; + values[vi++] = ((block29 & 1073741823L) << 5) | (block30 >>> 59); + values[vi++] = (block30 >>> 24) & 34359738367L; + final long block31 = blocks[bi++]; + values[vi++] = ((block30 & 16777215L) << 11) | (block31 >>> 53); + values[vi++] = (block31 >>> 18) & 34359738367L; + final long block32 = blocks[bi++]; + values[vi++] = ((block31 & 262143L) << 17) | (block32 >>> 47); + values[vi++] = (block32 >>> 12) & 34359738367L; + final long block33 = blocks[bi++]; + values[vi++] = ((block32 & 4095L) << 23) | (block33 >>> 41); + values[vi++] = (block33 >>> 6) & 34359738367L; + final long block34 = blocks[bi++]; + values[vi++] = ((block33 & 63L) << 29) | (block34 >>> 35); + values[vi++] = block34 & 34359738367L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 29) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 23) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 17) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi++] << 11) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi++] << 5) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 28) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 22) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi++] << 16) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | (values[vi++] << 10) | (values[vi] >>> 25); + blocks[bi++] = (values[vi++] << 39) | (values[vi++] << 4) | (values[vi] >>> 31); + blocks[bi++] = (values[vi++] << 33) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 27) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 21) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 15) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi++] << 9) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi++] << 3) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 26) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 20) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi++] << 14) | (values[vi] >>> 21); + blocks[bi++] = (values[vi++] << 43) | (values[vi++] << 8) | (values[vi] >>> 27); + blocks[bi++] = (values[vi++] << 37) | (values[vi++] << 2) | (values[vi] >>> 33); + blocks[bi++] = (values[vi++] << 31) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 25) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 19) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 13) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi++] << 7) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi++] << 1) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 24) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi++] << 18) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi++] << 12) | (values[vi] >>> 23); + blocks[bi++] = (values[vi++] << 41) | (values[vi++] << 6) | (values[vi] >>> 29); + blocks[bi++] = (values[vi++] << 35) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation36 extends BulkOperation { + + public int blocks() { + return 9; + } + + public int values() { + return 16; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 28; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 268435455L) << 8) | (block1 >>> 56); + values[vi++] = (block1 >>> 20) & 68719476735L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 1048575L) << 16) | (block2 >>> 48); + values[vi++] = (block2 >>> 12) & 68719476735L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 4095L) << 24) | (block3 >>> 40); + values[vi++] = (block3 >>> 4) & 68719476735L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 15L) << 32) | (block4 >>> 32); + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 4294967295L) << 4) | (block5 >>> 60); + values[vi++] = (block5 >>> 24) & 68719476735L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 16777215L) << 12) | (block6 >>> 52); + values[vi++] = (block6 >>> 16) & 68719476735L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 65535L) << 20) | (block7 >>> 44); + values[vi++] = (block7 >>> 8) & 68719476735L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 255L) << 28) | (block8 >>> 36); + values[vi++] = block8 & 68719476735L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 20) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 12) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi++] << 4) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 24) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 16) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi++] << 8) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation37 extends BulkOperation { + + public int blocks() { + return 37; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 27; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 134217727L) << 10) | (block1 >>> 54); + values[vi++] = (block1 >>> 17) & 137438953471L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 131071L) << 20) | (block2 >>> 44); + values[vi++] = (block2 >>> 7) & 137438953471L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 127L) << 30) | (block3 >>> 34); + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 17179869183L) << 3) | (block4 >>> 61); + values[vi++] = (block4 >>> 24) & 137438953471L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 16777215L) << 13) | (block5 >>> 51); + values[vi++] = (block5 >>> 14) & 137438953471L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 16383L) << 23) | (block6 >>> 41); + values[vi++] = (block6 >>> 4) & 137438953471L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 15L) << 33) | (block7 >>> 31); + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 2147483647L) << 6) | (block8 >>> 58); + values[vi++] = (block8 >>> 21) & 137438953471L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 2097151L) << 16) | (block9 >>> 48); + values[vi++] = (block9 >>> 11) & 137438953471L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 2047L) << 26) | (block10 >>> 38); + values[vi++] = (block10 >>> 1) & 137438953471L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 1L) << 36) | (block11 >>> 28); + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 268435455L) << 9) | (block12 >>> 55); + values[vi++] = (block12 >>> 18) & 137438953471L; + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 262143L) << 19) | (block13 >>> 45); + values[vi++] = (block13 >>> 8) & 137438953471L; + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 255L) << 29) | (block14 >>> 35); + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 34359738367L) << 2) | (block15 >>> 62); + values[vi++] = (block15 >>> 25) & 137438953471L; + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 33554431L) << 12) | (block16 >>> 52); + values[vi++] = (block16 >>> 15) & 137438953471L; + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 32767L) << 22) | (block17 >>> 42); + values[vi++] = (block17 >>> 5) & 137438953471L; + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 31L) << 32) | (block18 >>> 32); + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 4294967295L) << 5) | (block19 >>> 59); + values[vi++] = (block19 >>> 22) & 137438953471L; + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 4194303L) << 15) | (block20 >>> 49); + values[vi++] = (block20 >>> 12) & 137438953471L; + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 4095L) << 25) | (block21 >>> 39); + values[vi++] = (block21 >>> 2) & 137438953471L; + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 3L) << 35) | (block22 >>> 29); + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 536870911L) << 8) | (block23 >>> 56); + values[vi++] = (block23 >>> 19) & 137438953471L; + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 524287L) << 18) | (block24 >>> 46); + values[vi++] = (block24 >>> 9) & 137438953471L; + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 511L) << 28) | (block25 >>> 36); + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 68719476735L) << 1) | (block26 >>> 63); + values[vi++] = (block26 >>> 26) & 137438953471L; + final long block27 = blocks[bi++]; + values[vi++] = ((block26 & 67108863L) << 11) | (block27 >>> 53); + values[vi++] = (block27 >>> 16) & 137438953471L; + final long block28 = blocks[bi++]; + values[vi++] = ((block27 & 65535L) << 21) | (block28 >>> 43); + values[vi++] = (block28 >>> 6) & 137438953471L; + final long block29 = blocks[bi++]; + values[vi++] = ((block28 & 63L) << 31) | (block29 >>> 33); + final long block30 = blocks[bi++]; + values[vi++] = ((block29 & 8589934591L) << 4) | (block30 >>> 60); + values[vi++] = (block30 >>> 23) & 137438953471L; + final long block31 = blocks[bi++]; + values[vi++] = ((block30 & 8388607L) << 14) | (block31 >>> 50); + values[vi++] = (block31 >>> 13) & 137438953471L; + final long block32 = blocks[bi++]; + values[vi++] = ((block31 & 8191L) << 24) | (block32 >>> 40); + values[vi++] = (block32 >>> 3) & 137438953471L; + final long block33 = blocks[bi++]; + values[vi++] = ((block32 & 7L) << 34) | (block33 >>> 30); + final long block34 = blocks[bi++]; + values[vi++] = ((block33 & 1073741823L) << 7) | (block34 >>> 57); + values[vi++] = (block34 >>> 20) & 137438953471L; + final long block35 = blocks[bi++]; + values[vi++] = ((block34 & 1048575L) << 17) | (block35 >>> 47); + values[vi++] = (block35 >>> 10) & 137438953471L; + final long block36 = blocks[bi++]; + values[vi++] = ((block35 & 1023L) << 27) | (block36 >>> 37); + values[vi++] = block36 & 137438953471L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 27) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 17) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi++] << 7) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 24) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi++] << 14) | (values[vi] >>> 23); + blocks[bi++] = (values[vi++] << 41) | (values[vi++] << 4) | (values[vi] >>> 33); + blocks[bi++] = (values[vi++] << 31) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 21) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 11) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi++] << 1) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 18) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | (values[vi++] << 8) | (values[vi] >>> 29); + blocks[bi++] = (values[vi++] << 35) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 25) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 15) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi++] << 5) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 22) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi++] << 12) | (values[vi] >>> 25); + blocks[bi++] = (values[vi++] << 39) | (values[vi++] << 2) | (values[vi] >>> 35); + blocks[bi++] = (values[vi++] << 29) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 19) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi++] << 9) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 26) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi++] << 16) | (values[vi] >>> 21); + blocks[bi++] = (values[vi++] << 43) | (values[vi++] << 6) | (values[vi] >>> 31); + blocks[bi++] = (values[vi++] << 33) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 23) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 13) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi++] << 3) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 20) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi++] << 10) | (values[vi] >>> 27); + blocks[bi++] = (values[vi++] << 37) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation38 extends BulkOperation { + + public int blocks() { + return 19; + } + + public int values() { + return 32; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 26; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 67108863L) << 12) | (block1 >>> 52); + values[vi++] = (block1 >>> 14) & 274877906943L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 16383L) << 24) | (block2 >>> 40); + values[vi++] = (block2 >>> 2) & 274877906943L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 3L) << 36) | (block3 >>> 28); + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 268435455L) << 10) | (block4 >>> 54); + values[vi++] = (block4 >>> 16) & 274877906943L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 65535L) << 22) | (block5 >>> 42); + values[vi++] = (block5 >>> 4) & 274877906943L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 15L) << 34) | (block6 >>> 30); + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 1073741823L) << 8) | (block7 >>> 56); + values[vi++] = (block7 >>> 18) & 274877906943L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 262143L) << 20) | (block8 >>> 44); + values[vi++] = (block8 >>> 6) & 274877906943L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 63L) << 32) | (block9 >>> 32); + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 4294967295L) << 6) | (block10 >>> 58); + values[vi++] = (block10 >>> 20) & 274877906943L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 1048575L) << 18) | (block11 >>> 46); + values[vi++] = (block11 >>> 8) & 274877906943L; + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 255L) << 30) | (block12 >>> 34); + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 17179869183L) << 4) | (block13 >>> 60); + values[vi++] = (block13 >>> 22) & 274877906943L; + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 4194303L) << 16) | (block14 >>> 48); + values[vi++] = (block14 >>> 10) & 274877906943L; + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 1023L) << 28) | (block15 >>> 36); + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 68719476735L) << 2) | (block16 >>> 62); + values[vi++] = (block16 >>> 24) & 274877906943L; + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 16777215L) << 14) | (block17 >>> 50); + values[vi++] = (block17 >>> 12) & 274877906943L; + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 4095L) << 26) | (block18 >>> 38); + values[vi++] = block18 & 274877906943L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 26) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 14) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi++] << 2) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 16) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi++] << 4) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 18) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi++] << 6) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 20) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi++] << 8) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 22) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 10) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 24) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 12) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation39 extends BulkOperation { + + public int blocks() { + return 39; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 25; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 33554431L) << 14) | (block1 >>> 50); + values[vi++] = (block1 >>> 11) & 549755813887L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 2047L) << 28) | (block2 >>> 36); + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 68719476735L) << 3) | (block3 >>> 61); + values[vi++] = (block3 >>> 22) & 549755813887L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 4194303L) << 17) | (block4 >>> 47); + values[vi++] = (block4 >>> 8) & 549755813887L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 255L) << 31) | (block5 >>> 33); + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 8589934591L) << 6) | (block6 >>> 58); + values[vi++] = (block6 >>> 19) & 549755813887L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 524287L) << 20) | (block7 >>> 44); + values[vi++] = (block7 >>> 5) & 549755813887L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 31L) << 34) | (block8 >>> 30); + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 1073741823L) << 9) | (block9 >>> 55); + values[vi++] = (block9 >>> 16) & 549755813887L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 65535L) << 23) | (block10 >>> 41); + values[vi++] = (block10 >>> 2) & 549755813887L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 3L) << 37) | (block11 >>> 27); + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 134217727L) << 12) | (block12 >>> 52); + values[vi++] = (block12 >>> 13) & 549755813887L; + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 8191L) << 26) | (block13 >>> 38); + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 274877906943L) << 1) | (block14 >>> 63); + values[vi++] = (block14 >>> 24) & 549755813887L; + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 16777215L) << 15) | (block15 >>> 49); + values[vi++] = (block15 >>> 10) & 549755813887L; + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 1023L) << 29) | (block16 >>> 35); + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 34359738367L) << 4) | (block17 >>> 60); + values[vi++] = (block17 >>> 21) & 549755813887L; + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 2097151L) << 18) | (block18 >>> 46); + values[vi++] = (block18 >>> 7) & 549755813887L; + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 127L) << 32) | (block19 >>> 32); + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 4294967295L) << 7) | (block20 >>> 57); + values[vi++] = (block20 >>> 18) & 549755813887L; + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 262143L) << 21) | (block21 >>> 43); + values[vi++] = (block21 >>> 4) & 549755813887L; + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 15L) << 35) | (block22 >>> 29); + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 536870911L) << 10) | (block23 >>> 54); + values[vi++] = (block23 >>> 15) & 549755813887L; + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 32767L) << 24) | (block24 >>> 40); + values[vi++] = (block24 >>> 1) & 549755813887L; + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 1L) << 38) | (block25 >>> 26); + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 67108863L) << 13) | (block26 >>> 51); + values[vi++] = (block26 >>> 12) & 549755813887L; + final long block27 = blocks[bi++]; + values[vi++] = ((block26 & 4095L) << 27) | (block27 >>> 37); + final long block28 = blocks[bi++]; + values[vi++] = ((block27 & 137438953471L) << 2) | (block28 >>> 62); + values[vi++] = (block28 >>> 23) & 549755813887L; + final long block29 = blocks[bi++]; + values[vi++] = ((block28 & 8388607L) << 16) | (block29 >>> 48); + values[vi++] = (block29 >>> 9) & 549755813887L; + final long block30 = blocks[bi++]; + values[vi++] = ((block29 & 511L) << 30) | (block30 >>> 34); + final long block31 = blocks[bi++]; + values[vi++] = ((block30 & 17179869183L) << 5) | (block31 >>> 59); + values[vi++] = (block31 >>> 20) & 549755813887L; + final long block32 = blocks[bi++]; + values[vi++] = ((block31 & 1048575L) << 19) | (block32 >>> 45); + values[vi++] = (block32 >>> 6) & 549755813887L; + final long block33 = blocks[bi++]; + values[vi++] = ((block32 & 63L) << 33) | (block33 >>> 31); + final long block34 = blocks[bi++]; + values[vi++] = ((block33 & 2147483647L) << 8) | (block34 >>> 56); + values[vi++] = (block34 >>> 17) & 549755813887L; + final long block35 = blocks[bi++]; + values[vi++] = ((block34 & 131071L) << 22) | (block35 >>> 42); + values[vi++] = (block35 >>> 3) & 549755813887L; + final long block36 = blocks[bi++]; + values[vi++] = ((block35 & 7L) << 36) | (block36 >>> 28); + final long block37 = blocks[bi++]; + values[vi++] = ((block36 & 268435455L) << 11) | (block37 >>> 53); + values[vi++] = (block37 >>> 14) & 549755813887L; + final long block38 = blocks[bi++]; + values[vi++] = ((block37 & 16383L) << 25) | (block38 >>> 39); + values[vi++] = block38 & 549755813887L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 25) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 11) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 22) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi++] << 8) | (values[vi] >>> 31); + blocks[bi++] = (values[vi++] << 33) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 19) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi++] << 5) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 16) | (values[vi] >>> 23); + blocks[bi++] = (values[vi++] << 41) | (values[vi++] << 2) | (values[vi] >>> 37); + blocks[bi++] = (values[vi++] << 27) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 13) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 24) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi++] << 10) | (values[vi] >>> 29); + blocks[bi++] = (values[vi++] << 35) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 21) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi++] << 7) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 18) | (values[vi] >>> 21); + blocks[bi++] = (values[vi++] << 43) | (values[vi++] << 4) | (values[vi] >>> 35); + blocks[bi++] = (values[vi++] << 29) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 15) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi++] << 1) | (values[vi] >>> 38); + blocks[bi++] = (values[vi++] << 26) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi++] << 12) | (values[vi] >>> 27); + blocks[bi++] = (values[vi++] << 37) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 23) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 9) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 20) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | (values[vi++] << 6) | (values[vi] >>> 33); + blocks[bi++] = (values[vi++] << 31) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 17) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi++] << 3) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi++] << 14) | (values[vi] >>> 25); + blocks[bi++] = (values[vi++] << 39) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation40 extends BulkOperation { + + public int blocks() { + return 5; + } + + public int values() { + return 8; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 24; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 16777215L) << 16) | (block1 >>> 48); + values[vi++] = (block1 >>> 8) & 1099511627775L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 255L) << 32) | (block2 >>> 32); + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 4294967295L) << 8) | (block3 >>> 56); + values[vi++] = (block3 >>> 16) & 1099511627775L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 65535L) << 24) | (block4 >>> 40); + values[vi++] = block4 & 1099511627775L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 8) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 16) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation41 extends BulkOperation { + + public int blocks() { + return 41; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 23; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 8388607L) << 18) | (block1 >>> 46); + values[vi++] = (block1 >>> 5) & 2199023255551L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 31L) << 36) | (block2 >>> 28); + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 268435455L) << 13) | (block3 >>> 51); + values[vi++] = (block3 >>> 10) & 2199023255551L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 1023L) << 31) | (block4 >>> 33); + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 8589934591L) << 8) | (block5 >>> 56); + values[vi++] = (block5 >>> 15) & 2199023255551L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 32767L) << 26) | (block6 >>> 38); + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 274877906943L) << 3) | (block7 >>> 61); + values[vi++] = (block7 >>> 20) & 2199023255551L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 1048575L) << 21) | (block8 >>> 43); + values[vi++] = (block8 >>> 2) & 2199023255551L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 3L) << 39) | (block9 >>> 25); + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 33554431L) << 16) | (block10 >>> 48); + values[vi++] = (block10 >>> 7) & 2199023255551L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 127L) << 34) | (block11 >>> 30); + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 1073741823L) << 11) | (block12 >>> 53); + values[vi++] = (block12 >>> 12) & 2199023255551L; + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 4095L) << 29) | (block13 >>> 35); + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 34359738367L) << 6) | (block14 >>> 58); + values[vi++] = (block14 >>> 17) & 2199023255551L; + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 131071L) << 24) | (block15 >>> 40); + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 1099511627775L) << 1) | (block16 >>> 63); + values[vi++] = (block16 >>> 22) & 2199023255551L; + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 4194303L) << 19) | (block17 >>> 45); + values[vi++] = (block17 >>> 4) & 2199023255551L; + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 15L) << 37) | (block18 >>> 27); + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 134217727L) << 14) | (block19 >>> 50); + values[vi++] = (block19 >>> 9) & 2199023255551L; + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 511L) << 32) | (block20 >>> 32); + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 4294967295L) << 9) | (block21 >>> 55); + values[vi++] = (block21 >>> 14) & 2199023255551L; + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 16383L) << 27) | (block22 >>> 37); + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 137438953471L) << 4) | (block23 >>> 60); + values[vi++] = (block23 >>> 19) & 2199023255551L; + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 524287L) << 22) | (block24 >>> 42); + values[vi++] = (block24 >>> 1) & 2199023255551L; + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 1L) << 40) | (block25 >>> 24); + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 16777215L) << 17) | (block26 >>> 47); + values[vi++] = (block26 >>> 6) & 2199023255551L; + final long block27 = blocks[bi++]; + values[vi++] = ((block26 & 63L) << 35) | (block27 >>> 29); + final long block28 = blocks[bi++]; + values[vi++] = ((block27 & 536870911L) << 12) | (block28 >>> 52); + values[vi++] = (block28 >>> 11) & 2199023255551L; + final long block29 = blocks[bi++]; + values[vi++] = ((block28 & 2047L) << 30) | (block29 >>> 34); + final long block30 = blocks[bi++]; + values[vi++] = ((block29 & 17179869183L) << 7) | (block30 >>> 57); + values[vi++] = (block30 >>> 16) & 2199023255551L; + final long block31 = blocks[bi++]; + values[vi++] = ((block30 & 65535L) << 25) | (block31 >>> 39); + final long block32 = blocks[bi++]; + values[vi++] = ((block31 & 549755813887L) << 2) | (block32 >>> 62); + values[vi++] = (block32 >>> 21) & 2199023255551L; + final long block33 = blocks[bi++]; + values[vi++] = ((block32 & 2097151L) << 20) | (block33 >>> 44); + values[vi++] = (block33 >>> 3) & 2199023255551L; + final long block34 = blocks[bi++]; + values[vi++] = ((block33 & 7L) << 38) | (block34 >>> 26); + final long block35 = blocks[bi++]; + values[vi++] = ((block34 & 67108863L) << 15) | (block35 >>> 49); + values[vi++] = (block35 >>> 8) & 2199023255551L; + final long block36 = blocks[bi++]; + values[vi++] = ((block35 & 255L) << 33) | (block36 >>> 31); + final long block37 = blocks[bi++]; + values[vi++] = ((block36 & 2147483647L) << 10) | (block37 >>> 54); + values[vi++] = (block37 >>> 13) & 2199023255551L; + final long block38 = blocks[bi++]; + values[vi++] = ((block37 & 8191L) << 28) | (block38 >>> 36); + final long block39 = blocks[bi++]; + values[vi++] = ((block38 & 68719476735L) << 5) | (block39 >>> 59); + values[vi++] = (block39 >>> 18) & 2199023255551L; + final long block40 = blocks[bi++]; + values[vi++] = ((block39 & 262143L) << 23) | (block40 >>> 41); + values[vi++] = block40 & 2199023255551L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 23) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi++] << 5) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi++] << 10) | (values[vi] >>> 31); + blocks[bi++] = (values[vi++] << 33) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 15) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 20) | (values[vi] >>> 21); + blocks[bi++] = (values[vi++] << 43) | (values[vi++] << 2) | (values[vi] >>> 39); + blocks[bi++] = (values[vi++] << 25) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 7) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi++] << 12) | (values[vi] >>> 29); + blocks[bi++] = (values[vi++] << 35) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 17) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 22) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | (values[vi++] << 4) | (values[vi] >>> 37); + blocks[bi++] = (values[vi++] << 27) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 9) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 14) | (values[vi] >>> 27); + blocks[bi++] = (values[vi++] << 37) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 19) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi++] << 1) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi++] << 6) | (values[vi] >>> 35); + blocks[bi++] = (values[vi++] << 29) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 11) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 16) | (values[vi] >>> 25); + blocks[bi++] = (values[vi++] << 39) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 21) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi++] << 3) | (values[vi] >>> 38); + blocks[bi++] = (values[vi++] << 26) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi++] << 8) | (values[vi] >>> 33); + blocks[bi++] = (values[vi++] << 31) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 13) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 18) | (values[vi] >>> 23); + blocks[bi++] = (values[vi++] << 41) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation42 extends BulkOperation { + + public int blocks() { + return 21; + } + + public int values() { + return 32; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 22; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 4194303L) << 20) | (block1 >>> 44); + values[vi++] = (block1 >>> 2) & 4398046511103L; + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 3L) << 40) | (block2 >>> 24); + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 16777215L) << 18) | (block3 >>> 46); + values[vi++] = (block3 >>> 4) & 4398046511103L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 15L) << 38) | (block4 >>> 26); + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 67108863L) << 16) | (block5 >>> 48); + values[vi++] = (block5 >>> 6) & 4398046511103L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 63L) << 36) | (block6 >>> 28); + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 268435455L) << 14) | (block7 >>> 50); + values[vi++] = (block7 >>> 8) & 4398046511103L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 255L) << 34) | (block8 >>> 30); + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 1073741823L) << 12) | (block9 >>> 52); + values[vi++] = (block9 >>> 10) & 4398046511103L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 1023L) << 32) | (block10 >>> 32); + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 4294967295L) << 10) | (block11 >>> 54); + values[vi++] = (block11 >>> 12) & 4398046511103L; + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 4095L) << 30) | (block12 >>> 34); + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 17179869183L) << 8) | (block13 >>> 56); + values[vi++] = (block13 >>> 14) & 4398046511103L; + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 16383L) << 28) | (block14 >>> 36); + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 68719476735L) << 6) | (block15 >>> 58); + values[vi++] = (block15 >>> 16) & 4398046511103L; + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 65535L) << 26) | (block16 >>> 38); + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 274877906943L) << 4) | (block17 >>> 60); + values[vi++] = (block17 >>> 18) & 4398046511103L; + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 262143L) << 24) | (block18 >>> 40); + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 1099511627775L) << 2) | (block19 >>> 62); + values[vi++] = (block19 >>> 20) & 4398046511103L; + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 1048575L) << 22) | (block20 >>> 42); + values[vi++] = block20 & 4398046511103L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 22) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi++] << 2) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi++] << 4) | (values[vi] >>> 38); + blocks[bi++] = (values[vi++] << 26) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 6) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 8) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 10) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 12) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 14) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 16) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 18) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 20) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation43 extends BulkOperation { + + public int blocks() { + return 43; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 21; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 2097151L) << 22) | (block1 >>> 42); + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 4398046511103L) << 1) | (block2 >>> 63); + values[vi++] = (block2 >>> 20) & 8796093022207L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 1048575L) << 23) | (block3 >>> 41); + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 2199023255551L) << 2) | (block4 >>> 62); + values[vi++] = (block4 >>> 19) & 8796093022207L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 524287L) << 24) | (block5 >>> 40); + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 1099511627775L) << 3) | (block6 >>> 61); + values[vi++] = (block6 >>> 18) & 8796093022207L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 262143L) << 25) | (block7 >>> 39); + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 549755813887L) << 4) | (block8 >>> 60); + values[vi++] = (block8 >>> 17) & 8796093022207L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 131071L) << 26) | (block9 >>> 38); + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 274877906943L) << 5) | (block10 >>> 59); + values[vi++] = (block10 >>> 16) & 8796093022207L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 65535L) << 27) | (block11 >>> 37); + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 137438953471L) << 6) | (block12 >>> 58); + values[vi++] = (block12 >>> 15) & 8796093022207L; + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 32767L) << 28) | (block13 >>> 36); + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 68719476735L) << 7) | (block14 >>> 57); + values[vi++] = (block14 >>> 14) & 8796093022207L; + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 16383L) << 29) | (block15 >>> 35); + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 34359738367L) << 8) | (block16 >>> 56); + values[vi++] = (block16 >>> 13) & 8796093022207L; + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 8191L) << 30) | (block17 >>> 34); + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 17179869183L) << 9) | (block18 >>> 55); + values[vi++] = (block18 >>> 12) & 8796093022207L; + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 4095L) << 31) | (block19 >>> 33); + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 8589934591L) << 10) | (block20 >>> 54); + values[vi++] = (block20 >>> 11) & 8796093022207L; + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 2047L) << 32) | (block21 >>> 32); + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 4294967295L) << 11) | (block22 >>> 53); + values[vi++] = (block22 >>> 10) & 8796093022207L; + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 1023L) << 33) | (block23 >>> 31); + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 2147483647L) << 12) | (block24 >>> 52); + values[vi++] = (block24 >>> 9) & 8796093022207L; + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 511L) << 34) | (block25 >>> 30); + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 1073741823L) << 13) | (block26 >>> 51); + values[vi++] = (block26 >>> 8) & 8796093022207L; + final long block27 = blocks[bi++]; + values[vi++] = ((block26 & 255L) << 35) | (block27 >>> 29); + final long block28 = blocks[bi++]; + values[vi++] = ((block27 & 536870911L) << 14) | (block28 >>> 50); + values[vi++] = (block28 >>> 7) & 8796093022207L; + final long block29 = blocks[bi++]; + values[vi++] = ((block28 & 127L) << 36) | (block29 >>> 28); + final long block30 = blocks[bi++]; + values[vi++] = ((block29 & 268435455L) << 15) | (block30 >>> 49); + values[vi++] = (block30 >>> 6) & 8796093022207L; + final long block31 = blocks[bi++]; + values[vi++] = ((block30 & 63L) << 37) | (block31 >>> 27); + final long block32 = blocks[bi++]; + values[vi++] = ((block31 & 134217727L) << 16) | (block32 >>> 48); + values[vi++] = (block32 >>> 5) & 8796093022207L; + final long block33 = blocks[bi++]; + values[vi++] = ((block32 & 31L) << 38) | (block33 >>> 26); + final long block34 = blocks[bi++]; + values[vi++] = ((block33 & 67108863L) << 17) | (block34 >>> 47); + values[vi++] = (block34 >>> 4) & 8796093022207L; + final long block35 = blocks[bi++]; + values[vi++] = ((block34 & 15L) << 39) | (block35 >>> 25); + final long block36 = blocks[bi++]; + values[vi++] = ((block35 & 33554431L) << 18) | (block36 >>> 46); + values[vi++] = (block36 >>> 3) & 8796093022207L; + final long block37 = blocks[bi++]; + values[vi++] = ((block36 & 7L) << 40) | (block37 >>> 24); + final long block38 = blocks[bi++]; + values[vi++] = ((block37 & 16777215L) << 19) | (block38 >>> 45); + values[vi++] = (block38 >>> 2) & 8796093022207L; + final long block39 = blocks[bi++]; + values[vi++] = ((block38 & 3L) << 41) | (block39 >>> 23); + final long block40 = blocks[bi++]; + values[vi++] = ((block39 & 8388607L) << 20) | (block40 >>> 44); + values[vi++] = (block40 >>> 1) & 8796093022207L; + final long block41 = blocks[bi++]; + values[vi++] = ((block40 & 1L) << 42) | (block41 >>> 22); + final long block42 = blocks[bi++]; + values[vi++] = ((block41 & 4194303L) << 21) | (block42 >>> 43); + values[vi++] = block42 & 8796093022207L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 21) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 20) | (values[vi] >>> 23); + blocks[bi++] = (values[vi++] << 41) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 19) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 18) | (values[vi] >>> 25); + blocks[bi++] = (values[vi++] << 39) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 17) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 16) | (values[vi] >>> 27); + blocks[bi++] = (values[vi++] << 37) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 15) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 14) | (values[vi] >>> 29); + blocks[bi++] = (values[vi++] << 35) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 13) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 12) | (values[vi] >>> 31); + blocks[bi++] = (values[vi++] << 33) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 11) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi++] << 10) | (values[vi] >>> 33); + blocks[bi++] = (values[vi++] << 31) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 9) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi++] << 8) | (values[vi] >>> 35); + blocks[bi++] = (values[vi++] << 29) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 7) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi++] << 6) | (values[vi] >>> 37); + blocks[bi++] = (values[vi++] << 27) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 5) | (values[vi] >>> 38); + blocks[bi++] = (values[vi++] << 26) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi++] << 4) | (values[vi] >>> 39); + blocks[bi++] = (values[vi++] << 25) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi++] << 3) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | (values[vi++] << 2) | (values[vi] >>> 41); + blocks[bi++] = (values[vi++] << 23) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi++] << 1) | (values[vi] >>> 42); + blocks[bi++] = (values[vi++] << 22) | (values[vi] >>> 21); + blocks[bi++] = (values[vi++] << 43) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation44 extends BulkOperation { + + public int blocks() { + return 11; + } + + public int values() { + return 16; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 20; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 1048575L) << 24) | (block1 >>> 40); + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 1099511627775L) << 4) | (block2 >>> 60); + values[vi++] = (block2 >>> 16) & 17592186044415L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 65535L) << 28) | (block3 >>> 36); + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 68719476735L) << 8) | (block4 >>> 56); + values[vi++] = (block4 >>> 12) & 17592186044415L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 4095L) << 32) | (block5 >>> 32); + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 4294967295L) << 12) | (block6 >>> 52); + values[vi++] = (block6 >>> 8) & 17592186044415L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 255L) << 36) | (block7 >>> 28); + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 268435455L) << 16) | (block8 >>> 48); + values[vi++] = (block8 >>> 4) & 17592186044415L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 15L) << 40) | (block9 >>> 24); + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 16777215L) << 20) | (block10 >>> 44); + values[vi++] = block10 & 17592186044415L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 20) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 16) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 12) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 8) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 4) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation45 extends BulkOperation { + + public int blocks() { + return 45; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 19; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 524287L) << 26) | (block1 >>> 38); + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 274877906943L) << 7) | (block2 >>> 57); + values[vi++] = (block2 >>> 12) & 35184372088831L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 4095L) << 33) | (block3 >>> 31); + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 2147483647L) << 14) | (block4 >>> 50); + values[vi++] = (block4 >>> 5) & 35184372088831L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 31L) << 40) | (block5 >>> 24); + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 16777215L) << 21) | (block6 >>> 43); + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 8796093022207L) << 2) | (block7 >>> 62); + values[vi++] = (block7 >>> 17) & 35184372088831L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 131071L) << 28) | (block8 >>> 36); + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 68719476735L) << 9) | (block9 >>> 55); + values[vi++] = (block9 >>> 10) & 35184372088831L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 1023L) << 35) | (block10 >>> 29); + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 536870911L) << 16) | (block11 >>> 48); + values[vi++] = (block11 >>> 3) & 35184372088831L; + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 7L) << 42) | (block12 >>> 22); + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 4194303L) << 23) | (block13 >>> 41); + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 2199023255551L) << 4) | (block14 >>> 60); + values[vi++] = (block14 >>> 15) & 35184372088831L; + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 32767L) << 30) | (block15 >>> 34); + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 17179869183L) << 11) | (block16 >>> 53); + values[vi++] = (block16 >>> 8) & 35184372088831L; + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 255L) << 37) | (block17 >>> 27); + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 134217727L) << 18) | (block18 >>> 46); + values[vi++] = (block18 >>> 1) & 35184372088831L; + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 1L) << 44) | (block19 >>> 20); + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 1048575L) << 25) | (block20 >>> 39); + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 549755813887L) << 6) | (block21 >>> 58); + values[vi++] = (block21 >>> 13) & 35184372088831L; + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 8191L) << 32) | (block22 >>> 32); + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 4294967295L) << 13) | (block23 >>> 51); + values[vi++] = (block23 >>> 6) & 35184372088831L; + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 63L) << 39) | (block24 >>> 25); + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 33554431L) << 20) | (block25 >>> 44); + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 17592186044415L) << 1) | (block26 >>> 63); + values[vi++] = (block26 >>> 18) & 35184372088831L; + final long block27 = blocks[bi++]; + values[vi++] = ((block26 & 262143L) << 27) | (block27 >>> 37); + final long block28 = blocks[bi++]; + values[vi++] = ((block27 & 137438953471L) << 8) | (block28 >>> 56); + values[vi++] = (block28 >>> 11) & 35184372088831L; + final long block29 = blocks[bi++]; + values[vi++] = ((block28 & 2047L) << 34) | (block29 >>> 30); + final long block30 = blocks[bi++]; + values[vi++] = ((block29 & 1073741823L) << 15) | (block30 >>> 49); + values[vi++] = (block30 >>> 4) & 35184372088831L; + final long block31 = blocks[bi++]; + values[vi++] = ((block30 & 15L) << 41) | (block31 >>> 23); + final long block32 = blocks[bi++]; + values[vi++] = ((block31 & 8388607L) << 22) | (block32 >>> 42); + final long block33 = blocks[bi++]; + values[vi++] = ((block32 & 4398046511103L) << 3) | (block33 >>> 61); + values[vi++] = (block33 >>> 16) & 35184372088831L; + final long block34 = blocks[bi++]; + values[vi++] = ((block33 & 65535L) << 29) | (block34 >>> 35); + final long block35 = blocks[bi++]; + values[vi++] = ((block34 & 34359738367L) << 10) | (block35 >>> 54); + values[vi++] = (block35 >>> 9) & 35184372088831L; + final long block36 = blocks[bi++]; + values[vi++] = ((block35 & 511L) << 36) | (block36 >>> 28); + final long block37 = blocks[bi++]; + values[vi++] = ((block36 & 268435455L) << 17) | (block37 >>> 47); + values[vi++] = (block37 >>> 2) & 35184372088831L; + final long block38 = blocks[bi++]; + values[vi++] = ((block37 & 3L) << 43) | (block38 >>> 21); + final long block39 = blocks[bi++]; + values[vi++] = ((block38 & 2097151L) << 24) | (block39 >>> 40); + final long block40 = blocks[bi++]; + values[vi++] = ((block39 & 1099511627775L) << 5) | (block40 >>> 59); + values[vi++] = (block40 >>> 14) & 35184372088831L; + final long block41 = blocks[bi++]; + values[vi++] = ((block40 & 16383L) << 31) | (block41 >>> 33); + final long block42 = blocks[bi++]; + values[vi++] = ((block41 & 8589934591L) << 12) | (block42 >>> 52); + values[vi++] = (block42 >>> 7) & 35184372088831L; + final long block43 = blocks[bi++]; + values[vi++] = ((block42 & 127L) << 38) | (block43 >>> 26); + final long block44 = blocks[bi++]; + values[vi++] = ((block43 & 67108863L) << 19) | (block44 >>> 45); + values[vi++] = block44 & 35184372088831L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 19) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 12) | (values[vi] >>> 33); + blocks[bi++] = (values[vi++] << 31) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 5) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 21); + blocks[bi++] = (values[vi++] << 43) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 17) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 10) | (values[vi] >>> 35); + blocks[bi++] = (values[vi++] << 29) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 3) | (values[vi] >>> 42); + blocks[bi++] = (values[vi++] << 22) | (values[vi] >>> 23); + blocks[bi++] = (values[vi++] << 41) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 15) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi++] << 8) | (values[vi] >>> 37); + blocks[bi++] = (values[vi++] << 27) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi++] << 1) | (values[vi] >>> 44); + blocks[bi++] = (values[vi++] << 20) | (values[vi] >>> 25); + blocks[bi++] = (values[vi++] << 39) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 13) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi++] << 6) | (values[vi] >>> 39); + blocks[bi++] = (values[vi++] << 25) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 18) | (values[vi] >>> 27); + blocks[bi++] = (values[vi++] << 37) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 11) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi++] << 4) | (values[vi] >>> 41); + blocks[bi++] = (values[vi++] << 23) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 16) | (values[vi] >>> 29); + blocks[bi++] = (values[vi++] << 35) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 9) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi++] << 2) | (values[vi] >>> 43); + blocks[bi++] = (values[vi++] << 21) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 14) | (values[vi] >>> 31); + blocks[bi++] = (values[vi++] << 33) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 7) | (values[vi] >>> 38); + blocks[bi++] = (values[vi++] << 26) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation46 extends BulkOperation { + + public int blocks() { + return 23; + } + + public int values() { + return 32; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 18; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 262143L) << 28) | (block1 >>> 36); + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 68719476735L) << 10) | (block2 >>> 54); + values[vi++] = (block2 >>> 8) & 70368744177663L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 255L) << 38) | (block3 >>> 26); + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 67108863L) << 20) | (block4 >>> 44); + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 17592186044415L) << 2) | (block5 >>> 62); + values[vi++] = (block5 >>> 16) & 70368744177663L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 65535L) << 30) | (block6 >>> 34); + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 17179869183L) << 12) | (block7 >>> 52); + values[vi++] = (block7 >>> 6) & 70368744177663L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 63L) << 40) | (block8 >>> 24); + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 16777215L) << 22) | (block9 >>> 42); + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 4398046511103L) << 4) | (block10 >>> 60); + values[vi++] = (block10 >>> 14) & 70368744177663L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 16383L) << 32) | (block11 >>> 32); + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 4294967295L) << 14) | (block12 >>> 50); + values[vi++] = (block12 >>> 4) & 70368744177663L; + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 15L) << 42) | (block13 >>> 22); + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 4194303L) << 24) | (block14 >>> 40); + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 1099511627775L) << 6) | (block15 >>> 58); + values[vi++] = (block15 >>> 12) & 70368744177663L; + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 4095L) << 34) | (block16 >>> 30); + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 1073741823L) << 16) | (block17 >>> 48); + values[vi++] = (block17 >>> 2) & 70368744177663L; + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 3L) << 44) | (block18 >>> 20); + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 1048575L) << 26) | (block19 >>> 38); + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 274877906943L) << 8) | (block20 >>> 56); + values[vi++] = (block20 >>> 10) & 70368744177663L; + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 1023L) << 36) | (block21 >>> 28); + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 268435455L) << 18) | (block22 >>> 46); + values[vi++] = block22 & 70368744177663L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 18) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 8) | (values[vi] >>> 38); + blocks[bi++] = (values[vi++] << 26) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 16) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 6) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 14) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 4) | (values[vi] >>> 42); + blocks[bi++] = (values[vi++] << 22) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 12) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 2) | (values[vi] >>> 44); + blocks[bi++] = (values[vi++] << 20) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 10) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation47 extends BulkOperation { + + public int blocks() { + return 47; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 17; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 131071L) << 30) | (block1 >>> 34); + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 17179869183L) << 13) | (block2 >>> 51); + values[vi++] = (block2 >>> 4) & 140737488355327L; + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 15L) << 43) | (block3 >>> 21); + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 2097151L) << 26) | (block4 >>> 38); + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 274877906943L) << 9) | (block5 >>> 55); + values[vi++] = (block5 >>> 8) & 140737488355327L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 255L) << 39) | (block6 >>> 25); + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 33554431L) << 22) | (block7 >>> 42); + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 4398046511103L) << 5) | (block8 >>> 59); + values[vi++] = (block8 >>> 12) & 140737488355327L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 4095L) << 35) | (block9 >>> 29); + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 536870911L) << 18) | (block10 >>> 46); + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 70368744177663L) << 1) | (block11 >>> 63); + values[vi++] = (block11 >>> 16) & 140737488355327L; + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 65535L) << 31) | (block12 >>> 33); + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 8589934591L) << 14) | (block13 >>> 50); + values[vi++] = (block13 >>> 3) & 140737488355327L; + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 7L) << 44) | (block14 >>> 20); + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 1048575L) << 27) | (block15 >>> 37); + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 137438953471L) << 10) | (block16 >>> 54); + values[vi++] = (block16 >>> 7) & 140737488355327L; + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 127L) << 40) | (block17 >>> 24); + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 16777215L) << 23) | (block18 >>> 41); + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 2199023255551L) << 6) | (block19 >>> 58); + values[vi++] = (block19 >>> 11) & 140737488355327L; + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 2047L) << 36) | (block20 >>> 28); + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 268435455L) << 19) | (block21 >>> 45); + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 35184372088831L) << 2) | (block22 >>> 62); + values[vi++] = (block22 >>> 15) & 140737488355327L; + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 32767L) << 32) | (block23 >>> 32); + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 4294967295L) << 15) | (block24 >>> 49); + values[vi++] = (block24 >>> 2) & 140737488355327L; + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 3L) << 45) | (block25 >>> 19); + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 524287L) << 28) | (block26 >>> 36); + final long block27 = blocks[bi++]; + values[vi++] = ((block26 & 68719476735L) << 11) | (block27 >>> 53); + values[vi++] = (block27 >>> 6) & 140737488355327L; + final long block28 = blocks[bi++]; + values[vi++] = ((block27 & 63L) << 41) | (block28 >>> 23); + final long block29 = blocks[bi++]; + values[vi++] = ((block28 & 8388607L) << 24) | (block29 >>> 40); + final long block30 = blocks[bi++]; + values[vi++] = ((block29 & 1099511627775L) << 7) | (block30 >>> 57); + values[vi++] = (block30 >>> 10) & 140737488355327L; + final long block31 = blocks[bi++]; + values[vi++] = ((block30 & 1023L) << 37) | (block31 >>> 27); + final long block32 = blocks[bi++]; + values[vi++] = ((block31 & 134217727L) << 20) | (block32 >>> 44); + final long block33 = blocks[bi++]; + values[vi++] = ((block32 & 17592186044415L) << 3) | (block33 >>> 61); + values[vi++] = (block33 >>> 14) & 140737488355327L; + final long block34 = blocks[bi++]; + values[vi++] = ((block33 & 16383L) << 33) | (block34 >>> 31); + final long block35 = blocks[bi++]; + values[vi++] = ((block34 & 2147483647L) << 16) | (block35 >>> 48); + values[vi++] = (block35 >>> 1) & 140737488355327L; + final long block36 = blocks[bi++]; + values[vi++] = ((block35 & 1L) << 46) | (block36 >>> 18); + final long block37 = blocks[bi++]; + values[vi++] = ((block36 & 262143L) << 29) | (block37 >>> 35); + final long block38 = blocks[bi++]; + values[vi++] = ((block37 & 34359738367L) << 12) | (block38 >>> 52); + values[vi++] = (block38 >>> 5) & 140737488355327L; + final long block39 = blocks[bi++]; + values[vi++] = ((block38 & 31L) << 42) | (block39 >>> 22); + final long block40 = blocks[bi++]; + values[vi++] = ((block39 & 4194303L) << 25) | (block40 >>> 39); + final long block41 = blocks[bi++]; + values[vi++] = ((block40 & 549755813887L) << 8) | (block41 >>> 56); + values[vi++] = (block41 >>> 9) & 140737488355327L; + final long block42 = blocks[bi++]; + values[vi++] = ((block41 & 511L) << 38) | (block42 >>> 26); + final long block43 = blocks[bi++]; + values[vi++] = ((block42 & 67108863L) << 21) | (block43 >>> 43); + final long block44 = blocks[bi++]; + values[vi++] = ((block43 & 8796093022207L) << 4) | (block44 >>> 60); + values[vi++] = (block44 >>> 13) & 140737488355327L; + final long block45 = blocks[bi++]; + values[vi++] = ((block44 & 8191L) << 34) | (block45 >>> 30); + final long block46 = blocks[bi++]; + values[vi++] = ((block45 & 1073741823L) << 17) | (block46 >>> 47); + values[vi++] = block46 & 140737488355327L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 17) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi++] << 4) | (values[vi] >>> 43); + blocks[bi++] = (values[vi++] << 21) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 8) | (values[vi] >>> 39); + blocks[bi++] = (values[vi++] << 25) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 12) | (values[vi] >>> 35); + blocks[bi++] = (values[vi++] << 29) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 16) | (values[vi] >>> 31); + blocks[bi++] = (values[vi++] << 33) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 3) | (values[vi] >>> 44); + blocks[bi++] = (values[vi++] << 20) | (values[vi] >>> 27); + blocks[bi++] = (values[vi++] << 37) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 7) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 23); + blocks[bi++] = (values[vi++] << 41) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 11) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 15) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi++] << 2) | (values[vi] >>> 45); + blocks[bi++] = (values[vi++] << 19) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi++] << 6) | (values[vi] >>> 41); + blocks[bi++] = (values[vi++] << 23) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 10) | (values[vi] >>> 37); + blocks[bi++] = (values[vi++] << 27) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 14) | (values[vi] >>> 33); + blocks[bi++] = (values[vi++] << 31) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi++] << 1) | (values[vi] >>> 46); + blocks[bi++] = (values[vi++] << 18) | (values[vi] >>> 29); + blocks[bi++] = (values[vi++] << 35) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 5) | (values[vi] >>> 42); + blocks[bi++] = (values[vi++] << 22) | (values[vi] >>> 25); + blocks[bi++] = (values[vi++] << 39) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 9) | (values[vi] >>> 38); + blocks[bi++] = (values[vi++] << 26) | (values[vi] >>> 21); + blocks[bi++] = (values[vi++] << 43) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 13) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation48 extends BulkOperation { + + public int blocks() { + return 3; + } + + public int values() { + return 4; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 16; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 65535L) << 32) | (block1 >>> 32); + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 4294967295L) << 16) | (block2 >>> 48); + values[vi++] = block2 & 281474976710655L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 16) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation49 extends BulkOperation { + + public int blocks() { + return 49; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 15; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 32767L) << 34) | (block1 >>> 30); + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 1073741823L) << 19) | (block2 >>> 45); + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 35184372088831L) << 4) | (block3 >>> 60); + values[vi++] = (block3 >>> 11) & 562949953421311L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 2047L) << 38) | (block4 >>> 26); + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 67108863L) << 23) | (block5 >>> 41); + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 2199023255551L) << 8) | (block6 >>> 56); + values[vi++] = (block6 >>> 7) & 562949953421311L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 127L) << 42) | (block7 >>> 22); + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 4194303L) << 27) | (block8 >>> 37); + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 137438953471L) << 12) | (block9 >>> 52); + values[vi++] = (block9 >>> 3) & 562949953421311L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 7L) << 46) | (block10 >>> 18); + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 262143L) << 31) | (block11 >>> 33); + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 8589934591L) << 16) | (block12 >>> 48); + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 281474976710655L) << 1) | (block13 >>> 63); + values[vi++] = (block13 >>> 14) & 562949953421311L; + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 16383L) << 35) | (block14 >>> 29); + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 536870911L) << 20) | (block15 >>> 44); + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 17592186044415L) << 5) | (block16 >>> 59); + values[vi++] = (block16 >>> 10) & 562949953421311L; + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 1023L) << 39) | (block17 >>> 25); + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 33554431L) << 24) | (block18 >>> 40); + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 1099511627775L) << 9) | (block19 >>> 55); + values[vi++] = (block19 >>> 6) & 562949953421311L; + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 63L) << 43) | (block20 >>> 21); + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 2097151L) << 28) | (block21 >>> 36); + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 68719476735L) << 13) | (block22 >>> 51); + values[vi++] = (block22 >>> 2) & 562949953421311L; + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 3L) << 47) | (block23 >>> 17); + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 131071L) << 32) | (block24 >>> 32); + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 4294967295L) << 17) | (block25 >>> 47); + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 140737488355327L) << 2) | (block26 >>> 62); + values[vi++] = (block26 >>> 13) & 562949953421311L; + final long block27 = blocks[bi++]; + values[vi++] = ((block26 & 8191L) << 36) | (block27 >>> 28); + final long block28 = blocks[bi++]; + values[vi++] = ((block27 & 268435455L) << 21) | (block28 >>> 43); + final long block29 = blocks[bi++]; + values[vi++] = ((block28 & 8796093022207L) << 6) | (block29 >>> 58); + values[vi++] = (block29 >>> 9) & 562949953421311L; + final long block30 = blocks[bi++]; + values[vi++] = ((block29 & 511L) << 40) | (block30 >>> 24); + final long block31 = blocks[bi++]; + values[vi++] = ((block30 & 16777215L) << 25) | (block31 >>> 39); + final long block32 = blocks[bi++]; + values[vi++] = ((block31 & 549755813887L) << 10) | (block32 >>> 54); + values[vi++] = (block32 >>> 5) & 562949953421311L; + final long block33 = blocks[bi++]; + values[vi++] = ((block32 & 31L) << 44) | (block33 >>> 20); + final long block34 = blocks[bi++]; + values[vi++] = ((block33 & 1048575L) << 29) | (block34 >>> 35); + final long block35 = blocks[bi++]; + values[vi++] = ((block34 & 34359738367L) << 14) | (block35 >>> 50); + values[vi++] = (block35 >>> 1) & 562949953421311L; + final long block36 = blocks[bi++]; + values[vi++] = ((block35 & 1L) << 48) | (block36 >>> 16); + final long block37 = blocks[bi++]; + values[vi++] = ((block36 & 65535L) << 33) | (block37 >>> 31); + final long block38 = blocks[bi++]; + values[vi++] = ((block37 & 2147483647L) << 18) | (block38 >>> 46); + final long block39 = blocks[bi++]; + values[vi++] = ((block38 & 70368744177663L) << 3) | (block39 >>> 61); + values[vi++] = (block39 >>> 12) & 562949953421311L; + final long block40 = blocks[bi++]; + values[vi++] = ((block39 & 4095L) << 37) | (block40 >>> 27); + final long block41 = blocks[bi++]; + values[vi++] = ((block40 & 134217727L) << 22) | (block41 >>> 42); + final long block42 = blocks[bi++]; + values[vi++] = ((block41 & 4398046511103L) << 7) | (block42 >>> 57); + values[vi++] = (block42 >>> 8) & 562949953421311L; + final long block43 = blocks[bi++]; + values[vi++] = ((block42 & 255L) << 41) | (block43 >>> 23); + final long block44 = blocks[bi++]; + values[vi++] = ((block43 & 8388607L) << 26) | (block44 >>> 38); + final long block45 = blocks[bi++]; + values[vi++] = ((block44 & 274877906943L) << 11) | (block45 >>> 53); + values[vi++] = (block45 >>> 4) & 562949953421311L; + final long block46 = blocks[bi++]; + values[vi++] = ((block45 & 15L) << 45) | (block46 >>> 19); + final long block47 = blocks[bi++]; + values[vi++] = ((block46 & 524287L) << 30) | (block47 >>> 34); + final long block48 = blocks[bi++]; + values[vi++] = ((block47 & 17179869183L) << 15) | (block48 >>> 49); + values[vi++] = block48 & 562949953421311L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 15) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 11) | (values[vi] >>> 38); + blocks[bi++] = (values[vi++] << 26) | (values[vi] >>> 23); + blocks[bi++] = (values[vi++] << 41) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 7) | (values[vi] >>> 42); + blocks[bi++] = (values[vi++] << 22) | (values[vi] >>> 27); + blocks[bi++] = (values[vi++] << 37) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 3) | (values[vi] >>> 46); + blocks[bi++] = (values[vi++] << 18) | (values[vi] >>> 31); + blocks[bi++] = (values[vi++] << 33) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 14) | (values[vi] >>> 35); + blocks[bi++] = (values[vi++] << 29) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 10) | (values[vi] >>> 39); + blocks[bi++] = (values[vi++] << 25) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 6) | (values[vi] >>> 43); + blocks[bi++] = (values[vi++] << 21) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi++] << 2) | (values[vi] >>> 47); + blocks[bi++] = (values[vi++] << 17) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 13) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 21); + blocks[bi++] = (values[vi++] << 43) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 9) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 25); + blocks[bi++] = (values[vi++] << 39) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 5) | (values[vi] >>> 44); + blocks[bi++] = (values[vi++] << 20) | (values[vi] >>> 29); + blocks[bi++] = (values[vi++] << 35) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi++] << 1) | (values[vi] >>> 48); + blocks[bi++] = (values[vi++] << 16) | (values[vi] >>> 33); + blocks[bi++] = (values[vi++] << 31) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 12) | (values[vi] >>> 37); + blocks[bi++] = (values[vi++] << 27) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 8) | (values[vi] >>> 41); + blocks[bi++] = (values[vi++] << 23) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi++] << 4) | (values[vi] >>> 45); + blocks[bi++] = (values[vi++] << 19) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation50 extends BulkOperation { + + public int blocks() { + return 25; + } + + public int values() { + return 32; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 14; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 16383L) << 36) | (block1 >>> 28); + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 268435455L) << 22) | (block2 >>> 42); + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 4398046511103L) << 8) | (block3 >>> 56); + values[vi++] = (block3 >>> 6) & 1125899906842623L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 63L) << 44) | (block4 >>> 20); + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 1048575L) << 30) | (block5 >>> 34); + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 17179869183L) << 16) | (block6 >>> 48); + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 281474976710655L) << 2) | (block7 >>> 62); + values[vi++] = (block7 >>> 12) & 1125899906842623L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 4095L) << 38) | (block8 >>> 26); + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 67108863L) << 24) | (block9 >>> 40); + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 1099511627775L) << 10) | (block10 >>> 54); + values[vi++] = (block10 >>> 4) & 1125899906842623L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 15L) << 46) | (block11 >>> 18); + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 262143L) << 32) | (block12 >>> 32); + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 4294967295L) << 18) | (block13 >>> 46); + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 70368744177663L) << 4) | (block14 >>> 60); + values[vi++] = (block14 >>> 10) & 1125899906842623L; + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 1023L) << 40) | (block15 >>> 24); + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 16777215L) << 26) | (block16 >>> 38); + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 274877906943L) << 12) | (block17 >>> 52); + values[vi++] = (block17 >>> 2) & 1125899906842623L; + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 3L) << 48) | (block18 >>> 16); + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 65535L) << 34) | (block19 >>> 30); + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 1073741823L) << 20) | (block20 >>> 44); + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 17592186044415L) << 6) | (block21 >>> 58); + values[vi++] = (block21 >>> 8) & 1125899906842623L; + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 255L) << 42) | (block22 >>> 22); + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 4194303L) << 28) | (block23 >>> 36); + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 68719476735L) << 14) | (block24 >>> 50); + values[vi++] = block24 & 1125899906842623L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 14) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 6) | (values[vi] >>> 44); + blocks[bi++] = (values[vi++] << 20) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 12) | (values[vi] >>> 38); + blocks[bi++] = (values[vi++] << 26) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 4) | (values[vi] >>> 46); + blocks[bi++] = (values[vi++] << 18) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 10) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 2) | (values[vi] >>> 48); + blocks[bi++] = (values[vi++] << 16) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 8) | (values[vi] >>> 42); + blocks[bi++] = (values[vi++] << 22) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation51 extends BulkOperation { + + public int blocks() { + return 51; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 13; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 8191L) << 38) | (block1 >>> 26); + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 67108863L) << 25) | (block2 >>> 39); + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 549755813887L) << 12) | (block3 >>> 52); + values[vi++] = (block3 >>> 1) & 2251799813685247L; + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 1L) << 50) | (block4 >>> 14); + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 16383L) << 37) | (block5 >>> 27); + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 134217727L) << 24) | (block6 >>> 40); + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 1099511627775L) << 11) | (block7 >>> 53); + values[vi++] = (block7 >>> 2) & 2251799813685247L; + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 3L) << 49) | (block8 >>> 15); + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 32767L) << 36) | (block9 >>> 28); + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 268435455L) << 23) | (block10 >>> 41); + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 2199023255551L) << 10) | (block11 >>> 54); + values[vi++] = (block11 >>> 3) & 2251799813685247L; + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 7L) << 48) | (block12 >>> 16); + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 65535L) << 35) | (block13 >>> 29); + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 536870911L) << 22) | (block14 >>> 42); + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 4398046511103L) << 9) | (block15 >>> 55); + values[vi++] = (block15 >>> 4) & 2251799813685247L; + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 15L) << 47) | (block16 >>> 17); + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 131071L) << 34) | (block17 >>> 30); + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 1073741823L) << 21) | (block18 >>> 43); + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 8796093022207L) << 8) | (block19 >>> 56); + values[vi++] = (block19 >>> 5) & 2251799813685247L; + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 31L) << 46) | (block20 >>> 18); + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 262143L) << 33) | (block21 >>> 31); + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 2147483647L) << 20) | (block22 >>> 44); + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 17592186044415L) << 7) | (block23 >>> 57); + values[vi++] = (block23 >>> 6) & 2251799813685247L; + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 63L) << 45) | (block24 >>> 19); + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 524287L) << 32) | (block25 >>> 32); + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 4294967295L) << 19) | (block26 >>> 45); + final long block27 = blocks[bi++]; + values[vi++] = ((block26 & 35184372088831L) << 6) | (block27 >>> 58); + values[vi++] = (block27 >>> 7) & 2251799813685247L; + final long block28 = blocks[bi++]; + values[vi++] = ((block27 & 127L) << 44) | (block28 >>> 20); + final long block29 = blocks[bi++]; + values[vi++] = ((block28 & 1048575L) << 31) | (block29 >>> 33); + final long block30 = blocks[bi++]; + values[vi++] = ((block29 & 8589934591L) << 18) | (block30 >>> 46); + final long block31 = blocks[bi++]; + values[vi++] = ((block30 & 70368744177663L) << 5) | (block31 >>> 59); + values[vi++] = (block31 >>> 8) & 2251799813685247L; + final long block32 = blocks[bi++]; + values[vi++] = ((block31 & 255L) << 43) | (block32 >>> 21); + final long block33 = blocks[bi++]; + values[vi++] = ((block32 & 2097151L) << 30) | (block33 >>> 34); + final long block34 = blocks[bi++]; + values[vi++] = ((block33 & 17179869183L) << 17) | (block34 >>> 47); + final long block35 = blocks[bi++]; + values[vi++] = ((block34 & 140737488355327L) << 4) | (block35 >>> 60); + values[vi++] = (block35 >>> 9) & 2251799813685247L; + final long block36 = blocks[bi++]; + values[vi++] = ((block35 & 511L) << 42) | (block36 >>> 22); + final long block37 = blocks[bi++]; + values[vi++] = ((block36 & 4194303L) << 29) | (block37 >>> 35); + final long block38 = blocks[bi++]; + values[vi++] = ((block37 & 34359738367L) << 16) | (block38 >>> 48); + final long block39 = blocks[bi++]; + values[vi++] = ((block38 & 281474976710655L) << 3) | (block39 >>> 61); + values[vi++] = (block39 >>> 10) & 2251799813685247L; + final long block40 = blocks[bi++]; + values[vi++] = ((block39 & 1023L) << 41) | (block40 >>> 23); + final long block41 = blocks[bi++]; + values[vi++] = ((block40 & 8388607L) << 28) | (block41 >>> 36); + final long block42 = blocks[bi++]; + values[vi++] = ((block41 & 68719476735L) << 15) | (block42 >>> 49); + final long block43 = blocks[bi++]; + values[vi++] = ((block42 & 562949953421311L) << 2) | (block43 >>> 62); + values[vi++] = (block43 >>> 11) & 2251799813685247L; + final long block44 = blocks[bi++]; + values[vi++] = ((block43 & 2047L) << 40) | (block44 >>> 24); + final long block45 = blocks[bi++]; + values[vi++] = ((block44 & 16777215L) << 27) | (block45 >>> 37); + final long block46 = blocks[bi++]; + values[vi++] = ((block45 & 137438953471L) << 14) | (block46 >>> 50); + final long block47 = blocks[bi++]; + values[vi++] = ((block46 & 1125899906842623L) << 1) | (block47 >>> 63); + values[vi++] = (block47 >>> 12) & 2251799813685247L; + final long block48 = blocks[bi++]; + values[vi++] = ((block47 & 4095L) << 39) | (block48 >>> 25); + final long block49 = blocks[bi++]; + values[vi++] = ((block48 & 33554431L) << 26) | (block49 >>> 38); + final long block50 = blocks[bi++]; + values[vi++] = ((block49 & 274877906943L) << 13) | (block50 >>> 51); + values[vi++] = block50 & 2251799813685247L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 13) | (values[vi] >>> 38); + blocks[bi++] = (values[vi++] << 26) | (values[vi] >>> 25); + blocks[bi++] = (values[vi++] << 39) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi++] << 1) | (values[vi] >>> 50); + blocks[bi++] = (values[vi++] << 14) | (values[vi] >>> 37); + blocks[bi++] = (values[vi++] << 27) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi++] << 2) | (values[vi] >>> 49); + blocks[bi++] = (values[vi++] << 15) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 23); + blocks[bi++] = (values[vi++] << 41) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 3) | (values[vi] >>> 48); + blocks[bi++] = (values[vi++] << 16) | (values[vi] >>> 35); + blocks[bi++] = (values[vi++] << 29) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 4) | (values[vi] >>> 47); + blocks[bi++] = (values[vi++] << 17) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 21); + blocks[bi++] = (values[vi++] << 43) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 5) | (values[vi] >>> 46); + blocks[bi++] = (values[vi++] << 18) | (values[vi] >>> 33); + blocks[bi++] = (values[vi++] << 31) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 6) | (values[vi] >>> 45); + blocks[bi++] = (values[vi++] << 19) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 7) | (values[vi] >>> 44); + blocks[bi++] = (values[vi++] << 20) | (values[vi] >>> 31); + blocks[bi++] = (values[vi++] << 33) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 8) | (values[vi] >>> 43); + blocks[bi++] = (values[vi++] << 21) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 9) | (values[vi] >>> 42); + blocks[bi++] = (values[vi++] << 22) | (values[vi] >>> 29); + blocks[bi++] = (values[vi++] << 35) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 10) | (values[vi] >>> 41); + blocks[bi++] = (values[vi++] << 23) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 11) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 27); + blocks[bi++] = (values[vi++] << 37) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 12) | (values[vi] >>> 39); + blocks[bi++] = (values[vi++] << 25) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation52 extends BulkOperation { + + public int blocks() { + return 13; + } + + public int values() { + return 16; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 12; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 4095L) << 40) | (block1 >>> 24); + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 16777215L) << 28) | (block2 >>> 36); + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 68719476735L) << 16) | (block3 >>> 48); + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 281474976710655L) << 4) | (block4 >>> 60); + values[vi++] = (block4 >>> 8) & 4503599627370495L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 255L) << 44) | (block5 >>> 20); + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 1048575L) << 32) | (block6 >>> 32); + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 4294967295L) << 20) | (block7 >>> 44); + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 17592186044415L) << 8) | (block8 >>> 56); + values[vi++] = (block8 >>> 4) & 4503599627370495L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 15L) << 48) | (block9 >>> 16); + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 65535L) << 36) | (block10 >>> 28); + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 268435455L) << 24) | (block11 >>> 40); + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 1099511627775L) << 12) | (block12 >>> 52); + values[vi++] = block12 & 4503599627370495L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 12) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 8) | (values[vi] >>> 44); + blocks[bi++] = (values[vi++] << 20) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 4) | (values[vi] >>> 48); + blocks[bi++] = (values[vi++] << 16) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation53 extends BulkOperation { + + public int blocks() { + return 53; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 11; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 2047L) << 42) | (block1 >>> 22); + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 4194303L) << 31) | (block2 >>> 33); + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 8589934591L) << 20) | (block3 >>> 44); + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 17592186044415L) << 9) | (block4 >>> 55); + values[vi++] = (block4 >>> 2) & 9007199254740991L; + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 3L) << 51) | (block5 >>> 13); + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 8191L) << 40) | (block6 >>> 24); + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 16777215L) << 29) | (block7 >>> 35); + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 34359738367L) << 18) | (block8 >>> 46); + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 70368744177663L) << 7) | (block9 >>> 57); + values[vi++] = (block9 >>> 4) & 9007199254740991L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 15L) << 49) | (block10 >>> 15); + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 32767L) << 38) | (block11 >>> 26); + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 67108863L) << 27) | (block12 >>> 37); + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 137438953471L) << 16) | (block13 >>> 48); + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 281474976710655L) << 5) | (block14 >>> 59); + values[vi++] = (block14 >>> 6) & 9007199254740991L; + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 63L) << 47) | (block15 >>> 17); + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 131071L) << 36) | (block16 >>> 28); + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 268435455L) << 25) | (block17 >>> 39); + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 549755813887L) << 14) | (block18 >>> 50); + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 1125899906842623L) << 3) | (block19 >>> 61); + values[vi++] = (block19 >>> 8) & 9007199254740991L; + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 255L) << 45) | (block20 >>> 19); + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 524287L) << 34) | (block21 >>> 30); + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 1073741823L) << 23) | (block22 >>> 41); + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 2199023255551L) << 12) | (block23 >>> 52); + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 4503599627370495L) << 1) | (block24 >>> 63); + values[vi++] = (block24 >>> 10) & 9007199254740991L; + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 1023L) << 43) | (block25 >>> 21); + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 2097151L) << 32) | (block26 >>> 32); + final long block27 = blocks[bi++]; + values[vi++] = ((block26 & 4294967295L) << 21) | (block27 >>> 43); + final long block28 = blocks[bi++]; + values[vi++] = ((block27 & 8796093022207L) << 10) | (block28 >>> 54); + values[vi++] = (block28 >>> 1) & 9007199254740991L; + final long block29 = blocks[bi++]; + values[vi++] = ((block28 & 1L) << 52) | (block29 >>> 12); + final long block30 = blocks[bi++]; + values[vi++] = ((block29 & 4095L) << 41) | (block30 >>> 23); + final long block31 = blocks[bi++]; + values[vi++] = ((block30 & 8388607L) << 30) | (block31 >>> 34); + final long block32 = blocks[bi++]; + values[vi++] = ((block31 & 17179869183L) << 19) | (block32 >>> 45); + final long block33 = blocks[bi++]; + values[vi++] = ((block32 & 35184372088831L) << 8) | (block33 >>> 56); + values[vi++] = (block33 >>> 3) & 9007199254740991L; + final long block34 = blocks[bi++]; + values[vi++] = ((block33 & 7L) << 50) | (block34 >>> 14); + final long block35 = blocks[bi++]; + values[vi++] = ((block34 & 16383L) << 39) | (block35 >>> 25); + final long block36 = blocks[bi++]; + values[vi++] = ((block35 & 33554431L) << 28) | (block36 >>> 36); + final long block37 = blocks[bi++]; + values[vi++] = ((block36 & 68719476735L) << 17) | (block37 >>> 47); + final long block38 = blocks[bi++]; + values[vi++] = ((block37 & 140737488355327L) << 6) | (block38 >>> 58); + values[vi++] = (block38 >>> 5) & 9007199254740991L; + final long block39 = blocks[bi++]; + values[vi++] = ((block38 & 31L) << 48) | (block39 >>> 16); + final long block40 = blocks[bi++]; + values[vi++] = ((block39 & 65535L) << 37) | (block40 >>> 27); + final long block41 = blocks[bi++]; + values[vi++] = ((block40 & 134217727L) << 26) | (block41 >>> 38); + final long block42 = blocks[bi++]; + values[vi++] = ((block41 & 274877906943L) << 15) | (block42 >>> 49); + final long block43 = blocks[bi++]; + values[vi++] = ((block42 & 562949953421311L) << 4) | (block43 >>> 60); + values[vi++] = (block43 >>> 7) & 9007199254740991L; + final long block44 = blocks[bi++]; + values[vi++] = ((block43 & 127L) << 46) | (block44 >>> 18); + final long block45 = blocks[bi++]; + values[vi++] = ((block44 & 262143L) << 35) | (block45 >>> 29); + final long block46 = blocks[bi++]; + values[vi++] = ((block45 & 536870911L) << 24) | (block46 >>> 40); + final long block47 = blocks[bi++]; + values[vi++] = ((block46 & 1099511627775L) << 13) | (block47 >>> 51); + final long block48 = blocks[bi++]; + values[vi++] = ((block47 & 2251799813685247L) << 2) | (block48 >>> 62); + values[vi++] = (block48 >>> 9) & 9007199254740991L; + final long block49 = blocks[bi++]; + values[vi++] = ((block48 & 511L) << 44) | (block49 >>> 20); + final long block50 = blocks[bi++]; + values[vi++] = ((block49 & 1048575L) << 33) | (block50 >>> 31); + final long block51 = blocks[bi++]; + values[vi++] = ((block50 & 2147483647L) << 22) | (block51 >>> 42); + final long block52 = blocks[bi++]; + values[vi++] = ((block51 & 4398046511103L) << 11) | (block52 >>> 53); + values[vi++] = block52 & 9007199254740991L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 11) | (values[vi] >>> 42); + blocks[bi++] = (values[vi++] << 22) | (values[vi] >>> 31); + blocks[bi++] = (values[vi++] << 33) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi++] << 2) | (values[vi] >>> 51); + blocks[bi++] = (values[vi++] << 13) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 29); + blocks[bi++] = (values[vi++] << 35) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 4) | (values[vi] >>> 49); + blocks[bi++] = (values[vi++] << 15) | (values[vi] >>> 38); + blocks[bi++] = (values[vi++] << 26) | (values[vi] >>> 27); + blocks[bi++] = (values[vi++] << 37) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 6) | (values[vi] >>> 47); + blocks[bi++] = (values[vi++] << 17) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 25); + blocks[bi++] = (values[vi++] << 39) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 8) | (values[vi] >>> 45); + blocks[bi++] = (values[vi++] << 19) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 23); + blocks[bi++] = (values[vi++] << 41) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 10) | (values[vi] >>> 43); + blocks[bi++] = (values[vi++] << 21) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 21); + blocks[bi++] = (values[vi++] << 43) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi++] << 1) | (values[vi] >>> 52); + blocks[bi++] = (values[vi++] << 12) | (values[vi] >>> 41); + blocks[bi++] = (values[vi++] << 23) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 3) | (values[vi] >>> 50); + blocks[bi++] = (values[vi++] << 14) | (values[vi] >>> 39); + blocks[bi++] = (values[vi++] << 25) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 5) | (values[vi] >>> 48); + blocks[bi++] = (values[vi++] << 16) | (values[vi] >>> 37); + blocks[bi++] = (values[vi++] << 27) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 7) | (values[vi] >>> 46); + blocks[bi++] = (values[vi++] << 18) | (values[vi] >>> 35); + blocks[bi++] = (values[vi++] << 29) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 9) | (values[vi] >>> 44); + blocks[bi++] = (values[vi++] << 20) | (values[vi] >>> 33); + blocks[bi++] = (values[vi++] << 31) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation54 extends BulkOperation { + + public int blocks() { + return 27; + } + + public int values() { + return 32; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 10; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 1023L) << 44) | (block1 >>> 20); + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 1048575L) << 34) | (block2 >>> 30); + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 1073741823L) << 24) | (block3 >>> 40); + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 1099511627775L) << 14) | (block4 >>> 50); + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 1125899906842623L) << 4) | (block5 >>> 60); + values[vi++] = (block5 >>> 6) & 18014398509481983L; + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 63L) << 48) | (block6 >>> 16); + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 65535L) << 38) | (block7 >>> 26); + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 67108863L) << 28) | (block8 >>> 36); + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 68719476735L) << 18) | (block9 >>> 46); + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 70368744177663L) << 8) | (block10 >>> 56); + values[vi++] = (block10 >>> 2) & 18014398509481983L; + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 3L) << 52) | (block11 >>> 12); + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 4095L) << 42) | (block12 >>> 22); + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 4194303L) << 32) | (block13 >>> 32); + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 4294967295L) << 22) | (block14 >>> 42); + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 4398046511103L) << 12) | (block15 >>> 52); + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 4503599627370495L) << 2) | (block16 >>> 62); + values[vi++] = (block16 >>> 8) & 18014398509481983L; + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 255L) << 46) | (block17 >>> 18); + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 262143L) << 36) | (block18 >>> 28); + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 268435455L) << 26) | (block19 >>> 38); + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 274877906943L) << 16) | (block20 >>> 48); + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 281474976710655L) << 6) | (block21 >>> 58); + values[vi++] = (block21 >>> 4) & 18014398509481983L; + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 15L) << 50) | (block22 >>> 14); + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 16383L) << 40) | (block23 >>> 24); + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 16777215L) << 30) | (block24 >>> 34); + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 17179869183L) << 20) | (block25 >>> 44); + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 17592186044415L) << 10) | (block26 >>> 54); + values[vi++] = block26 & 18014398509481983L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 10) | (values[vi] >>> 44); + blocks[bi++] = (values[vi++] << 20) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 6) | (values[vi] >>> 48); + blocks[bi++] = (values[vi++] << 16) | (values[vi] >>> 38); + blocks[bi++] = (values[vi++] << 26) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 2) | (values[vi] >>> 52); + blocks[bi++] = (values[vi++] << 12) | (values[vi] >>> 42); + blocks[bi++] = (values[vi++] << 22) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 8) | (values[vi] >>> 46); + blocks[bi++] = (values[vi++] << 18) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 4) | (values[vi] >>> 50); + blocks[bi++] = (values[vi++] << 14) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation55 extends BulkOperation { + + public int blocks() { + return 55; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 9; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 511L) << 46) | (block1 >>> 18); + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 262143L) << 37) | (block2 >>> 27); + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 134217727L) << 28) | (block3 >>> 36); + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 68719476735L) << 19) | (block4 >>> 45); + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 35184372088831L) << 10) | (block5 >>> 54); + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 18014398509481983L) << 1) | (block6 >>> 63); + values[vi++] = (block6 >>> 8) & 36028797018963967L; + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 255L) << 47) | (block7 >>> 17); + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 131071L) << 38) | (block8 >>> 26); + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 67108863L) << 29) | (block9 >>> 35); + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 34359738367L) << 20) | (block10 >>> 44); + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 17592186044415L) << 11) | (block11 >>> 53); + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 9007199254740991L) << 2) | (block12 >>> 62); + values[vi++] = (block12 >>> 7) & 36028797018963967L; + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 127L) << 48) | (block13 >>> 16); + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 65535L) << 39) | (block14 >>> 25); + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 33554431L) << 30) | (block15 >>> 34); + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 17179869183L) << 21) | (block16 >>> 43); + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 8796093022207L) << 12) | (block17 >>> 52); + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 4503599627370495L) << 3) | (block18 >>> 61); + values[vi++] = (block18 >>> 6) & 36028797018963967L; + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 63L) << 49) | (block19 >>> 15); + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 32767L) << 40) | (block20 >>> 24); + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 16777215L) << 31) | (block21 >>> 33); + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 8589934591L) << 22) | (block22 >>> 42); + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 4398046511103L) << 13) | (block23 >>> 51); + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 2251799813685247L) << 4) | (block24 >>> 60); + values[vi++] = (block24 >>> 5) & 36028797018963967L; + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 31L) << 50) | (block25 >>> 14); + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 16383L) << 41) | (block26 >>> 23); + final long block27 = blocks[bi++]; + values[vi++] = ((block26 & 8388607L) << 32) | (block27 >>> 32); + final long block28 = blocks[bi++]; + values[vi++] = ((block27 & 4294967295L) << 23) | (block28 >>> 41); + final long block29 = blocks[bi++]; + values[vi++] = ((block28 & 2199023255551L) << 14) | (block29 >>> 50); + final long block30 = blocks[bi++]; + values[vi++] = ((block29 & 1125899906842623L) << 5) | (block30 >>> 59); + values[vi++] = (block30 >>> 4) & 36028797018963967L; + final long block31 = blocks[bi++]; + values[vi++] = ((block30 & 15L) << 51) | (block31 >>> 13); + final long block32 = blocks[bi++]; + values[vi++] = ((block31 & 8191L) << 42) | (block32 >>> 22); + final long block33 = blocks[bi++]; + values[vi++] = ((block32 & 4194303L) << 33) | (block33 >>> 31); + final long block34 = blocks[bi++]; + values[vi++] = ((block33 & 2147483647L) << 24) | (block34 >>> 40); + final long block35 = blocks[bi++]; + values[vi++] = ((block34 & 1099511627775L) << 15) | (block35 >>> 49); + final long block36 = blocks[bi++]; + values[vi++] = ((block35 & 562949953421311L) << 6) | (block36 >>> 58); + values[vi++] = (block36 >>> 3) & 36028797018963967L; + final long block37 = blocks[bi++]; + values[vi++] = ((block36 & 7L) << 52) | (block37 >>> 12); + final long block38 = blocks[bi++]; + values[vi++] = ((block37 & 4095L) << 43) | (block38 >>> 21); + final long block39 = blocks[bi++]; + values[vi++] = ((block38 & 2097151L) << 34) | (block39 >>> 30); + final long block40 = blocks[bi++]; + values[vi++] = ((block39 & 1073741823L) << 25) | (block40 >>> 39); + final long block41 = blocks[bi++]; + values[vi++] = ((block40 & 549755813887L) << 16) | (block41 >>> 48); + final long block42 = blocks[bi++]; + values[vi++] = ((block41 & 281474976710655L) << 7) | (block42 >>> 57); + values[vi++] = (block42 >>> 2) & 36028797018963967L; + final long block43 = blocks[bi++]; + values[vi++] = ((block42 & 3L) << 53) | (block43 >>> 11); + final long block44 = blocks[bi++]; + values[vi++] = ((block43 & 2047L) << 44) | (block44 >>> 20); + final long block45 = blocks[bi++]; + values[vi++] = ((block44 & 1048575L) << 35) | (block45 >>> 29); + final long block46 = blocks[bi++]; + values[vi++] = ((block45 & 536870911L) << 26) | (block46 >>> 38); + final long block47 = blocks[bi++]; + values[vi++] = ((block46 & 274877906943L) << 17) | (block47 >>> 47); + final long block48 = blocks[bi++]; + values[vi++] = ((block47 & 140737488355327L) << 8) | (block48 >>> 56); + values[vi++] = (block48 >>> 1) & 36028797018963967L; + final long block49 = blocks[bi++]; + values[vi++] = ((block48 & 1L) << 54) | (block49 >>> 10); + final long block50 = blocks[bi++]; + values[vi++] = ((block49 & 1023L) << 45) | (block50 >>> 19); + final long block51 = blocks[bi++]; + values[vi++] = ((block50 & 524287L) << 36) | (block51 >>> 28); + final long block52 = blocks[bi++]; + values[vi++] = ((block51 & 268435455L) << 27) | (block52 >>> 37); + final long block53 = blocks[bi++]; + values[vi++] = ((block52 & 137438953471L) << 18) | (block53 >>> 46); + final long block54 = blocks[bi++]; + values[vi++] = ((block53 & 70368744177663L) << 9) | (block54 >>> 55); + values[vi++] = block54 & 36028797018963967L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 9) | (values[vi] >>> 46); + blocks[bi++] = (values[vi++] << 18) | (values[vi] >>> 37); + blocks[bi++] = (values[vi++] << 27) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 8) | (values[vi] >>> 47); + blocks[bi++] = (values[vi++] << 17) | (values[vi] >>> 38); + blocks[bi++] = (values[vi++] << 26) | (values[vi] >>> 29); + blocks[bi++] = (values[vi++] << 35) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 7) | (values[vi] >>> 48); + blocks[bi++] = (values[vi++] << 16) | (values[vi] >>> 39); + blocks[bi++] = (values[vi++] << 25) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 21); + blocks[bi++] = (values[vi++] << 43) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 6) | (values[vi] >>> 49); + blocks[bi++] = (values[vi++] << 15) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 31); + blocks[bi++] = (values[vi++] << 33) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 5) | (values[vi] >>> 50); + blocks[bi++] = (values[vi++] << 14) | (values[vi] >>> 41); + blocks[bi++] = (values[vi++] << 23) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 23); + blocks[bi++] = (values[vi++] << 41) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 4) | (values[vi] >>> 51); + blocks[bi++] = (values[vi++] << 13) | (values[vi] >>> 42); + blocks[bi++] = (values[vi++] << 22) | (values[vi] >>> 33); + blocks[bi++] = (values[vi++] << 31) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 3) | (values[vi] >>> 52); + blocks[bi++] = (values[vi++] << 12) | (values[vi] >>> 43); + blocks[bi++] = (values[vi++] << 21) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 25); + blocks[bi++] = (values[vi++] << 39) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi++] << 2) | (values[vi] >>> 53); + blocks[bi++] = (values[vi++] << 11) | (values[vi] >>> 44); + blocks[bi++] = (values[vi++] << 20) | (values[vi] >>> 35); + blocks[bi++] = (values[vi++] << 29) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi++] << 1) | (values[vi] >>> 54); + blocks[bi++] = (values[vi++] << 10) | (values[vi] >>> 45); + blocks[bi++] = (values[vi++] << 19) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 27); + blocks[bi++] = (values[vi++] << 37) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation56 extends BulkOperation { + + public int blocks() { + return 7; + } + + public int values() { + return 8; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 8; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 255L) << 48) | (block1 >>> 16); + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 65535L) << 40) | (block2 >>> 24); + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 16777215L) << 32) | (block3 >>> 32); + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 4294967295L) << 24) | (block4 >>> 40); + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 1099511627775L) << 16) | (block5 >>> 48); + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 281474976710655L) << 8) | (block6 >>> 56); + values[vi++] = block6 & 72057594037927935L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 8) | (values[vi] >>> 48); + blocks[bi++] = (values[vi++] << 16) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation57 extends BulkOperation { + + public int blocks() { + return 57; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 7; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 127L) << 50) | (block1 >>> 14); + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 16383L) << 43) | (block2 >>> 21); + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 2097151L) << 36) | (block3 >>> 28); + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 268435455L) << 29) | (block4 >>> 35); + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 34359738367L) << 22) | (block5 >>> 42); + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 4398046511103L) << 15) | (block6 >>> 49); + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 562949953421311L) << 8) | (block7 >>> 56); + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 72057594037927935L) << 1) | (block8 >>> 63); + values[vi++] = (block8 >>> 6) & 144115188075855871L; + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 63L) << 51) | (block9 >>> 13); + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 8191L) << 44) | (block10 >>> 20); + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 1048575L) << 37) | (block11 >>> 27); + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 134217727L) << 30) | (block12 >>> 34); + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 17179869183L) << 23) | (block13 >>> 41); + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 2199023255551L) << 16) | (block14 >>> 48); + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 281474976710655L) << 9) | (block15 >>> 55); + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 36028797018963967L) << 2) | (block16 >>> 62); + values[vi++] = (block16 >>> 5) & 144115188075855871L; + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 31L) << 52) | (block17 >>> 12); + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 4095L) << 45) | (block18 >>> 19); + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 524287L) << 38) | (block19 >>> 26); + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 67108863L) << 31) | (block20 >>> 33); + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 8589934591L) << 24) | (block21 >>> 40); + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 1099511627775L) << 17) | (block22 >>> 47); + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 140737488355327L) << 10) | (block23 >>> 54); + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 18014398509481983L) << 3) | (block24 >>> 61); + values[vi++] = (block24 >>> 4) & 144115188075855871L; + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 15L) << 53) | (block25 >>> 11); + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 2047L) << 46) | (block26 >>> 18); + final long block27 = blocks[bi++]; + values[vi++] = ((block26 & 262143L) << 39) | (block27 >>> 25); + final long block28 = blocks[bi++]; + values[vi++] = ((block27 & 33554431L) << 32) | (block28 >>> 32); + final long block29 = blocks[bi++]; + values[vi++] = ((block28 & 4294967295L) << 25) | (block29 >>> 39); + final long block30 = blocks[bi++]; + values[vi++] = ((block29 & 549755813887L) << 18) | (block30 >>> 46); + final long block31 = blocks[bi++]; + values[vi++] = ((block30 & 70368744177663L) << 11) | (block31 >>> 53); + final long block32 = blocks[bi++]; + values[vi++] = ((block31 & 9007199254740991L) << 4) | (block32 >>> 60); + values[vi++] = (block32 >>> 3) & 144115188075855871L; + final long block33 = blocks[bi++]; + values[vi++] = ((block32 & 7L) << 54) | (block33 >>> 10); + final long block34 = blocks[bi++]; + values[vi++] = ((block33 & 1023L) << 47) | (block34 >>> 17); + final long block35 = blocks[bi++]; + values[vi++] = ((block34 & 131071L) << 40) | (block35 >>> 24); + final long block36 = blocks[bi++]; + values[vi++] = ((block35 & 16777215L) << 33) | (block36 >>> 31); + final long block37 = blocks[bi++]; + values[vi++] = ((block36 & 2147483647L) << 26) | (block37 >>> 38); + final long block38 = blocks[bi++]; + values[vi++] = ((block37 & 274877906943L) << 19) | (block38 >>> 45); + final long block39 = blocks[bi++]; + values[vi++] = ((block38 & 35184372088831L) << 12) | (block39 >>> 52); + final long block40 = blocks[bi++]; + values[vi++] = ((block39 & 4503599627370495L) << 5) | (block40 >>> 59); + values[vi++] = (block40 >>> 2) & 144115188075855871L; + final long block41 = blocks[bi++]; + values[vi++] = ((block40 & 3L) << 55) | (block41 >>> 9); + final long block42 = blocks[bi++]; + values[vi++] = ((block41 & 511L) << 48) | (block42 >>> 16); + final long block43 = blocks[bi++]; + values[vi++] = ((block42 & 65535L) << 41) | (block43 >>> 23); + final long block44 = blocks[bi++]; + values[vi++] = ((block43 & 8388607L) << 34) | (block44 >>> 30); + final long block45 = blocks[bi++]; + values[vi++] = ((block44 & 1073741823L) << 27) | (block45 >>> 37); + final long block46 = blocks[bi++]; + values[vi++] = ((block45 & 137438953471L) << 20) | (block46 >>> 44); + final long block47 = blocks[bi++]; + values[vi++] = ((block46 & 17592186044415L) << 13) | (block47 >>> 51); + final long block48 = blocks[bi++]; + values[vi++] = ((block47 & 2251799813685247L) << 6) | (block48 >>> 58); + values[vi++] = (block48 >>> 1) & 144115188075855871L; + final long block49 = blocks[bi++]; + values[vi++] = ((block48 & 1L) << 56) | (block49 >>> 8); + final long block50 = blocks[bi++]; + values[vi++] = ((block49 & 255L) << 49) | (block50 >>> 15); + final long block51 = blocks[bi++]; + values[vi++] = ((block50 & 32767L) << 42) | (block51 >>> 22); + final long block52 = blocks[bi++]; + values[vi++] = ((block51 & 4194303L) << 35) | (block52 >>> 29); + final long block53 = blocks[bi++]; + values[vi++] = ((block52 & 536870911L) << 28) | (block53 >>> 36); + final long block54 = blocks[bi++]; + values[vi++] = ((block53 & 68719476735L) << 21) | (block54 >>> 43); + final long block55 = blocks[bi++]; + values[vi++] = ((block54 & 8796093022207L) << 14) | (block55 >>> 50); + final long block56 = blocks[bi++]; + values[vi++] = ((block55 & 1125899906842623L) << 7) | (block56 >>> 57); + values[vi++] = block56 & 144115188075855871L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 7) | (values[vi] >>> 50); + blocks[bi++] = (values[vi++] << 14) | (values[vi] >>> 43); + blocks[bi++] = (values[vi++] << 21) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 29); + blocks[bi++] = (values[vi++] << 35) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 6) | (values[vi] >>> 51); + blocks[bi++] = (values[vi++] << 13) | (values[vi] >>> 44); + blocks[bi++] = (values[vi++] << 20) | (values[vi] >>> 37); + blocks[bi++] = (values[vi++] << 27) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 23); + blocks[bi++] = (values[vi++] << 41) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 5) | (values[vi] >>> 52); + blocks[bi++] = (values[vi++] << 12) | (values[vi] >>> 45); + blocks[bi++] = (values[vi++] << 19) | (values[vi] >>> 38); + blocks[bi++] = (values[vi++] << 26) | (values[vi] >>> 31); + blocks[bi++] = (values[vi++] << 33) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 4) | (values[vi] >>> 53); + blocks[bi++] = (values[vi++] << 11) | (values[vi] >>> 46); + blocks[bi++] = (values[vi++] << 18) | (values[vi] >>> 39); + blocks[bi++] = (values[vi++] << 25) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 25); + blocks[bi++] = (values[vi++] << 39) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 3) | (values[vi] >>> 54); + blocks[bi++] = (values[vi++] << 10) | (values[vi] >>> 47); + blocks[bi++] = (values[vi++] << 17) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 33); + blocks[bi++] = (values[vi++] << 31) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi++] << 2) | (values[vi] >>> 55); + blocks[bi++] = (values[vi++] << 9) | (values[vi] >>> 48); + blocks[bi++] = (values[vi++] << 16) | (values[vi] >>> 41); + blocks[bi++] = (values[vi++] << 23) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 27); + blocks[bi++] = (values[vi++] << 37) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi++] << 1) | (values[vi] >>> 56); + blocks[bi++] = (values[vi++] << 8) | (values[vi] >>> 49); + blocks[bi++] = (values[vi++] << 15) | (values[vi] >>> 42); + blocks[bi++] = (values[vi++] << 22) | (values[vi] >>> 35); + blocks[bi++] = (values[vi++] << 29) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 21); + blocks[bi++] = (values[vi++] << 43) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation58 extends BulkOperation { + + public int blocks() { + return 29; + } + + public int values() { + return 32; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 6; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 63L) << 52) | (block1 >>> 12); + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 4095L) << 46) | (block2 >>> 18); + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 262143L) << 40) | (block3 >>> 24); + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 16777215L) << 34) | (block4 >>> 30); + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 1073741823L) << 28) | (block5 >>> 36); + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 68719476735L) << 22) | (block6 >>> 42); + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 4398046511103L) << 16) | (block7 >>> 48); + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 281474976710655L) << 10) | (block8 >>> 54); + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 18014398509481983L) << 4) | (block9 >>> 60); + values[vi++] = (block9 >>> 2) & 288230376151711743L; + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 3L) << 56) | (block10 >>> 8); + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 255L) << 50) | (block11 >>> 14); + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 16383L) << 44) | (block12 >>> 20); + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 1048575L) << 38) | (block13 >>> 26); + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 67108863L) << 32) | (block14 >>> 32); + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 4294967295L) << 26) | (block15 >>> 38); + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 274877906943L) << 20) | (block16 >>> 44); + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 17592186044415L) << 14) | (block17 >>> 50); + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 1125899906842623L) << 8) | (block18 >>> 56); + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 72057594037927935L) << 2) | (block19 >>> 62); + values[vi++] = (block19 >>> 4) & 288230376151711743L; + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 15L) << 54) | (block20 >>> 10); + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 1023L) << 48) | (block21 >>> 16); + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 65535L) << 42) | (block22 >>> 22); + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 4194303L) << 36) | (block23 >>> 28); + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 268435455L) << 30) | (block24 >>> 34); + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 17179869183L) << 24) | (block25 >>> 40); + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 1099511627775L) << 18) | (block26 >>> 46); + final long block27 = blocks[bi++]; + values[vi++] = ((block26 & 70368744177663L) << 12) | (block27 >>> 52); + final long block28 = blocks[bi++]; + values[vi++] = ((block27 & 4503599627370495L) << 6) | (block28 >>> 58); + values[vi++] = block28 & 288230376151711743L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 6) | (values[vi] >>> 52); + blocks[bi++] = (values[vi++] << 12) | (values[vi] >>> 46); + blocks[bi++] = (values[vi++] << 18) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 2) | (values[vi] >>> 56); + blocks[bi++] = (values[vi++] << 8) | (values[vi] >>> 50); + blocks[bi++] = (values[vi++] << 14) | (values[vi] >>> 44); + blocks[bi++] = (values[vi++] << 20) | (values[vi] >>> 38); + blocks[bi++] = (values[vi++] << 26) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 4) | (values[vi] >>> 54); + blocks[bi++] = (values[vi++] << 10) | (values[vi] >>> 48); + blocks[bi++] = (values[vi++] << 16) | (values[vi] >>> 42); + blocks[bi++] = (values[vi++] << 22) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation59 extends BulkOperation { + + public int blocks() { + return 59; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 5; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 31L) << 54) | (block1 >>> 10); + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 1023L) << 49) | (block2 >>> 15); + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 32767L) << 44) | (block3 >>> 20); + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 1048575L) << 39) | (block4 >>> 25); + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 33554431L) << 34) | (block5 >>> 30); + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 1073741823L) << 29) | (block6 >>> 35); + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 34359738367L) << 24) | (block7 >>> 40); + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 1099511627775L) << 19) | (block8 >>> 45); + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 35184372088831L) << 14) | (block9 >>> 50); + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 1125899906842623L) << 9) | (block10 >>> 55); + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 36028797018963967L) << 4) | (block11 >>> 60); + values[vi++] = (block11 >>> 1) & 576460752303423487L; + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 1L) << 58) | (block12 >>> 6); + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 63L) << 53) | (block13 >>> 11); + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 2047L) << 48) | (block14 >>> 16); + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 65535L) << 43) | (block15 >>> 21); + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 2097151L) << 38) | (block16 >>> 26); + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 67108863L) << 33) | (block17 >>> 31); + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 2147483647L) << 28) | (block18 >>> 36); + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 68719476735L) << 23) | (block19 >>> 41); + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 2199023255551L) << 18) | (block20 >>> 46); + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 70368744177663L) << 13) | (block21 >>> 51); + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 2251799813685247L) << 8) | (block22 >>> 56); + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 72057594037927935L) << 3) | (block23 >>> 61); + values[vi++] = (block23 >>> 2) & 576460752303423487L; + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 3L) << 57) | (block24 >>> 7); + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 127L) << 52) | (block25 >>> 12); + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 4095L) << 47) | (block26 >>> 17); + final long block27 = blocks[bi++]; + values[vi++] = ((block26 & 131071L) << 42) | (block27 >>> 22); + final long block28 = blocks[bi++]; + values[vi++] = ((block27 & 4194303L) << 37) | (block28 >>> 27); + final long block29 = blocks[bi++]; + values[vi++] = ((block28 & 134217727L) << 32) | (block29 >>> 32); + final long block30 = blocks[bi++]; + values[vi++] = ((block29 & 4294967295L) << 27) | (block30 >>> 37); + final long block31 = blocks[bi++]; + values[vi++] = ((block30 & 137438953471L) << 22) | (block31 >>> 42); + final long block32 = blocks[bi++]; + values[vi++] = ((block31 & 4398046511103L) << 17) | (block32 >>> 47); + final long block33 = blocks[bi++]; + values[vi++] = ((block32 & 140737488355327L) << 12) | (block33 >>> 52); + final long block34 = blocks[bi++]; + values[vi++] = ((block33 & 4503599627370495L) << 7) | (block34 >>> 57); + final long block35 = blocks[bi++]; + values[vi++] = ((block34 & 144115188075855871L) << 2) | (block35 >>> 62); + values[vi++] = (block35 >>> 3) & 576460752303423487L; + final long block36 = blocks[bi++]; + values[vi++] = ((block35 & 7L) << 56) | (block36 >>> 8); + final long block37 = blocks[bi++]; + values[vi++] = ((block36 & 255L) << 51) | (block37 >>> 13); + final long block38 = blocks[bi++]; + values[vi++] = ((block37 & 8191L) << 46) | (block38 >>> 18); + final long block39 = blocks[bi++]; + values[vi++] = ((block38 & 262143L) << 41) | (block39 >>> 23); + final long block40 = blocks[bi++]; + values[vi++] = ((block39 & 8388607L) << 36) | (block40 >>> 28); + final long block41 = blocks[bi++]; + values[vi++] = ((block40 & 268435455L) << 31) | (block41 >>> 33); + final long block42 = blocks[bi++]; + values[vi++] = ((block41 & 8589934591L) << 26) | (block42 >>> 38); + final long block43 = blocks[bi++]; + values[vi++] = ((block42 & 274877906943L) << 21) | (block43 >>> 43); + final long block44 = blocks[bi++]; + values[vi++] = ((block43 & 8796093022207L) << 16) | (block44 >>> 48); + final long block45 = blocks[bi++]; + values[vi++] = ((block44 & 281474976710655L) << 11) | (block45 >>> 53); + final long block46 = blocks[bi++]; + values[vi++] = ((block45 & 9007199254740991L) << 6) | (block46 >>> 58); + final long block47 = blocks[bi++]; + values[vi++] = ((block46 & 288230376151711743L) << 1) | (block47 >>> 63); + values[vi++] = (block47 >>> 4) & 576460752303423487L; + final long block48 = blocks[bi++]; + values[vi++] = ((block47 & 15L) << 55) | (block48 >>> 9); + final long block49 = blocks[bi++]; + values[vi++] = ((block48 & 511L) << 50) | (block49 >>> 14); + final long block50 = blocks[bi++]; + values[vi++] = ((block49 & 16383L) << 45) | (block50 >>> 19); + final long block51 = blocks[bi++]; + values[vi++] = ((block50 & 524287L) << 40) | (block51 >>> 24); + final long block52 = blocks[bi++]; + values[vi++] = ((block51 & 16777215L) << 35) | (block52 >>> 29); + final long block53 = blocks[bi++]; + values[vi++] = ((block52 & 536870911L) << 30) | (block53 >>> 34); + final long block54 = blocks[bi++]; + values[vi++] = ((block53 & 17179869183L) << 25) | (block54 >>> 39); + final long block55 = blocks[bi++]; + values[vi++] = ((block54 & 549755813887L) << 20) | (block55 >>> 44); + final long block56 = blocks[bi++]; + values[vi++] = ((block55 & 17592186044415L) << 15) | (block56 >>> 49); + final long block57 = blocks[bi++]; + values[vi++] = ((block56 & 562949953421311L) << 10) | (block57 >>> 54); + final long block58 = blocks[bi++]; + values[vi++] = ((block57 & 18014398509481983L) << 5) | (block58 >>> 59); + values[vi++] = block58 & 576460752303423487L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 5) | (values[vi] >>> 54); + blocks[bi++] = (values[vi++] << 10) | (values[vi] >>> 49); + blocks[bi++] = (values[vi++] << 15) | (values[vi] >>> 44); + blocks[bi++] = (values[vi++] << 20) | (values[vi] >>> 39); + blocks[bi++] = (values[vi++] << 25) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 29); + blocks[bi++] = (values[vi++] << 35) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi++] << 1) | (values[vi] >>> 58); + blocks[bi++] = (values[vi++] << 6) | (values[vi] >>> 53); + blocks[bi++] = (values[vi++] << 11) | (values[vi] >>> 48); + blocks[bi++] = (values[vi++] << 16) | (values[vi] >>> 43); + blocks[bi++] = (values[vi++] << 21) | (values[vi] >>> 38); + blocks[bi++] = (values[vi++] << 26) | (values[vi] >>> 33); + blocks[bi++] = (values[vi++] << 31) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 23); + blocks[bi++] = (values[vi++] << 41) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi++] << 2) | (values[vi] >>> 57); + blocks[bi++] = (values[vi++] << 7) | (values[vi] >>> 52); + blocks[bi++] = (values[vi++] << 12) | (values[vi] >>> 47); + blocks[bi++] = (values[vi++] << 17) | (values[vi] >>> 42); + blocks[bi++] = (values[vi++] << 22) | (values[vi] >>> 37); + blocks[bi++] = (values[vi++] << 27) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 27); + blocks[bi++] = (values[vi++] << 37) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 3) | (values[vi] >>> 56); + blocks[bi++] = (values[vi++] << 8) | (values[vi] >>> 51); + blocks[bi++] = (values[vi++] << 13) | (values[vi] >>> 46); + blocks[bi++] = (values[vi++] << 18) | (values[vi] >>> 41); + blocks[bi++] = (values[vi++] << 23) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 31); + blocks[bi++] = (values[vi++] << 33) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi] >>> 21); + blocks[bi++] = (values[vi++] << 43) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 4) | (values[vi] >>> 55); + blocks[bi++] = (values[vi++] << 9) | (values[vi] >>> 50); + blocks[bi++] = (values[vi++] << 14) | (values[vi] >>> 45); + blocks[bi++] = (values[vi++] << 19) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 35); + blocks[bi++] = (values[vi++] << 29) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 25); + blocks[bi++] = (values[vi++] << 39) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation60 extends BulkOperation { + + public int blocks() { + return 15; + } + + public int values() { + return 16; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 4; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 15L) << 56) | (block1 >>> 8); + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 255L) << 52) | (block2 >>> 12); + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 4095L) << 48) | (block3 >>> 16); + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 65535L) << 44) | (block4 >>> 20); + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 1048575L) << 40) | (block5 >>> 24); + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 16777215L) << 36) | (block6 >>> 28); + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 268435455L) << 32) | (block7 >>> 32); + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 4294967295L) << 28) | (block8 >>> 36); + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 68719476735L) << 24) | (block9 >>> 40); + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 1099511627775L) << 20) | (block10 >>> 44); + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 17592186044415L) << 16) | (block11 >>> 48); + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 281474976710655L) << 12) | (block12 >>> 52); + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 4503599627370495L) << 8) | (block13 >>> 56); + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 72057594037927935L) << 4) | (block14 >>> 60); + values[vi++] = block14 & 1152921504606846975L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 4) | (values[vi] >>> 56); + blocks[bi++] = (values[vi++] << 8) | (values[vi] >>> 52); + blocks[bi++] = (values[vi++] << 12) | (values[vi] >>> 48); + blocks[bi++] = (values[vi++] << 16) | (values[vi] >>> 44); + blocks[bi++] = (values[vi++] << 20) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation61 extends BulkOperation { + + public int blocks() { + return 61; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 3; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 7L) << 58) | (block1 >>> 6); + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 63L) << 55) | (block2 >>> 9); + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 511L) << 52) | (block3 >>> 12); + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 4095L) << 49) | (block4 >>> 15); + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 32767L) << 46) | (block5 >>> 18); + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 262143L) << 43) | (block6 >>> 21); + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 2097151L) << 40) | (block7 >>> 24); + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 16777215L) << 37) | (block8 >>> 27); + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 134217727L) << 34) | (block9 >>> 30); + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 1073741823L) << 31) | (block10 >>> 33); + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 8589934591L) << 28) | (block11 >>> 36); + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 68719476735L) << 25) | (block12 >>> 39); + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 549755813887L) << 22) | (block13 >>> 42); + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 4398046511103L) << 19) | (block14 >>> 45); + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 35184372088831L) << 16) | (block15 >>> 48); + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 281474976710655L) << 13) | (block16 >>> 51); + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 2251799813685247L) << 10) | (block17 >>> 54); + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 18014398509481983L) << 7) | (block18 >>> 57); + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 144115188075855871L) << 4) | (block19 >>> 60); + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 1152921504606846975L) << 1) | (block20 >>> 63); + values[vi++] = (block20 >>> 2) & 2305843009213693951L; + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 3L) << 59) | (block21 >>> 5); + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 31L) << 56) | (block22 >>> 8); + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 255L) << 53) | (block23 >>> 11); + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 2047L) << 50) | (block24 >>> 14); + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 16383L) << 47) | (block25 >>> 17); + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 131071L) << 44) | (block26 >>> 20); + final long block27 = blocks[bi++]; + values[vi++] = ((block26 & 1048575L) << 41) | (block27 >>> 23); + final long block28 = blocks[bi++]; + values[vi++] = ((block27 & 8388607L) << 38) | (block28 >>> 26); + final long block29 = blocks[bi++]; + values[vi++] = ((block28 & 67108863L) << 35) | (block29 >>> 29); + final long block30 = blocks[bi++]; + values[vi++] = ((block29 & 536870911L) << 32) | (block30 >>> 32); + final long block31 = blocks[bi++]; + values[vi++] = ((block30 & 4294967295L) << 29) | (block31 >>> 35); + final long block32 = blocks[bi++]; + values[vi++] = ((block31 & 34359738367L) << 26) | (block32 >>> 38); + final long block33 = blocks[bi++]; + values[vi++] = ((block32 & 274877906943L) << 23) | (block33 >>> 41); + final long block34 = blocks[bi++]; + values[vi++] = ((block33 & 2199023255551L) << 20) | (block34 >>> 44); + final long block35 = blocks[bi++]; + values[vi++] = ((block34 & 17592186044415L) << 17) | (block35 >>> 47); + final long block36 = blocks[bi++]; + values[vi++] = ((block35 & 140737488355327L) << 14) | (block36 >>> 50); + final long block37 = blocks[bi++]; + values[vi++] = ((block36 & 1125899906842623L) << 11) | (block37 >>> 53); + final long block38 = blocks[bi++]; + values[vi++] = ((block37 & 9007199254740991L) << 8) | (block38 >>> 56); + final long block39 = blocks[bi++]; + values[vi++] = ((block38 & 72057594037927935L) << 5) | (block39 >>> 59); + final long block40 = blocks[bi++]; + values[vi++] = ((block39 & 576460752303423487L) << 2) | (block40 >>> 62); + values[vi++] = (block40 >>> 1) & 2305843009213693951L; + final long block41 = blocks[bi++]; + values[vi++] = ((block40 & 1L) << 60) | (block41 >>> 4); + final long block42 = blocks[bi++]; + values[vi++] = ((block41 & 15L) << 57) | (block42 >>> 7); + final long block43 = blocks[bi++]; + values[vi++] = ((block42 & 127L) << 54) | (block43 >>> 10); + final long block44 = blocks[bi++]; + values[vi++] = ((block43 & 1023L) << 51) | (block44 >>> 13); + final long block45 = blocks[bi++]; + values[vi++] = ((block44 & 8191L) << 48) | (block45 >>> 16); + final long block46 = blocks[bi++]; + values[vi++] = ((block45 & 65535L) << 45) | (block46 >>> 19); + final long block47 = blocks[bi++]; + values[vi++] = ((block46 & 524287L) << 42) | (block47 >>> 22); + final long block48 = blocks[bi++]; + values[vi++] = ((block47 & 4194303L) << 39) | (block48 >>> 25); + final long block49 = blocks[bi++]; + values[vi++] = ((block48 & 33554431L) << 36) | (block49 >>> 28); + final long block50 = blocks[bi++]; + values[vi++] = ((block49 & 268435455L) << 33) | (block50 >>> 31); + final long block51 = blocks[bi++]; + values[vi++] = ((block50 & 2147483647L) << 30) | (block51 >>> 34); + final long block52 = blocks[bi++]; + values[vi++] = ((block51 & 17179869183L) << 27) | (block52 >>> 37); + final long block53 = blocks[bi++]; + values[vi++] = ((block52 & 137438953471L) << 24) | (block53 >>> 40); + final long block54 = blocks[bi++]; + values[vi++] = ((block53 & 1099511627775L) << 21) | (block54 >>> 43); + final long block55 = blocks[bi++]; + values[vi++] = ((block54 & 8796093022207L) << 18) | (block55 >>> 46); + final long block56 = blocks[bi++]; + values[vi++] = ((block55 & 70368744177663L) << 15) | (block56 >>> 49); + final long block57 = blocks[bi++]; + values[vi++] = ((block56 & 562949953421311L) << 12) | (block57 >>> 52); + final long block58 = blocks[bi++]; + values[vi++] = ((block57 & 4503599627370495L) << 9) | (block58 >>> 55); + final long block59 = blocks[bi++]; + values[vi++] = ((block58 & 36028797018963967L) << 6) | (block59 >>> 58); + final long block60 = blocks[bi++]; + values[vi++] = ((block59 & 288230376151711743L) << 3) | (block60 >>> 61); + values[vi++] = block60 & 2305843009213693951L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 3) | (values[vi] >>> 58); + blocks[bi++] = (values[vi++] << 6) | (values[vi] >>> 55); + blocks[bi++] = (values[vi++] << 9) | (values[vi] >>> 52); + blocks[bi++] = (values[vi++] << 12) | (values[vi] >>> 49); + blocks[bi++] = (values[vi++] << 15) | (values[vi] >>> 46); + blocks[bi++] = (values[vi++] << 18) | (values[vi] >>> 43); + blocks[bi++] = (values[vi++] << 21) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 37); + blocks[bi++] = (values[vi++] << 27) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 31); + blocks[bi++] = (values[vi++] << 33) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 25); + blocks[bi++] = (values[vi++] << 39) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | (values[vi++] << 2) | (values[vi] >>> 59); + blocks[bi++] = (values[vi++] << 5) | (values[vi] >>> 56); + blocks[bi++] = (values[vi++] << 8) | (values[vi] >>> 53); + blocks[bi++] = (values[vi++] << 11) | (values[vi] >>> 50); + blocks[bi++] = (values[vi++] << 14) | (values[vi] >>> 47); + blocks[bi++] = (values[vi++] << 17) | (values[vi] >>> 44); + blocks[bi++] = (values[vi++] << 20) | (values[vi] >>> 41); + blocks[bi++] = (values[vi++] << 23) | (values[vi] >>> 38); + blocks[bi++] = (values[vi++] << 26) | (values[vi] >>> 35); + blocks[bi++] = (values[vi++] << 29) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 29); + blocks[bi++] = (values[vi++] << 35) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi] >>> 23); + blocks[bi++] = (values[vi++] << 41) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi++] << 1) | (values[vi] >>> 60); + blocks[bi++] = (values[vi++] << 4) | (values[vi] >>> 57); + blocks[bi++] = (values[vi++] << 7) | (values[vi] >>> 54); + blocks[bi++] = (values[vi++] << 10) | (values[vi] >>> 51); + blocks[bi++] = (values[vi++] << 13) | (values[vi] >>> 48); + blocks[bi++] = (values[vi++] << 16) | (values[vi] >>> 45); + blocks[bi++] = (values[vi++] << 19) | (values[vi] >>> 42); + blocks[bi++] = (values[vi++] << 22) | (values[vi] >>> 39); + blocks[bi++] = (values[vi++] << 25) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 33); + blocks[bi++] = (values[vi++] << 31) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 27); + blocks[bi++] = (values[vi++] << 37) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 21); + blocks[bi++] = (values[vi++] << 43) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation62 extends BulkOperation { + + public int blocks() { + return 31; + } + + public int values() { + return 32; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 2; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 3L) << 60) | (block1 >>> 4); + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 15L) << 58) | (block2 >>> 6); + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 63L) << 56) | (block3 >>> 8); + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 255L) << 54) | (block4 >>> 10); + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 1023L) << 52) | (block5 >>> 12); + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 4095L) << 50) | (block6 >>> 14); + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 16383L) << 48) | (block7 >>> 16); + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 65535L) << 46) | (block8 >>> 18); + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 262143L) << 44) | (block9 >>> 20); + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 1048575L) << 42) | (block10 >>> 22); + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 4194303L) << 40) | (block11 >>> 24); + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 16777215L) << 38) | (block12 >>> 26); + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 67108863L) << 36) | (block13 >>> 28); + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 268435455L) << 34) | (block14 >>> 30); + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 1073741823L) << 32) | (block15 >>> 32); + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 4294967295L) << 30) | (block16 >>> 34); + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 17179869183L) << 28) | (block17 >>> 36); + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 68719476735L) << 26) | (block18 >>> 38); + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 274877906943L) << 24) | (block19 >>> 40); + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 1099511627775L) << 22) | (block20 >>> 42); + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 4398046511103L) << 20) | (block21 >>> 44); + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 17592186044415L) << 18) | (block22 >>> 46); + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 70368744177663L) << 16) | (block23 >>> 48); + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 281474976710655L) << 14) | (block24 >>> 50); + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 1125899906842623L) << 12) | (block25 >>> 52); + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 4503599627370495L) << 10) | (block26 >>> 54); + final long block27 = blocks[bi++]; + values[vi++] = ((block26 & 18014398509481983L) << 8) | (block27 >>> 56); + final long block28 = blocks[bi++]; + values[vi++] = ((block27 & 72057594037927935L) << 6) | (block28 >>> 58); + final long block29 = blocks[bi++]; + values[vi++] = ((block28 & 288230376151711743L) << 4) | (block29 >>> 60); + final long block30 = blocks[bi++]; + values[vi++] = ((block29 & 1152921504606846975L) << 2) | (block30 >>> 62); + values[vi++] = block30 & 4611686018427387903L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 2) | (values[vi] >>> 60); + blocks[bi++] = (values[vi++] << 4) | (values[vi] >>> 58); + blocks[bi++] = (values[vi++] << 6) | (values[vi] >>> 56); + blocks[bi++] = (values[vi++] << 8) | (values[vi] >>> 54); + blocks[bi++] = (values[vi++] << 10) | (values[vi] >>> 52); + blocks[bi++] = (values[vi++] << 12) | (values[vi] >>> 50); + blocks[bi++] = (values[vi++] << 14) | (values[vi] >>> 48); + blocks[bi++] = (values[vi++] << 16) | (values[vi] >>> 46); + blocks[bi++] = (values[vi++] << 18) | (values[vi] >>> 44); + blocks[bi++] = (values[vi++] << 20) | (values[vi] >>> 42); + blocks[bi++] = (values[vi++] << 22) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 38); + blocks[bi++] = (values[vi++] << 26) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation63 extends BulkOperation { + + public int blocks() { + return 63; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block0 = blocks[bi++]; + values[vi++] = block0 >>> 1; + final long block1 = blocks[bi++]; + values[vi++] = ((block0 & 1L) << 62) | (block1 >>> 2); + final long block2 = blocks[bi++]; + values[vi++] = ((block1 & 3L) << 61) | (block2 >>> 3); + final long block3 = blocks[bi++]; + values[vi++] = ((block2 & 7L) << 60) | (block3 >>> 4); + final long block4 = blocks[bi++]; + values[vi++] = ((block3 & 15L) << 59) | (block4 >>> 5); + final long block5 = blocks[bi++]; + values[vi++] = ((block4 & 31L) << 58) | (block5 >>> 6); + final long block6 = blocks[bi++]; + values[vi++] = ((block5 & 63L) << 57) | (block6 >>> 7); + final long block7 = blocks[bi++]; + values[vi++] = ((block6 & 127L) << 56) | (block7 >>> 8); + final long block8 = blocks[bi++]; + values[vi++] = ((block7 & 255L) << 55) | (block8 >>> 9); + final long block9 = blocks[bi++]; + values[vi++] = ((block8 & 511L) << 54) | (block9 >>> 10); + final long block10 = blocks[bi++]; + values[vi++] = ((block9 & 1023L) << 53) | (block10 >>> 11); + final long block11 = blocks[bi++]; + values[vi++] = ((block10 & 2047L) << 52) | (block11 >>> 12); + final long block12 = blocks[bi++]; + values[vi++] = ((block11 & 4095L) << 51) | (block12 >>> 13); + final long block13 = blocks[bi++]; + values[vi++] = ((block12 & 8191L) << 50) | (block13 >>> 14); + final long block14 = blocks[bi++]; + values[vi++] = ((block13 & 16383L) << 49) | (block14 >>> 15); + final long block15 = blocks[bi++]; + values[vi++] = ((block14 & 32767L) << 48) | (block15 >>> 16); + final long block16 = blocks[bi++]; + values[vi++] = ((block15 & 65535L) << 47) | (block16 >>> 17); + final long block17 = blocks[bi++]; + values[vi++] = ((block16 & 131071L) << 46) | (block17 >>> 18); + final long block18 = blocks[bi++]; + values[vi++] = ((block17 & 262143L) << 45) | (block18 >>> 19); + final long block19 = blocks[bi++]; + values[vi++] = ((block18 & 524287L) << 44) | (block19 >>> 20); + final long block20 = blocks[bi++]; + values[vi++] = ((block19 & 1048575L) << 43) | (block20 >>> 21); + final long block21 = blocks[bi++]; + values[vi++] = ((block20 & 2097151L) << 42) | (block21 >>> 22); + final long block22 = blocks[bi++]; + values[vi++] = ((block21 & 4194303L) << 41) | (block22 >>> 23); + final long block23 = blocks[bi++]; + values[vi++] = ((block22 & 8388607L) << 40) | (block23 >>> 24); + final long block24 = blocks[bi++]; + values[vi++] = ((block23 & 16777215L) << 39) | (block24 >>> 25); + final long block25 = blocks[bi++]; + values[vi++] = ((block24 & 33554431L) << 38) | (block25 >>> 26); + final long block26 = blocks[bi++]; + values[vi++] = ((block25 & 67108863L) << 37) | (block26 >>> 27); + final long block27 = blocks[bi++]; + values[vi++] = ((block26 & 134217727L) << 36) | (block27 >>> 28); + final long block28 = blocks[bi++]; + values[vi++] = ((block27 & 268435455L) << 35) | (block28 >>> 29); + final long block29 = blocks[bi++]; + values[vi++] = ((block28 & 536870911L) << 34) | (block29 >>> 30); + final long block30 = blocks[bi++]; + values[vi++] = ((block29 & 1073741823L) << 33) | (block30 >>> 31); + final long block31 = blocks[bi++]; + values[vi++] = ((block30 & 2147483647L) << 32) | (block31 >>> 32); + final long block32 = blocks[bi++]; + values[vi++] = ((block31 & 4294967295L) << 31) | (block32 >>> 33); + final long block33 = blocks[bi++]; + values[vi++] = ((block32 & 8589934591L) << 30) | (block33 >>> 34); + final long block34 = blocks[bi++]; + values[vi++] = ((block33 & 17179869183L) << 29) | (block34 >>> 35); + final long block35 = blocks[bi++]; + values[vi++] = ((block34 & 34359738367L) << 28) | (block35 >>> 36); + final long block36 = blocks[bi++]; + values[vi++] = ((block35 & 68719476735L) << 27) | (block36 >>> 37); + final long block37 = blocks[bi++]; + values[vi++] = ((block36 & 137438953471L) << 26) | (block37 >>> 38); + final long block38 = blocks[bi++]; + values[vi++] = ((block37 & 274877906943L) << 25) | (block38 >>> 39); + final long block39 = blocks[bi++]; + values[vi++] = ((block38 & 549755813887L) << 24) | (block39 >>> 40); + final long block40 = blocks[bi++]; + values[vi++] = ((block39 & 1099511627775L) << 23) | (block40 >>> 41); + final long block41 = blocks[bi++]; + values[vi++] = ((block40 & 2199023255551L) << 22) | (block41 >>> 42); + final long block42 = blocks[bi++]; + values[vi++] = ((block41 & 4398046511103L) << 21) | (block42 >>> 43); + final long block43 = blocks[bi++]; + values[vi++] = ((block42 & 8796093022207L) << 20) | (block43 >>> 44); + final long block44 = blocks[bi++]; + values[vi++] = ((block43 & 17592186044415L) << 19) | (block44 >>> 45); + final long block45 = blocks[bi++]; + values[vi++] = ((block44 & 35184372088831L) << 18) | (block45 >>> 46); + final long block46 = blocks[bi++]; + values[vi++] = ((block45 & 70368744177663L) << 17) | (block46 >>> 47); + final long block47 = blocks[bi++]; + values[vi++] = ((block46 & 140737488355327L) << 16) | (block47 >>> 48); + final long block48 = blocks[bi++]; + values[vi++] = ((block47 & 281474976710655L) << 15) | (block48 >>> 49); + final long block49 = blocks[bi++]; + values[vi++] = ((block48 & 562949953421311L) << 14) | (block49 >>> 50); + final long block50 = blocks[bi++]; + values[vi++] = ((block49 & 1125899906842623L) << 13) | (block50 >>> 51); + final long block51 = blocks[bi++]; + values[vi++] = ((block50 & 2251799813685247L) << 12) | (block51 >>> 52); + final long block52 = blocks[bi++]; + values[vi++] = ((block51 & 4503599627370495L) << 11) | (block52 >>> 53); + final long block53 = blocks[bi++]; + values[vi++] = ((block52 & 9007199254740991L) << 10) | (block53 >>> 54); + final long block54 = blocks[bi++]; + values[vi++] = ((block53 & 18014398509481983L) << 9) | (block54 >>> 55); + final long block55 = blocks[bi++]; + values[vi++] = ((block54 & 36028797018963967L) << 8) | (block55 >>> 56); + final long block56 = blocks[bi++]; + values[vi++] = ((block55 & 72057594037927935L) << 7) | (block56 >>> 57); + final long block57 = blocks[bi++]; + values[vi++] = ((block56 & 144115188075855871L) << 6) | (block57 >>> 58); + final long block58 = blocks[bi++]; + values[vi++] = ((block57 & 288230376151711743L) << 5) | (block58 >>> 59); + final long block59 = blocks[bi++]; + values[vi++] = ((block58 & 576460752303423487L) << 4) | (block59 >>> 60); + final long block60 = blocks[bi++]; + values[vi++] = ((block59 & 1152921504606846975L) << 3) | (block60 >>> 61); + final long block61 = blocks[bi++]; + values[vi++] = ((block60 & 2305843009213693951L) << 2) | (block61 >>> 62); + final long block62 = blocks[bi++]; + values[vi++] = ((block61 & 4611686018427387903L) << 1) | (block62 >>> 63); + values[vi++] = block62 & 9223372036854775807L; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = (values[vi++] << 1) | (values[vi] >>> 62); + blocks[bi++] = (values[vi++] << 2) | (values[vi] >>> 61); + blocks[bi++] = (values[vi++] << 3) | (values[vi] >>> 60); + blocks[bi++] = (values[vi++] << 4) | (values[vi] >>> 59); + blocks[bi++] = (values[vi++] << 5) | (values[vi] >>> 58); + blocks[bi++] = (values[vi++] << 6) | (values[vi] >>> 57); + blocks[bi++] = (values[vi++] << 7) | (values[vi] >>> 56); + blocks[bi++] = (values[vi++] << 8) | (values[vi] >>> 55); + blocks[bi++] = (values[vi++] << 9) | (values[vi] >>> 54); + blocks[bi++] = (values[vi++] << 10) | (values[vi] >>> 53); + blocks[bi++] = (values[vi++] << 11) | (values[vi] >>> 52); + blocks[bi++] = (values[vi++] << 12) | (values[vi] >>> 51); + blocks[bi++] = (values[vi++] << 13) | (values[vi] >>> 50); + blocks[bi++] = (values[vi++] << 14) | (values[vi] >>> 49); + blocks[bi++] = (values[vi++] << 15) | (values[vi] >>> 48); + blocks[bi++] = (values[vi++] << 16) | (values[vi] >>> 47); + blocks[bi++] = (values[vi++] << 17) | (values[vi] >>> 46); + blocks[bi++] = (values[vi++] << 18) | (values[vi] >>> 45); + blocks[bi++] = (values[vi++] << 19) | (values[vi] >>> 44); + blocks[bi++] = (values[vi++] << 20) | (values[vi] >>> 43); + blocks[bi++] = (values[vi++] << 21) | (values[vi] >>> 42); + blocks[bi++] = (values[vi++] << 22) | (values[vi] >>> 41); + blocks[bi++] = (values[vi++] << 23) | (values[vi] >>> 40); + blocks[bi++] = (values[vi++] << 24) | (values[vi] >>> 39); + blocks[bi++] = (values[vi++] << 25) | (values[vi] >>> 38); + blocks[bi++] = (values[vi++] << 26) | (values[vi] >>> 37); + blocks[bi++] = (values[vi++] << 27) | (values[vi] >>> 36); + blocks[bi++] = (values[vi++] << 28) | (values[vi] >>> 35); + blocks[bi++] = (values[vi++] << 29) | (values[vi] >>> 34); + blocks[bi++] = (values[vi++] << 30) | (values[vi] >>> 33); + blocks[bi++] = (values[vi++] << 31) | (values[vi] >>> 32); + blocks[bi++] = (values[vi++] << 32) | (values[vi] >>> 31); + blocks[bi++] = (values[vi++] << 33) | (values[vi] >>> 30); + blocks[bi++] = (values[vi++] << 34) | (values[vi] >>> 29); + blocks[bi++] = (values[vi++] << 35) | (values[vi] >>> 28); + blocks[bi++] = (values[vi++] << 36) | (values[vi] >>> 27); + blocks[bi++] = (values[vi++] << 37) | (values[vi] >>> 26); + blocks[bi++] = (values[vi++] << 38) | (values[vi] >>> 25); + blocks[bi++] = (values[vi++] << 39) | (values[vi] >>> 24); + blocks[bi++] = (values[vi++] << 40) | (values[vi] >>> 23); + blocks[bi++] = (values[vi++] << 41) | (values[vi] >>> 22); + blocks[bi++] = (values[vi++] << 42) | (values[vi] >>> 21); + blocks[bi++] = (values[vi++] << 43) | (values[vi] >>> 20); + blocks[bi++] = (values[vi++] << 44) | (values[vi] >>> 19); + blocks[bi++] = (values[vi++] << 45) | (values[vi] >>> 18); + blocks[bi++] = (values[vi++] << 46) | (values[vi] >>> 17); + blocks[bi++] = (values[vi++] << 47) | (values[vi] >>> 16); + blocks[bi++] = (values[vi++] << 48) | (values[vi] >>> 15); + blocks[bi++] = (values[vi++] << 49) | (values[vi] >>> 14); + blocks[bi++] = (values[vi++] << 50) | (values[vi] >>> 13); + blocks[bi++] = (values[vi++] << 51) | (values[vi] >>> 12); + blocks[bi++] = (values[vi++] << 52) | (values[vi] >>> 11); + blocks[bi++] = (values[vi++] << 53) | (values[vi] >>> 10); + blocks[bi++] = (values[vi++] << 54) | (values[vi] >>> 9); + blocks[bi++] = (values[vi++] << 55) | (values[vi] >>> 8); + blocks[bi++] = (values[vi++] << 56) | (values[vi] >>> 7); + blocks[bi++] = (values[vi++] << 57) | (values[vi] >>> 6); + blocks[bi++] = (values[vi++] << 58) | (values[vi] >>> 5); + blocks[bi++] = (values[vi++] << 59) | (values[vi] >>> 4); + blocks[bi++] = (values[vi++] << 60) | (values[vi] >>> 3); + blocks[bi++] = (values[vi++] << 61) | (values[vi] >>> 2); + blocks[bi++] = (values[vi++] << 62) | (values[vi] >>> 1); + blocks[bi++] = (values[vi++] << 63) | values[vi++]; + } + } + } + + static final class Packed64BulkOperation64 extends BulkOperation { + + public int blocks() { + return 1; + } + + public int values() { + return 1; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + System.arraycopy(blocks, bi, values, vi, iterations); + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + System.arraycopy(values, bi, blocks, vi, iterations); + } + } + + static final class Packed64SingleBlockBulkOperation1 extends BulkOperation { + + public int blocks() { + return 1; + } + + public int values() { + return 64; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block = blocks[bi++]; + values[vi++] = block & 1L; + values[vi++] = (block >>> 1) & 1L; + values[vi++] = (block >>> 2) & 1L; + values[vi++] = (block >>> 3) & 1L; + values[vi++] = (block >>> 4) & 1L; + values[vi++] = (block >>> 5) & 1L; + values[vi++] = (block >>> 6) & 1L; + values[vi++] = (block >>> 7) & 1L; + values[vi++] = (block >>> 8) & 1L; + values[vi++] = (block >>> 9) & 1L; + values[vi++] = (block >>> 10) & 1L; + values[vi++] = (block >>> 11) & 1L; + values[vi++] = (block >>> 12) & 1L; + values[vi++] = (block >>> 13) & 1L; + values[vi++] = (block >>> 14) & 1L; + values[vi++] = (block >>> 15) & 1L; + values[vi++] = (block >>> 16) & 1L; + values[vi++] = (block >>> 17) & 1L; + values[vi++] = (block >>> 18) & 1L; + values[vi++] = (block >>> 19) & 1L; + values[vi++] = (block >>> 20) & 1L; + values[vi++] = (block >>> 21) & 1L; + values[vi++] = (block >>> 22) & 1L; + values[vi++] = (block >>> 23) & 1L; + values[vi++] = (block >>> 24) & 1L; + values[vi++] = (block >>> 25) & 1L; + values[vi++] = (block >>> 26) & 1L; + values[vi++] = (block >>> 27) & 1L; + values[vi++] = (block >>> 28) & 1L; + values[vi++] = (block >>> 29) & 1L; + values[vi++] = (block >>> 30) & 1L; + values[vi++] = (block >>> 31) & 1L; + values[vi++] = (block >>> 32) & 1L; + values[vi++] = (block >>> 33) & 1L; + values[vi++] = (block >>> 34) & 1L; + values[vi++] = (block >>> 35) & 1L; + values[vi++] = (block >>> 36) & 1L; + values[vi++] = (block >>> 37) & 1L; + values[vi++] = (block >>> 38) & 1L; + values[vi++] = (block >>> 39) & 1L; + values[vi++] = (block >>> 40) & 1L; + values[vi++] = (block >>> 41) & 1L; + values[vi++] = (block >>> 42) & 1L; + values[vi++] = (block >>> 43) & 1L; + values[vi++] = (block >>> 44) & 1L; + values[vi++] = (block >>> 45) & 1L; + values[vi++] = (block >>> 46) & 1L; + values[vi++] = (block >>> 47) & 1L; + values[vi++] = (block >>> 48) & 1L; + values[vi++] = (block >>> 49) & 1L; + values[vi++] = (block >>> 50) & 1L; + values[vi++] = (block >>> 51) & 1L; + values[vi++] = (block >>> 52) & 1L; + values[vi++] = (block >>> 53) & 1L; + values[vi++] = (block >>> 54) & 1L; + values[vi++] = (block >>> 55) & 1L; + values[vi++] = (block >>> 56) & 1L; + values[vi++] = (block >>> 57) & 1L; + values[vi++] = (block >>> 58) & 1L; + values[vi++] = (block >>> 59) & 1L; + values[vi++] = (block >>> 60) & 1L; + values[vi++] = (block >>> 61) & 1L; + values[vi++] = (block >>> 62) & 1L; + values[vi++] = block >>> 63; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = values[vi++] | (values[vi++] << 1) | (values[vi++] << 2) | (values[vi++] << 3) | (values[vi++] << 4) | (values[vi++] << 5) | (values[vi++] << 6) | (values[vi++] << 7) | (values[vi++] << 8) | (values[vi++] << 9) | (values[vi++] << 10) | (values[vi++] << 11) | (values[vi++] << 12) | (values[vi++] << 13) | (values[vi++] << 14) | (values[vi++] << 15) | (values[vi++] << 16) | (values[vi++] << 17) | (values[vi++] << 18) | (values[vi++] << 19) | (values[vi++] << 20) | (values[vi++] << 21) | (values[vi++] << 22) | (values[vi++] << 23) | (values[vi++] << 24) | (values[vi++] << 25) | (values[vi++] << 26) | (values[vi++] << 27) | (values[vi++] << 28) | (values[vi++] << 29) | (values[vi++] << 30) | (values[vi++] << 31) | (values[vi++] << 32) | (values[vi++] << 33) | (values[vi++] << 34) | (values[vi++] << 35) | (values[vi++] << 36) | (values[vi++] << 37) | (values[vi++] << 38) | (values[vi++] << 39) | (values[vi++] << 40) | (values[vi++] << 41) | (values[vi++] << 42) | (values[vi++] << 43) | (values[vi++] << 44) | (values[vi++] << 45) | (values[vi++] << 46) | (values[vi++] << 47) | (values[vi++] << 48) | (values[vi++] << 49) | (values[vi++] << 50) | (values[vi++] << 51) | (values[vi++] << 52) | (values[vi++] << 53) | (values[vi++] << 54) | (values[vi++] << 55) | (values[vi++] << 56) | (values[vi++] << 57) | (values[vi++] << 58) | (values[vi++] << 59) | (values[vi++] << 60) | (values[vi++] << 61) | (values[vi++] << 62) | (values[vi++] << 63); + } + } + } + + static final class Packed64SingleBlockBulkOperation2 extends BulkOperation { + + public int blocks() { + return 1; + } + + public int values() { + return 32; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block = blocks[bi++]; + values[vi++] = block & 3L; + values[vi++] = (block >>> 2) & 3L; + values[vi++] = (block >>> 4) & 3L; + values[vi++] = (block >>> 6) & 3L; + values[vi++] = (block >>> 8) & 3L; + values[vi++] = (block >>> 10) & 3L; + values[vi++] = (block >>> 12) & 3L; + values[vi++] = (block >>> 14) & 3L; + values[vi++] = (block >>> 16) & 3L; + values[vi++] = (block >>> 18) & 3L; + values[vi++] = (block >>> 20) & 3L; + values[vi++] = (block >>> 22) & 3L; + values[vi++] = (block >>> 24) & 3L; + values[vi++] = (block >>> 26) & 3L; + values[vi++] = (block >>> 28) & 3L; + values[vi++] = (block >>> 30) & 3L; + values[vi++] = (block >>> 32) & 3L; + values[vi++] = (block >>> 34) & 3L; + values[vi++] = (block >>> 36) & 3L; + values[vi++] = (block >>> 38) & 3L; + values[vi++] = (block >>> 40) & 3L; + values[vi++] = (block >>> 42) & 3L; + values[vi++] = (block >>> 44) & 3L; + values[vi++] = (block >>> 46) & 3L; + values[vi++] = (block >>> 48) & 3L; + values[vi++] = (block >>> 50) & 3L; + values[vi++] = (block >>> 52) & 3L; + values[vi++] = (block >>> 54) & 3L; + values[vi++] = (block >>> 56) & 3L; + values[vi++] = (block >>> 58) & 3L; + values[vi++] = (block >>> 60) & 3L; + values[vi++] = block >>> 62; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = values[vi++] | (values[vi++] << 2) | (values[vi++] << 4) | (values[vi++] << 6) | (values[vi++] << 8) | (values[vi++] << 10) | (values[vi++] << 12) | (values[vi++] << 14) | (values[vi++] << 16) | (values[vi++] << 18) | (values[vi++] << 20) | (values[vi++] << 22) | (values[vi++] << 24) | (values[vi++] << 26) | (values[vi++] << 28) | (values[vi++] << 30) | (values[vi++] << 32) | (values[vi++] << 34) | (values[vi++] << 36) | (values[vi++] << 38) | (values[vi++] << 40) | (values[vi++] << 42) | (values[vi++] << 44) | (values[vi++] << 46) | (values[vi++] << 48) | (values[vi++] << 50) | (values[vi++] << 52) | (values[vi++] << 54) | (values[vi++] << 56) | (values[vi++] << 58) | (values[vi++] << 60) | (values[vi++] << 62); + } + } + } + + static final class Packed64SingleBlockBulkOperation3 extends BulkOperation { + + public int blocks() { + return 1; + } + + public int values() { + return 21; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block = blocks[bi++]; + values[vi++] = block & 7L; + values[vi++] = (block >>> 3) & 7L; + values[vi++] = (block >>> 6) & 7L; + values[vi++] = (block >>> 9) & 7L; + values[vi++] = (block >>> 12) & 7L; + values[vi++] = (block >>> 15) & 7L; + values[vi++] = (block >>> 18) & 7L; + values[vi++] = (block >>> 21) & 7L; + values[vi++] = (block >>> 24) & 7L; + values[vi++] = (block >>> 27) & 7L; + values[vi++] = (block >>> 30) & 7L; + values[vi++] = (block >>> 33) & 7L; + values[vi++] = (block >>> 36) & 7L; + values[vi++] = (block >>> 39) & 7L; + values[vi++] = (block >>> 42) & 7L; + values[vi++] = (block >>> 45) & 7L; + values[vi++] = (block >>> 48) & 7L; + values[vi++] = (block >>> 51) & 7L; + values[vi++] = (block >>> 54) & 7L; + values[vi++] = (block >>> 57) & 7L; + values[vi++] = block >>> 60; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = values[vi++] | (values[vi++] << 3) | (values[vi++] << 6) | (values[vi++] << 9) | (values[vi++] << 12) | (values[vi++] << 15) | (values[vi++] << 18) | (values[vi++] << 21) | (values[vi++] << 24) | (values[vi++] << 27) | (values[vi++] << 30) | (values[vi++] << 33) | (values[vi++] << 36) | (values[vi++] << 39) | (values[vi++] << 42) | (values[vi++] << 45) | (values[vi++] << 48) | (values[vi++] << 51) | (values[vi++] << 54) | (values[vi++] << 57) | (values[vi++] << 60); + } + } + } + + static final class Packed64SingleBlockBulkOperation4 extends BulkOperation { + + public int blocks() { + return 1; + } + + public int values() { + return 16; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block = blocks[bi++]; + values[vi++] = block & 15L; + values[vi++] = (block >>> 4) & 15L; + values[vi++] = (block >>> 8) & 15L; + values[vi++] = (block >>> 12) & 15L; + values[vi++] = (block >>> 16) & 15L; + values[vi++] = (block >>> 20) & 15L; + values[vi++] = (block >>> 24) & 15L; + values[vi++] = (block >>> 28) & 15L; + values[vi++] = (block >>> 32) & 15L; + values[vi++] = (block >>> 36) & 15L; + values[vi++] = (block >>> 40) & 15L; + values[vi++] = (block >>> 44) & 15L; + values[vi++] = (block >>> 48) & 15L; + values[vi++] = (block >>> 52) & 15L; + values[vi++] = (block >>> 56) & 15L; + values[vi++] = block >>> 60; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = values[vi++] | (values[vi++] << 4) | (values[vi++] << 8) | (values[vi++] << 12) | (values[vi++] << 16) | (values[vi++] << 20) | (values[vi++] << 24) | (values[vi++] << 28) | (values[vi++] << 32) | (values[vi++] << 36) | (values[vi++] << 40) | (values[vi++] << 44) | (values[vi++] << 48) | (values[vi++] << 52) | (values[vi++] << 56) | (values[vi++] << 60); + } + } + } + + static final class Packed64SingleBlockBulkOperation5 extends BulkOperation { + + public int blocks() { + return 1; + } + + public int values() { + return 12; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block = blocks[bi++]; + values[vi++] = block & 31L; + values[vi++] = (block >>> 5) & 31L; + values[vi++] = (block >>> 10) & 31L; + values[vi++] = (block >>> 15) & 31L; + values[vi++] = (block >>> 20) & 31L; + values[vi++] = (block >>> 25) & 31L; + values[vi++] = (block >>> 30) & 31L; + values[vi++] = (block >>> 35) & 31L; + values[vi++] = (block >>> 40) & 31L; + values[vi++] = (block >>> 45) & 31L; + values[vi++] = (block >>> 50) & 31L; + values[vi++] = block >>> 55; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = values[vi++] | (values[vi++] << 5) | (values[vi++] << 10) | (values[vi++] << 15) | (values[vi++] << 20) | (values[vi++] << 25) | (values[vi++] << 30) | (values[vi++] << 35) | (values[vi++] << 40) | (values[vi++] << 45) | (values[vi++] << 50) | (values[vi++] << 55); + } + } + } + + static final class Packed64SingleBlockBulkOperation6 extends BulkOperation { + + public int blocks() { + return 1; + } + + public int values() { + return 10; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block = blocks[bi++]; + values[vi++] = block & 63L; + values[vi++] = (block >>> 6) & 63L; + values[vi++] = (block >>> 12) & 63L; + values[vi++] = (block >>> 18) & 63L; + values[vi++] = (block >>> 24) & 63L; + values[vi++] = (block >>> 30) & 63L; + values[vi++] = (block >>> 36) & 63L; + values[vi++] = (block >>> 42) & 63L; + values[vi++] = (block >>> 48) & 63L; + values[vi++] = block >>> 54; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = values[vi++] | (values[vi++] << 6) | (values[vi++] << 12) | (values[vi++] << 18) | (values[vi++] << 24) | (values[vi++] << 30) | (values[vi++] << 36) | (values[vi++] << 42) | (values[vi++] << 48) | (values[vi++] << 54); + } + } + } + + static final class Packed64SingleBlockBulkOperation7 extends BulkOperation { + + public int blocks() { + return 1; + } + + public int values() { + return 9; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block = blocks[bi++]; + values[vi++] = block & 127L; + values[vi++] = (block >>> 7) & 127L; + values[vi++] = (block >>> 14) & 127L; + values[vi++] = (block >>> 21) & 127L; + values[vi++] = (block >>> 28) & 127L; + values[vi++] = (block >>> 35) & 127L; + values[vi++] = (block >>> 42) & 127L; + values[vi++] = (block >>> 49) & 127L; + values[vi++] = block >>> 56; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = values[vi++] | (values[vi++] << 7) | (values[vi++] << 14) | (values[vi++] << 21) | (values[vi++] << 28) | (values[vi++] << 35) | (values[vi++] << 42) | (values[vi++] << 49) | (values[vi++] << 56); + } + } + } + + static final class Packed64SingleBlockBulkOperation8 extends BulkOperation { + + public int blocks() { + return 1; + } + + public int values() { + return 8; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block = blocks[bi++]; + values[vi++] = block & 255L; + values[vi++] = (block >>> 8) & 255L; + values[vi++] = (block >>> 16) & 255L; + values[vi++] = (block >>> 24) & 255L; + values[vi++] = (block >>> 32) & 255L; + values[vi++] = (block >>> 40) & 255L; + values[vi++] = (block >>> 48) & 255L; + values[vi++] = block >>> 56; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = values[vi++] | (values[vi++] << 8) | (values[vi++] << 16) | (values[vi++] << 24) | (values[vi++] << 32) | (values[vi++] << 40) | (values[vi++] << 48) | (values[vi++] << 56); + } + } + } + + static final class Packed64SingleBlockBulkOperation9 extends BulkOperation { + + public int blocks() { + return 1; + } + + public int values() { + return 7; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block = blocks[bi++]; + values[vi++] = block & 511L; + values[vi++] = (block >>> 9) & 511L; + values[vi++] = (block >>> 18) & 511L; + values[vi++] = (block >>> 27) & 511L; + values[vi++] = (block >>> 36) & 511L; + values[vi++] = (block >>> 45) & 511L; + values[vi++] = block >>> 54; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = values[vi++] | (values[vi++] << 9) | (values[vi++] << 18) | (values[vi++] << 27) | (values[vi++] << 36) | (values[vi++] << 45) | (values[vi++] << 54); + } + } + } + + static final class Packed64SingleBlockBulkOperation10 extends BulkOperation { + + public int blocks() { + return 1; + } + + public int values() { + return 6; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block = blocks[bi++]; + values[vi++] = block & 1023L; + values[vi++] = (block >>> 10) & 1023L; + values[vi++] = (block >>> 20) & 1023L; + values[vi++] = (block >>> 30) & 1023L; + values[vi++] = (block >>> 40) & 1023L; + values[vi++] = block >>> 50; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = values[vi++] | (values[vi++] << 10) | (values[vi++] << 20) | (values[vi++] << 30) | (values[vi++] << 40) | (values[vi++] << 50); + } + } + } + + static final class Packed64SingleBlockBulkOperation12 extends BulkOperation { + + public int blocks() { + return 1; + } + + public int values() { + return 5; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block = blocks[bi++]; + values[vi++] = block & 4095L; + values[vi++] = (block >>> 12) & 4095L; + values[vi++] = (block >>> 24) & 4095L; + values[vi++] = (block >>> 36) & 4095L; + values[vi++] = block >>> 48; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = values[vi++] | (values[vi++] << 12) | (values[vi++] << 24) | (values[vi++] << 36) | (values[vi++] << 48); + } + } + } + + static final class Packed64SingleBlockBulkOperation16 extends BulkOperation { + + public int blocks() { + return 1; + } + + public int values() { + return 4; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block = blocks[bi++]; + values[vi++] = block & 65535L; + values[vi++] = (block >>> 16) & 65535L; + values[vi++] = (block >>> 32) & 65535L; + values[vi++] = block >>> 48; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = values[vi++] | (values[vi++] << 16) | (values[vi++] << 32) | (values[vi++] << 48); + } + } + } + + static final class Packed64SingleBlockBulkOperation21 extends BulkOperation { + + public int blocks() { + return 1; + } + + public int values() { + return 3; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block = blocks[bi++]; + values[vi++] = block & 2097151L; + values[vi++] = (block >>> 21) & 2097151L; + values[vi++] = block >>> 42; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = values[vi++] | (values[vi++] << 21) | (values[vi++] << 42); + } + } + } + + static final class Packed64SingleBlockBulkOperation32 extends BulkOperation { + + public int blocks() { + return 1; + } + + public int values() { + return 2; + } + + public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + final long block = blocks[bi++]; + values[vi++] = block & 4294967295L; + values[vi++] = block >>> 32; + } + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + assert bi + iterations * blocks() <= blocks.length; + assert vi + iterations * values() <= values.length; + for (int i = 0; i < iterations; ++i) { + blocks[bi++] = values[vi++] | (values[vi++] << 32); + } + } + } +} \ No newline at end of file diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/Direct16.java b/lucene/core/src/java/org/apache/lucene/util/packed/Direct16.java index 3eb698c..bb48192 100644 --- a/lucene/core/src/java/org/apache/lucene/util/packed/Direct16.java +++ b/lucene/core/src/java/org/apache/lucene/util/packed/Direct16.java @@ -1,6 +1,8 @@ +// This file has been automatically generated, DO NOT EDIT + package org.apache.lucene.util.packed; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -24,57 +26,37 @@ import java.io.IOException; import java.util.Arrays; /** - * Direct wrapping of 16 bit values to a backing array of shorts. + * Direct wrapping of 16-bits values to a backing array. * @lucene.internal */ +final class Direct16 extends PackedInts.MutableImpl { + final short[] values; -class Direct16 extends PackedInts.ReaderImpl - implements PackedInts.Mutable { - private short[] values; - private static final int BITS_PER_VALUE = 16; - - public Direct16(int valueCount) { - super(valueCount, BITS_PER_VALUE); + Direct16(int valueCount) { + super(valueCount, 16); values = new short[valueCount]; } - public Direct16(DataInput in, int valueCount) throws IOException { - super(valueCount, BITS_PER_VALUE); - short[] values = new short[valueCount]; - for(int i=0;i

      - * Note: The values are used directly, so changes to the values will - * affect the structure. - * @param values used as the internal backing array. - */ - public Direct16(short[] values) { - super(values.length, BITS_PER_VALUE); - this.values = values; } + @Override public long get(final int index) { - assert index >= 0 && index < size(); - return 0xFFFFL & values[index]; + return values[index] & 0xFFFFL; } public void set(final int index, final long value) { - values[index] = (short)(value & 0xFFFF); + values[index] = (short) (value); } public long ramBytesUsed() { @@ -82,7 +64,7 @@ class Direct16 extends PackedInts.ReaderImpl } public void clear() { - Arrays.fill(values, (short)0); + Arrays.fill(values, (short) 0L); } @Override @@ -94,4 +76,35 @@ class Direct16 extends PackedInts.ReaderImpl public boolean hasArray() { return true; } + + @Override + public int get(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + assert off + len <= arr.length; + + final int gets = Math.min(valueCount - index, len); + for (int i = index, o = off, end = index + gets; i < end; ++i, ++o) { + arr[o] = values[i] & 0xFFFFL; + } + return gets; + } + + public int set(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + assert off + len <= arr.length; + + final int sets = Math.min(valueCount - index, len); + for (int i = index, o = off, end = index + sets; i < end; ++i, ++o) { + values[i] = (short) arr[o]; + } + return sets; + } + + @Override + public void fill(int fromIndex, int toIndex, long val) { + assert val == (val & 0xFFFFL); + Arrays.fill(values, fromIndex, toIndex, (short) val); + } } diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/Direct32.java b/lucene/core/src/java/org/apache/lucene/util/packed/Direct32.java index 38f6794..836e32e 100644 --- a/lucene/core/src/java/org/apache/lucene/util/packed/Direct32.java +++ b/lucene/core/src/java/org/apache/lucene/util/packed/Direct32.java @@ -1,6 +1,8 @@ +// This file has been automatically generated, DO NOT EDIT + package org.apache.lucene.util.packed; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -24,53 +26,37 @@ import java.io.IOException; import java.util.Arrays; /** - * Direct wrapping of 32 bit values to a backing array of ints. + * Direct wrapping of 32-bits values to a backing array. * @lucene.internal */ +final class Direct32 extends PackedInts.MutableImpl { + final int[] values; -class Direct32 extends PackedInts.ReaderImpl - implements PackedInts.Mutable { - private int[] values; - private static final int BITS_PER_VALUE = 32; - - public Direct32(int valueCount) { - super(valueCount, BITS_PER_VALUE); + Direct32(int valueCount) { + super(valueCount, 32); values = new int[valueCount]; } - public Direct32(DataInput in, int valueCount) throws IOException { - super(valueCount, BITS_PER_VALUE); - int[] values = new int[valueCount]; - for(int i=0;i

      - * Note: The values are used directly, so changes to the given values will - * affect the structure. - * @param values used as the internal backing array. - */ - public Direct32(int[] values) { - super(values.length, BITS_PER_VALUE); - this.values = values; } + @Override public long get(final int index) { - assert index >= 0 && index < size(); - return 0xFFFFFFFFL & values[index]; + return values[index] & 0xFFFFFFFFL; } public void set(final int index, final long value) { - values[index] = (int)(value & 0xFFFFFFFF); + values[index] = (int) (value); } public long ramBytesUsed() { @@ -78,11 +64,11 @@ class Direct32 extends PackedInts.ReaderImpl } public void clear() { - Arrays.fill(values, 0); + Arrays.fill(values, (int) 0L); } - + @Override - public int[] getArray() { + public Object getArray() { return values; } @@ -90,4 +76,35 @@ class Direct32 extends PackedInts.ReaderImpl public boolean hasArray() { return true; } + + @Override + public int get(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + assert off + len <= arr.length; + + final int gets = Math.min(valueCount - index, len); + for (int i = index, o = off, end = index + gets; i < end; ++i, ++o) { + arr[o] = values[i] & 0xFFFFFFFFL; + } + return gets; + } + + public int set(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + assert off + len <= arr.length; + + final int sets = Math.min(valueCount - index, len); + for (int i = index, o = off, end = index + sets; i < end; ++i, ++o) { + values[i] = (int) arr[o]; + } + return sets; + } + + @Override + public void fill(int fromIndex, int toIndex, long val) { + assert val == (val & 0xFFFFFFFFL); + Arrays.fill(values, fromIndex, toIndex, (int) val); + } } diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/Direct64.java b/lucene/core/src/java/org/apache/lucene/util/packed/Direct64.java index 6f88514..6033daa 100644 --- a/lucene/core/src/java/org/apache/lucene/util/packed/Direct64.java +++ b/lucene/core/src/java/org/apache/lucene/util/packed/Direct64.java @@ -1,6 +1,8 @@ +// This file has been automatically generated, DO NOT EDIT + package org.apache.lucene.util.packed; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -24,49 +26,31 @@ import java.io.IOException; import java.util.Arrays; /** - * Direct wrapping of 32 bit values to a backing array of ints. + * Direct wrapping of 64-bits values to a backing array. * @lucene.internal */ +final class Direct64 extends PackedInts.MutableImpl { + final long[] values; -class Direct64 extends PackedInts.ReaderImpl - implements PackedInts.Mutable { - private long[] values; - private static final int BITS_PER_VALUE = 64; - - public Direct64(int valueCount) { - super(valueCount, BITS_PER_VALUE); + Direct64(int valueCount) { + super(valueCount, 64); values = new long[valueCount]; } - public Direct64(DataInput in, int valueCount) throws IOException { - super(valueCount, BITS_PER_VALUE); - long[] values = new long[valueCount]; - for(int i=0;i

      - * Note: The values are used directly, so changes to the given values will - * affect the structure. - * @param values used as the internal backing array. - */ - public Direct64(long[] values) { - super(values.length, BITS_PER_VALUE); - this.values = values; } + @Override public long get(final int index) { - assert index >= 0 && index < size(); return values[index]; } public void set(final int index, final long value) { - values[index] = value; + values[index] = (value); } public long ramBytesUsed() { @@ -78,7 +62,7 @@ class Direct64 extends PackedInts.ReaderImpl } @Override - public long[] getArray() { + public Object getArray() { return values; } @@ -86,4 +70,30 @@ class Direct64 extends PackedInts.ReaderImpl public boolean hasArray() { return true; } + + @Override + public int get(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + assert off + len <= arr.length; + + final int gets = Math.min(valueCount - index, len); + System.arraycopy(values, index, arr, off, gets); + return gets; + } + + public int set(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + assert off + len <= arr.length; + + final int sets = Math.min(valueCount - index, len); + System.arraycopy(arr, off, values, index, sets); + return sets; + } + + @Override + public void fill(int fromIndex, int toIndex, long val) { + Arrays.fill(values, fromIndex, toIndex, val); + } } diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/Direct8.java b/lucene/core/src/java/org/apache/lucene/util/packed/Direct8.java index ac42eab..d79c7f1 100644 --- a/lucene/core/src/java/org/apache/lucene/util/packed/Direct8.java +++ b/lucene/core/src/java/org/apache/lucene/util/packed/Direct8.java @@ -1,6 +1,8 @@ +// This file has been automatically generated, DO NOT EDIT + package org.apache.lucene.util.packed; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -24,58 +26,37 @@ import java.io.IOException; import java.util.Arrays; /** - * Direct wrapping of 8 bit values to a backing array of bytes. + * Direct wrapping of 8-bits values to a backing array. * @lucene.internal */ +final class Direct8 extends PackedInts.MutableImpl { + final byte[] values; -class Direct8 extends PackedInts.ReaderImpl - implements PackedInts.Mutable { - private byte[] values; - private static final int BITS_PER_VALUE = 8; - - public Direct8(int valueCount) { - super(valueCount, BITS_PER_VALUE); + Direct8(int valueCount) { + super(valueCount, 8); values = new byte[valueCount]; } - public Direct8(DataInput in, int valueCount) - throws IOException { - super(valueCount, BITS_PER_VALUE); - byte[] values = new byte[valueCount]; - for(int i=0;i

      - * Note: The values are used directly, so changes to the given values will - * affect the structure. - * @param values used as the internal backing array. - */ - public Direct8(byte[] values) { - super(values.length, BITS_PER_VALUE); - this.values = values; } + @Override public long get(final int index) { - assert index >= 0 && index < size(); - return 0xFFL & values[index]; + return values[index] & 0xFFL; } public void set(final int index, final long value) { - values[index] = (byte)(value & 0xFF); + values[index] = (byte) (value); } public long ramBytesUsed() { @@ -83,7 +64,7 @@ class Direct8 extends PackedInts.ReaderImpl } public void clear() { - Arrays.fill(values, (byte)0); + Arrays.fill(values, (byte) 0L); } @Override @@ -95,4 +76,35 @@ class Direct8 extends PackedInts.ReaderImpl public boolean hasArray() { return true; } + + @Override + public int get(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + assert off + len <= arr.length; + + final int gets = Math.min(valueCount - index, len); + for (int i = index, o = off, end = index + gets; i < end; ++i, ++o) { + arr[o] = values[i] & 0xFFL; + } + return gets; + } + + public int set(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + assert off + len <= arr.length; + + final int sets = Math.min(valueCount - index, len); + for (int i = index, o = off, end = index + sets; i < end; ++i, ++o) { + values[i] = (byte) arr[o]; + } + return sets; + } + + @Override + public void fill(int fromIndex, int toIndex, long val) { + assert val == (val & 0xFFL); + Arrays.fill(values, fromIndex, toIndex, (byte) val); + } } diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/DirectPacked64SingleBlockReader.java b/lucene/core/src/java/org/apache/lucene/util/packed/DirectPacked64SingleBlockReader.java index 4b7b9d8..ce93592 100644 --- a/lucene/core/src/java/org/apache/lucene/util/packed/DirectPacked64SingleBlockReader.java +++ b/lucene/core/src/java/org/apache/lucene/util/packed/DirectPacked64SingleBlockReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.packed; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -51,4 +51,9 @@ final class DirectPacked64SingleBlockReader extends PackedInts.ReaderImpl { throw new IllegalStateException("failed", e); } } + + @Override + public long ramBytesUsed() { + return 0; + } } diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/DirectPackedReader.java b/lucene/core/src/java/org/apache/lucene/util/packed/DirectPackedReader.java index f6bf694..8a7e75d 100644 --- a/lucene/core/src/java/org/apache/lucene/util/packed/DirectPackedReader.java +++ b/lucene/core/src/java/org/apache/lucene/util/packed/DirectPackedReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.packed; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -32,8 +32,7 @@ final class DirectPackedReader extends PackedInts.ReaderImpl { // masks[n-1] masks for bottom n bits private final long[] masks; - public DirectPackedReader(int bitsPerValue, int valueCount, IndexInput in) - throws IOException { + public DirectPackedReader(int bitsPerValue, int valueCount, IndexInput in) { super(valueCount, bitsPerValue); this.in = in; @@ -73,4 +72,9 @@ final class DirectPackedReader extends PackedInts.ReaderImpl { throw new IllegalStateException("failed", ioe); } } + + @Override + public long ramBytesUsed() { + return 0; + } } diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java b/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java index 756bf74..d87d434 100644 --- a/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.packed; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,6 +17,10 @@ package org.apache.lucene.util.packed; * limitations under the License. */ +import java.io.IOException; + +import org.apache.lucene.store.DataOutput; + /** * Implements {@link PackedInts.Mutable}, but grows the * bit count of the underlying packed ints on-demand. @@ -62,21 +66,21 @@ public class GrowableWriter implements PackedInts.Mutable { return current.hasArray(); } - public void set(int index, long value) { - if (value >= currentMaxValue) { - int bpv = getBitsPerValue(); - while(currentMaxValue <= value && currentMaxValue != Long.MAX_VALUE) { - bpv++; - currentMaxValue *= 2; - } - final int valueCount = size(); - PackedInts.Mutable next = PackedInts.getMutable(valueCount, bpv, acceptableOverheadRatio); - for(int i=0;i= 0; + if (value <= currentMaxValue) { + return; } + final int bitsRequired = PackedInts.bitsRequired(value); + final int valueCount = size(); + PackedInts.Mutable next = PackedInts.getMutable(valueCount, bitsRequired, acceptableOverheadRatio); + PackedInts.copy(current, 0, next, 0, valueCount, PackedInts.DEFAULT_BUFFER_SIZE); + current = next; + currentMaxValue = PackedInts.maxValue(current.getBitsPerValue()); + } + + public void set(int index, long value) { + ensureCapacity(value); current.set(index, value); } @@ -87,10 +91,38 @@ public class GrowableWriter implements PackedInts.Mutable { public GrowableWriter resize(int newSize) { GrowableWriter next = new GrowableWriter(getBitsPerValue(), newSize, acceptableOverheadRatio); final int limit = Math.min(size(), newSize); - for(int i=0;i MAX_SIZE) { throw new ArrayIndexOutOfBoundsException("MAX_SIZE exceeded"); } - this.blocks = new short[3 * valueCount]; + blocks = new short[valueCount * 3]; } Packed16ThreeBlocks(DataInput in, int valueCount) throws IOException { this(valueCount); - for (int i = 0; i < blocks.length; i++) { + for (int i = 0; i < 3 * valueCount; ++i) { blocks[i] = in.readShort(); } final int mod = blocks.length % 4; if (mod != 0) { - final int pad = 4 - mod; - // round out long - for (int i = 0; i < pad; i++) { - in.readShort(); + for (int i = mod; i < 4; ++i) { + in.readShort(); } } } @@ -57,18 +58,59 @@ final class Packed16ThreeBlocks extends PackedInts.ReaderImpl @Override public long get(int index) { final int o = index * 3; - return (blocks[o] & 0xffffL) << 32 | (blocks[o+1] & 0xffffL) << 16 | (blocks[o+2] & 0xffffL); + return (blocks[o] & 0xFFFFL) << 32 | (blocks[o+1] & 0xFFFFL) << 16 | (blocks[o+2] & 0xFFFFL); + } + + @Override + public int get(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + assert off + len <= arr.length; + + final int gets = Math.min(valueCount - index, len); + for (int i = index * 3, end = (index + gets) * 3; i < end; i+=3) { + arr[off++] = (blocks[i] & 0xFFFFL) << 32 | (blocks[i+1] & 0xFFFFL) << 16 | (blocks[i+2] & 0xFFFFL); + } + return gets; } @Override public void set(int index, long value) { final int o = index * 3; - blocks[o] = (short) (value >> 32); - blocks[o+1] = (short) (value >> 16); + blocks[o] = (short) (value >>> 32); + blocks[o+1] = (short) (value >>> 16); blocks[o+2] = (short) value; } @Override + public int set(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + assert off + len <= arr.length; + + final int sets = Math.min(valueCount - index, len); + for (int i = off, o = index * 3, end = off + sets; i < end; ++i) { + final long value = arr[i]; + blocks[o++] = (short) (value >>> 32); + blocks[o++] = (short) (value >>> 16); + blocks[o++] = (short) value; + } + return sets; + } + + @Override + public void fill(int fromIndex, int toIndex, long val) { + final short block1 = (short) (val >>> 32); + final short block2 = (short) (val >>> 16); + final short block3 = (short) val; + for (int i = fromIndex * 3, end = toIndex * 3; i < end; i += 3) { + blocks[i] = block1; + blocks[i+1] = block2; + blocks[i+2] = block3; + } + } + + @Override public void clear() { Arrays.fill(blocks, (short) 0); } diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java b/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java index 39ba0d5..83c7dd11 100644 --- a/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java +++ b/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.packed; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -25,89 +25,40 @@ import java.util.Arrays; /** * Space optimized random access capable array of values with a fixed number of - * bits. For 32 bits/value and less, performance on 32 bit machines is not - * optimal. Consider using {@link Packed32} for such a setup. + * bits/value. Values are packed contiguously. + *

      + * The implementation strives to perform af fast as possible under the + * constraint of contiguous bits, by avoiding expensive operations. This comes + * at the cost of code clarity. *

      - * The implementation strives to avoid conditionals and expensive operations, - * sacrificing code clarity to achieve better performance. + * Technical details: This implementation is a refinement of a non-branching + * version. The non-branching get and set methods meant that 2 or 4 atomics in + * the underlying array were always accessed, even for the cases where only + * 1 or 2 were needed. Even with caching, this had a detrimental effect on + * performance. + * Related to this issue, the old implementation used lookup tables for shifts + * and masks, which also proved to be a bit slower than calculating the shifts + * and masks on the fly. + * See https://issues.apache.org/jira/browse/LUCENE-4062 for details. + * */ - -class Packed64 extends PackedInts.ReaderImpl implements PackedInts.Mutable { +class Packed64 extends PackedInts.MutableImpl { static final int BLOCK_SIZE = 64; // 32 = int, 64 = long static final int BLOCK_BITS = 6; // The #bits representing BLOCK_SIZE static final int MOD_MASK = BLOCK_SIZE - 1; // x % BLOCK_SIZE - private static final int ENTRY_SIZE = BLOCK_SIZE + 1; - static final int FAC_BITPOS = 3; - - /* - * In order to make an efficient value-getter, conditionals should be - * avoided. A value can be positioned inside of a block, requiring shifting - * left or right or it can span two blocks, requiring a left-shift on the - * first block and a right-shift on the right block. - *

      - * By always shifting the first block both left and right, we get exactly - * the right bits. By always shifting the second block right and applying - * a mask, we get the right bits there. After that, we | the two bitsets. - */ - static final int[][] SHIFTS = - new int[ENTRY_SIZE][ENTRY_SIZE * FAC_BITPOS]; - static final long[][] MASKS = new long[ENTRY_SIZE][ENTRY_SIZE]; - - static { // Generate shifts - for (int elementBits = 1 ; elementBits <= BLOCK_SIZE ; elementBits++) { - for (int bitPos = 0 ; bitPos < BLOCK_SIZE ; bitPos++) { - int[] currentShifts = SHIFTS[elementBits]; - int base = bitPos * FAC_BITPOS; - currentShifts[base ] = bitPos; - currentShifts[base + 1] = BLOCK_SIZE - elementBits; - if (bitPos <= BLOCK_SIZE - elementBits) { // Single block - currentShifts[base + 2] = 0; - MASKS[elementBits][bitPos] = 0; - } else { // Two blocks - int rBits = elementBits - (BLOCK_SIZE - bitPos); - currentShifts[base + 2] = BLOCK_SIZE - rBits; - MASKS[elementBits][bitPos] = ~(~0L << rBits); - } - } - } - } - - /* - * The setter requires more masking than the getter. - */ - private static final long[][] WRITE_MASKS = - new long[ENTRY_SIZE][ENTRY_SIZE * FAC_BITPOS]; - static { - for (int elementBits = 1 ; elementBits <= BLOCK_SIZE ; elementBits++) { - long elementPosMask = ~(~0L << elementBits); - int[] currentShifts = SHIFTS[elementBits]; - long[] currentMasks = WRITE_MASKS[elementBits]; - for (int bitPos = 0 ; bitPos < BLOCK_SIZE ; bitPos++) { - int base = bitPos * FAC_BITPOS; - currentMasks[base ] =~((elementPosMask - << currentShifts[base + 1]) - >>> currentShifts[base]); - if (bitPos <= BLOCK_SIZE - elementBits) { // Second block not used - currentMasks[base+1] = ~0; // Keep all bits - currentMasks[base+2] = 0; // Or with 0 - } else { - currentMasks[base+1] = ~(elementPosMask - << currentShifts[base + 2]); - currentMasks[base+2] = currentShifts[base + 2] == 0 ? 0 : ~0; - } - } - } - } - - /* The bits */ - private long[] blocks; - - // Cached calculations - private int maxPos; // blocks.length * BLOCK_SIZE / elementBits - 1 - private int[] shifts; // The shifts for the current elementBits - private long[] readMasks; - private long[] writeMasks; + /** + * Values are stores contiguously in the blocks array. + */ + private final long[] blocks; + /** + * A right-aligned mask of width BitsPerValue used by {@link #get(int)}. + */ + private final long maskRight; + /** + * Optimization: Saves one lookup in {@link #get(int)}. + */ + private final int bpvMinusBlockSize; /** * Creates an array with the internal structures adjusted for the given @@ -116,18 +67,18 @@ class Packed64 extends PackedInts.ReaderImpl implements PackedInts.Mutable { * @param bitsPerValue the number of bits available for any given value. */ public Packed64(int valueCount, int bitsPerValue) { - // TODO: Test for edge-cases (2^31 values, 63 bitsPerValue) - // +2 due to the avoid-conditionals-trick. The last entry is always 0 - this(new long[(int)((long)valueCount * bitsPerValue / BLOCK_SIZE + 2)], + // NOTE: block-size was previously calculated as + // valueCount * bitsPerValue / BLOCK_SIZE + 1 + // due to memory layout requirements dictated by non-branching code + this(new long[size(valueCount, bitsPerValue)], valueCount, bitsPerValue); } - /** * Creates an array backed by the given blocks. *

      * Note: The blocks are used directly, so changes to the given block will - * affect the Packed32-structure. + * affect the Packed64-structure. * @param blocks used as the internal backing array. Not that the last * element cannot be addressed directly. * @param valueCount the number of values. @@ -136,7 +87,8 @@ class Packed64 extends PackedInts.ReaderImpl implements PackedInts.Mutable { public Packed64(long[] blocks, int valueCount, int bitsPerValue) { super(valueCount, bitsPerValue); this.blocks = blocks; - updateCached(); + maskRight = ~0L << (BLOCK_SIZE-bitsPerValue) >>> (BLOCK_SIZE-bitsPerValue); + bpvMinusBlockSize = bitsPerValue - BLOCK_SIZE; } /** @@ -151,12 +103,12 @@ class Packed64 extends PackedInts.ReaderImpl implements PackedInts.Mutable { throws IOException { super(valueCount, bitsPerValue); int size = size(valueCount, bitsPerValue); - blocks = new long[size+1]; // +1 due to non-conditional tricks - // TODO: find a faster way to bulk-read longs... + blocks = new long[size]; // Previously +1 due to non-conditional tricks for(int i=0;i>> (BLOCK_SIZE-bitsPerValue); + bpvMinusBlockSize = bitsPerValue - BLOCK_SIZE; } private static int size(int valueCount, int bitsPerValue) { @@ -164,52 +116,207 @@ class Packed64 extends PackedInts.ReaderImpl implements PackedInts.Mutable { return (int)(totBitCount/64 + ((totBitCount % 64 == 0 ) ? 0:1)); } - private void updateCached() { - readMasks = MASKS[bitsPerValue]; - shifts = SHIFTS[bitsPerValue]; - writeMasks = WRITE_MASKS[bitsPerValue]; - maxPos = (int)((((long)blocks.length) * BLOCK_SIZE / bitsPerValue) - 2); - } - /** * @param index the position of the value. * @return the value at the given index. */ + @Override public long get(final int index) { - assert index >= 0 && index < size(); + // The abstract index in a bit stream final long majorBitPos = (long)index * bitsPerValue; - final int elementPos = (int)(majorBitPos >>> BLOCK_BITS); // / BLOCK_SIZE - final int bitPos = (int)(majorBitPos & MOD_MASK); // % BLOCK_SIZE); + // The index in the backing long-array + final int elementPos = (int)(majorBitPos >>> BLOCK_BITS); + // The number of value-bits in the second long + final long endBits = (majorBitPos & MOD_MASK) + bpvMinusBlockSize; + + if (endBits <= 0) { // Single block + return (blocks[elementPos] >>> -endBits) & maskRight; + } + // Two blocks + return ((blocks[elementPos] << endBits) + | (blocks[elementPos+1] >>> (BLOCK_SIZE - endBits))) + & maskRight; + } + + @Override + public int get(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + len = Math.min(len, valueCount - index); + assert off + len <= arr.length; + + final int originalIndex = index; + final BulkOperation op = BulkOperation.of(PackedInts.Format.PACKED, bitsPerValue); + + // go to the next block where the value does not span across two blocks + final int offsetInBlocks = index % op.values(); + if (offsetInBlocks != 0) { + for (int i = offsetInBlocks; i < op.values() && len > 0; ++i) { + arr[off++] = get(index++); + --len; + } + if (len == 0) { + return index - originalIndex; + } + } - final int base = bitPos * FAC_BITPOS; - assert elementPos < blocks.length : "elementPos: " + elementPos + "; blocks.len: " + blocks.length; - return ((blocks[elementPos] << shifts[base]) >>> shifts[base+1]) | - ((blocks[elementPos+1] >>> shifts[base+2]) & readMasks[bitPos]); + // bulk get + assert index % op.values() == 0; + int blockIndex = (int) ((long) index * bitsPerValue) >>> BLOCK_BITS; + assert (((long)index * bitsPerValue) & MOD_MASK) == 0; + final int iterations = len / op.values(); + op.get(blocks, blockIndex, arr, off, iterations); + final int gotValues = iterations * op.values(); + index += gotValues; + len -= gotValues; + assert len >= 0; + + if (index > originalIndex) { + // stay at the block boundary + return index - originalIndex; + } else { + // no progress so far => already at a block boundary but no full block to get + assert index == originalIndex; + return super.get(index, arr, off, len); + } } + @Override public void set(final int index, final long value) { + // The abstract index in a contiguous bit stream final long majorBitPos = (long)index * bitsPerValue; + // The index in the backing long-array final int elementPos = (int)(majorBitPos >>> BLOCK_BITS); // / BLOCK_SIZE - final int bitPos = (int)(majorBitPos & MOD_MASK); // % BLOCK_SIZE); - final int base = bitPos * FAC_BITPOS; + // The number of value-bits in the second long + final long endBits = (majorBitPos & MOD_MASK) + bpvMinusBlockSize; - blocks[elementPos ] = (blocks[elementPos ] & writeMasks[base]) - | (value << shifts[base + 1] >>> shifts[base]); - blocks[elementPos+1] = (blocks[elementPos+1] & writeMasks[base+1]) - | ((value << shifts[base + 2]) & writeMasks[base+2]); + if (endBits <= 0) { // Single block + blocks[elementPos] = blocks[elementPos] & ~(maskRight << -endBits) + | (value << -endBits); + return; + } + // Two blocks + blocks[elementPos] = blocks[elementPos] & ~(maskRight >>> endBits) + | (value >>> endBits); + blocks[elementPos+1] = blocks[elementPos+1] & (~0L >>> endBits) + | (value << (BLOCK_SIZE - endBits)); + } + + @Override + public int set(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + len = Math.min(len, valueCount - index); + assert off + len <= arr.length; + + final int originalIndex = index; + final BulkOperation op = BulkOperation.of(PackedInts.Format.PACKED, bitsPerValue); + + // go to the next block where the value does not span across two blocks + final int offsetInBlocks = index % op.values(); + if (offsetInBlocks != 0) { + for (int i = offsetInBlocks; i < op.values() && len > 0; ++i) { + set(index++, arr[off++]); + --len; + } + if (len == 0) { + return index - originalIndex; + } + } + + // bulk get + assert index % op.values() == 0; + int blockIndex = (int) ((long) index * bitsPerValue) >>> BLOCK_BITS; + assert (((long)index * bitsPerValue) & MOD_MASK) == 0; + final int iterations = len / op.values(); + op.set(blocks, blockIndex, arr, off, iterations); + final int setValues = iterations * op.values(); + index += setValues; + len -= setValues; + assert len >= 0; + + if (index > originalIndex) { + // stay at the block boundary + return index - originalIndex; + } else { + // no progress so far => already at a block boundary but no full block to get + assert index == originalIndex; + return super.set(index, arr, off, len); + } } @Override public String toString() { return "Packed64(bitsPerValue=" + bitsPerValue + ", size=" - + size() + ", maxPos=" + maxPos - + ", elements.length=" + blocks.length + ")"; + + size() + ", elements.length=" + blocks.length + ")"; } + @Override public long ramBytesUsed() { return RamUsageEstimator.sizeOf(blocks); } + @Override + public void fill(int fromIndex, int toIndex, long val) { + assert PackedInts.bitsRequired(val) <= getBitsPerValue(); + assert fromIndex <= toIndex; + + // minimum number of values that use an exact number of full blocks + final int nAlignedValues = 64 / gcd(64, bitsPerValue); + final int span = toIndex - fromIndex; + if (span <= 3 * nAlignedValues) { + // there needs be at least 2 * nAlignedValues aligned values for the + // block approach to be worth trying + super.fill(fromIndex, toIndex, val); + return; + } + + // fill the first values naively until the next block start + final int fromIndexModNAlignedValues = fromIndex % nAlignedValues; + if (fromIndexModNAlignedValues != 0) { + for (int i = fromIndexModNAlignedValues; i < nAlignedValues; ++i) { + set(fromIndex++, val); + } + } + assert fromIndex % nAlignedValues == 0; + + // compute the long[] blocks for nAlignedValues consecutive values and + // use them to set as many values as possible without applying any mask + // or shift + final int nAlignedBlocks = (nAlignedValues * bitsPerValue) >> 6; + final long[] nAlignedValuesBlocks; + { + Packed64 values = new Packed64(nAlignedValues, bitsPerValue); + for (int i = 0; i < nAlignedValues; ++i) { + values.set(i, val); + } + nAlignedValuesBlocks = values.blocks; + assert nAlignedBlocks <= nAlignedValuesBlocks.length; + } + final int startBlock = (int) (((long) fromIndex * bitsPerValue) >>> 6); + final int endBlock = (int) (((long) toIndex * bitsPerValue) >>> 6); + for (int block = startBlock; block < endBlock; ++block) { + final long blockValue = nAlignedValuesBlocks[block % nAlignedBlocks]; + blocks[block] = blockValue; + } + + // fill the gap + for (int i = (int) (((long) endBlock << 6) / bitsPerValue); i < toIndex; ++i) { + set(i, val); + } + } + + private static int gcd(int a, int b) { + if (a < b) { + return gcd(b, a); + } else if (b == 0) { + return a; + } else { + return gcd(b, a % b); + } + } + + @Override public void clear() { Arrays.fill(blocks, 0L); } diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java b/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java index 101c564..3854b8d 100644 --- a/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java +++ b/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java @@ -1,12 +1,8 @@ -package org.apache.lucene.util.packed; - -import java.io.IOException; -import java.util.Arrays; +// This file has been automatically generated, DO NOT EDIT -import org.apache.lucene.store.DataInput; -import org.apache.lucene.util.RamUsageEstimator; +package org.apache.lucene.util.packed; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF @@ -23,35 +19,195 @@ import org.apache.lucene.util.RamUsageEstimator; * the License. */ +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.store.DataInput; +import org.apache.lucene.util.RamUsageEstimator; + /** * This class is similar to {@link Packed64} except that it trades space for * speed by ensuring that a single block needs to be read/written in order to * read/write a value. */ -abstract class Packed64SingleBlock extends PackedInts.ReaderImpl - implements PackedInts.Mutable { +abstract class Packed64SingleBlock extends PackedInts.MutableImpl { + + public static final int MAX_SUPPORTED_BITS_PER_VALUE = 32; + private static final int[] SUPPORTED_BITS_PER_VALUE = new int[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 16, 21, 32}; + + public static boolean isSupported(int bitsPerValue) { + return Arrays.binarySearch(SUPPORTED_BITS_PER_VALUE, bitsPerValue) >= 0; + } + + private static int requiredCapacity(int valueCount, int valuesPerBlock) { + return valueCount / valuesPerBlock + + (valueCount % valuesPerBlock == 0 ? 0 : 1); + } + + final long[] blocks; + + Packed64SingleBlock(int valueCount, int bitsPerValue) { + super(valueCount, bitsPerValue); + assert isSupported(bitsPerValue); + final int valuesPerBlock = 64 / bitsPerValue; + blocks = new long[requiredCapacity(valueCount, valuesPerBlock)]; + } + + @Override + public void clear() { + Arrays.fill(blocks, 0L); + } + + public long ramBytesUsed() { + return RamUsageEstimator.sizeOf(blocks); + } + + @Override + public int get(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + len = Math.min(len, valueCount - index); + assert off + len <= arr.length; + + final int originalIndex = index; + + // go to the next block boundary + final int valuesPerBlock = 64 / bitsPerValue; + final int offsetInBlock = index % valuesPerBlock; + if (offsetInBlock != 0) { + for (int i = offsetInBlock; i < valuesPerBlock && len > 0; ++i) { + arr[off++] = get(index++); + --len; + } + if (len == 0) { + return index - originalIndex; + } + } + + // bulk get + assert index % valuesPerBlock == 0; + final BulkOperation op = BulkOperation.of(PackedInts.Format.PACKED_SINGLE_BLOCK, bitsPerValue); + assert op.blocks() == 1; + assert op.values() == valuesPerBlock; + final int blockIndex = index / valuesPerBlock; + final int nblocks = (index + len) / valuesPerBlock - blockIndex; + op.get(blocks, blockIndex, arr, off, nblocks); + final int diff = nblocks * valuesPerBlock; + index += diff; len -= diff; + + if (index > originalIndex) { + // stay at the block boundary + return index - originalIndex; + } else { + // no progress so far => already at a block boundary but no full block to + // get + assert index == originalIndex; + return super.get(index, arr, off, len); + } + } - private static final int[] SUPPORTED_BITS_PER_VALUE = new int[] {1, 2, 3, 4, - 5, 6, 7, 9, 10, 12, 21}; - private static final long[][] WRITE_MASKS = new long[22][]; - private static final int[][] SHIFTS = new int[22][]; - static { - for (int bpv : SUPPORTED_BITS_PER_VALUE) { - initMasks(bpv); + @Override + public int set(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + len = Math.min(len, valueCount - index); + assert off + len <= arr.length; + + final int originalIndex = index; + + // go to the next block boundary + final int valuesPerBlock = 64 / bitsPerValue; + final int offsetInBlock = index % valuesPerBlock; + if (offsetInBlock != 0) { + for (int i = offsetInBlock; i < valuesPerBlock && len > 0; ++i) { + set(index++, arr[off++]); + --len; + } + if (len == 0) { + return index - originalIndex; + } + } + + // bulk set + assert index % valuesPerBlock == 0; + final BulkOperation op = BulkOperation.of(PackedInts.Format.PACKED_SINGLE_BLOCK, bitsPerValue); + assert op.blocks() == 1; + assert op.values() == valuesPerBlock; + final int blockIndex = index / valuesPerBlock; + final int nblocks = (index + len) / valuesPerBlock - blockIndex; + op.set(blocks, blockIndex, arr, off, nblocks); + final int diff = nblocks * valuesPerBlock; + index += diff; len -= diff; + + if (index > originalIndex) { + // stay at the block boundary + return index - originalIndex; + } else { + // no progress so far => already at a block boundary but no full block to + // set + assert index == originalIndex; + return super.set(index, arr, off, len); } } - protected static void initMasks(int bpv) { - int valuesPerBlock = Long.SIZE / bpv; - long[] writeMasks = new long[valuesPerBlock]; - int[] shifts = new int[valuesPerBlock]; - long bits = (1L << bpv) - 1; + @Override + public void fill(int fromIndex, int toIndex, long val) { + assert fromIndex >= 0; + assert fromIndex <= toIndex; + assert PackedInts.bitsRequired(val) <= bitsPerValue; + + final int valuesPerBlock = 64 / bitsPerValue; + if (toIndex - fromIndex <= valuesPerBlock << 1) { + // there needs to be at least one full block to set for the block + // approach to be worth trying + super.fill(fromIndex, toIndex, val); + return; + } + + // set values naively until the next block start + int fromOffsetInBlock = fromIndex % valuesPerBlock; + if (fromOffsetInBlock != 0) { + for (int i = fromOffsetInBlock; i < valuesPerBlock; ++i) { + set(fromIndex++, val); + } + assert fromIndex % valuesPerBlock == 0; + } + + // bulk set of the inner blocks + final int fromBlock = fromIndex / valuesPerBlock; + final int toBlock = toIndex / valuesPerBlock; + assert fromBlock * valuesPerBlock == fromIndex; + + long blockValue = 0L; for (int i = 0; i < valuesPerBlock; ++i) { - shifts[i] = bpv * i; - writeMasks[i] = ~(bits << shifts[i]); + blockValue = blockValue | (val << (i * bitsPerValue)); + } + Arrays.fill(blocks, fromBlock, toBlock, blockValue); + + // fill the gap + for (int i = valuesPerBlock * toBlock; i < toIndex; ++i) { + set(i, val); + } + } + + @Override + protected PackedInts.Format getFormat() { + return PackedInts.Format.PACKED_SINGLE_BLOCK; + } + + @Override + public String toString() { + return getClass().getSimpleName() + "(bitsPerValue=" + bitsPerValue + + ", size=" + size() + ", elements.length=" + blocks.length + ")"; + } + + public static Packed64SingleBlock create(DataInput in, + int valueCount, int bitsPerValue) throws IOException { + Packed64SingleBlock reader = create(valueCount, bitsPerValue); + for (int i = 0; i < reader.blocks.length; ++i) { + reader.blocks[i] = in.readLong(); } - WRITE_MASKS[bpv] = writeMasks; - SHIFTS[bpv] = shifts; + return reader; } public static Packed64SingleBlock create(int valueCount, int bitsPerValue) { @@ -70,296 +226,359 @@ abstract class Packed64SingleBlock extends PackedInts.ReaderImpl return new Packed64SingleBlock6(valueCount); case 7: return new Packed64SingleBlock7(valueCount); + case 8: + return new Packed64SingleBlock8(valueCount); case 9: return new Packed64SingleBlock9(valueCount); case 10: return new Packed64SingleBlock10(valueCount); case 12: return new Packed64SingleBlock12(valueCount); + case 16: + return new Packed64SingleBlock16(valueCount); case 21: return new Packed64SingleBlock21(valueCount); + case 32: + return new Packed64SingleBlock32(valueCount); default: - throw new IllegalArgumentException("Unsupported bitsPerValue: " - + bitsPerValue); + throw new IllegalArgumentException("Unsupported number of bits per value: " + 32); } } - public static Packed64SingleBlock create(DataInput in, - int valueCount, int bitsPerValue) throws IOException { - Packed64SingleBlock reader = create(valueCount, bitsPerValue); - for (int i = 0; i < reader.blocks.length; ++i) { - reader.blocks[i] = in.readLong(); - } - return reader; - } + static class Packed64SingleBlock1 extends Packed64SingleBlock { - public static boolean isSupported(int bitsPerValue) { - return Arrays.binarySearch(SUPPORTED_BITS_PER_VALUE, bitsPerValue) >= 0; - } + Packed64SingleBlock1(int valueCount) { + super(valueCount, 1); + } - public static float overheadPerValue(int bitsPerValue) { - int valuesPerBlock = 64 / bitsPerValue; - int overhead = 64 % bitsPerValue; - return (float) overhead / valuesPerBlock; - } + @Override + public long get(int index) { + final int o = index >>> 6; + final int b = index & 63; + final int shift = b << 0; + return (blocks[o] >>> shift) & 1L; + } - protected final long[] blocks; - protected final int valuesPerBlock; - protected final int[] shifts; - protected final long[] writeMasks; - protected final long readMask; + @Override + public void set(int index, long value) { + final int o = index >>> 6; + final int b = index & 63; + final int shift = b << 0; + blocks[o] = (blocks[o] & ~(1L << shift)) | (value << shift); + } - Packed64SingleBlock(int valueCount, int bitsPerValue) { - super(valueCount, bitsPerValue); - valuesPerBlock = Long.SIZE / bitsPerValue; - blocks = new long[requiredCapacity(valueCount, valuesPerBlock)]; - shifts = SHIFTS[bitsPerValue]; - writeMasks = WRITE_MASKS[bitsPerValue]; - readMask = ~writeMasks[0]; } - private static int requiredCapacity(int valueCount, int valuesPerBlock) { - return valueCount / valuesPerBlock - + (valueCount % valuesPerBlock == 0 ? 0 : 1); - } + static class Packed64SingleBlock2 extends Packed64SingleBlock { - protected int blockOffset(int offset) { - return offset / valuesPerBlock; - } + Packed64SingleBlock2(int valueCount) { + super(valueCount, 2); + } - protected int offsetInBlock(int offset) { - return offset % valuesPerBlock; - } + @Override + public long get(int index) { + final int o = index >>> 5; + final int b = index & 31; + final int shift = b << 1; + return (blocks[o] >>> shift) & 3L; + } - @Override - public long get(int index) { - final int o = blockOffset(index); - final int b = offsetInBlock(index); + @Override + public void set(int index, long value) { + final int o = index >>> 5; + final int b = index & 31; + final int shift = b << 1; + blocks[o] = (blocks[o] & ~(3L << shift)) | (value << shift); + } - return (blocks[o] >> shifts[b]) & readMask; } - @Override - public void set(int index, long value) { - final int o = blockOffset(index); - final int b = offsetInBlock(index); + static class Packed64SingleBlock3 extends Packed64SingleBlock { - blocks[o] = (blocks[o] & writeMasks[b]) | (value << shifts[b]); - } + Packed64SingleBlock3(int valueCount) { + super(valueCount, 3); + } - @Override - public void clear() { - Arrays.fill(blocks, 0L); - } + @Override + public long get(int index) { + final int o = index / 21; + final int b = index % 21; + final int shift = b * 3; + return (blocks[o] >>> shift) & 7L; + } - public long ramBytesUsed() { - return RamUsageEstimator.sizeOf(blocks); - } + @Override + public void set(int index, long value) { + final int o = index / 21; + final int b = index % 21; + final int shift = b * 3; + blocks[o] = (blocks[o] & ~(7L << shift)) | (value << shift); + } - @Override - public String toString() { - return getClass().getSimpleName() + "(bitsPerValue=" + bitsPerValue - + ", size=" + size() + ", elements.length=" + blocks.length + ")"; } - // Specialisations that allow the JVM to optimize computation of the block - // offset as well as the offset in block - - static final class Packed64SingleBlock21 extends Packed64SingleBlock { + static class Packed64SingleBlock4 extends Packed64SingleBlock { - Packed64SingleBlock21(int valueCount) { - super(valueCount, 21); - assert valuesPerBlock == 3; + Packed64SingleBlock4(int valueCount) { + super(valueCount, 4); } @Override - protected int blockOffset(int offset) { - return offset / 3; + public long get(int index) { + final int o = index >>> 4; + final int b = index & 15; + final int shift = b << 2; + return (blocks[o] >>> shift) & 15L; } @Override - protected int offsetInBlock(int offset) { - return offset % 3; + public void set(int index, long value) { + final int o = index >>> 4; + final int b = index & 15; + final int shift = b << 2; + blocks[o] = (blocks[o] & ~(15L << shift)) | (value << shift); } + } - static final class Packed64SingleBlock12 extends Packed64SingleBlock { + static class Packed64SingleBlock5 extends Packed64SingleBlock { - Packed64SingleBlock12(int valueCount) { - super(valueCount, 12); - assert valuesPerBlock == 5; + Packed64SingleBlock5(int valueCount) { + super(valueCount, 5); } @Override - protected int blockOffset(int offset) { - return offset / 5; + public long get(int index) { + final int o = index / 12; + final int b = index % 12; + final int shift = b * 5; + return (blocks[o] >>> shift) & 31L; } @Override - protected int offsetInBlock(int offset) { - return offset % 5; + public void set(int index, long value) { + final int o = index / 12; + final int b = index % 12; + final int shift = b * 5; + blocks[o] = (blocks[o] & ~(31L << shift)) | (value << shift); } + } - static final class Packed64SingleBlock10 extends Packed64SingleBlock { + static class Packed64SingleBlock6 extends Packed64SingleBlock { - Packed64SingleBlock10(int valueCount) { - super(valueCount, 10); - assert valuesPerBlock == 6; + Packed64SingleBlock6(int valueCount) { + super(valueCount, 6); } @Override - protected int blockOffset(int offset) { - return offset / 6; + public long get(int index) { + final int o = index / 10; + final int b = index % 10; + final int shift = b * 6; + return (blocks[o] >>> shift) & 63L; } @Override - protected int offsetInBlock(int offset) { - return offset % 6; + public void set(int index, long value) { + final int o = index / 10; + final int b = index % 10; + final int shift = b * 6; + blocks[o] = (blocks[o] & ~(63L << shift)) | (value << shift); } + } - static final class Packed64SingleBlock9 extends Packed64SingleBlock { + static class Packed64SingleBlock7 extends Packed64SingleBlock { - Packed64SingleBlock9(int valueCount) { - super(valueCount, 9); - assert valuesPerBlock == 7; + Packed64SingleBlock7(int valueCount) { + super(valueCount, 7); } @Override - protected int blockOffset(int offset) { - return offset / 7; + public long get(int index) { + final int o = index / 9; + final int b = index % 9; + final int shift = b * 7; + return (blocks[o] >>> shift) & 127L; } @Override - protected int offsetInBlock(int offset) { - return offset % 7; + public void set(int index, long value) { + final int o = index / 9; + final int b = index % 9; + final int shift = b * 7; + blocks[o] = (blocks[o] & ~(127L << shift)) | (value << shift); } + } - static final class Packed64SingleBlock7 extends Packed64SingleBlock { + static class Packed64SingleBlock8 extends Packed64SingleBlock { - Packed64SingleBlock7(int valueCount) { - super(valueCount, 7); - assert valuesPerBlock == 9; + Packed64SingleBlock8(int valueCount) { + super(valueCount, 8); } @Override - protected int blockOffset(int offset) { - return offset / 9; + public long get(int index) { + final int o = index >>> 3; + final int b = index & 7; + final int shift = b << 3; + return (blocks[o] >>> shift) & 255L; } @Override - protected int offsetInBlock(int offset) { - return offset % 9; + public void set(int index, long value) { + final int o = index >>> 3; + final int b = index & 7; + final int shift = b << 3; + blocks[o] = (blocks[o] & ~(255L << shift)) | (value << shift); } + } - static final class Packed64SingleBlock6 extends Packed64SingleBlock { + static class Packed64SingleBlock9 extends Packed64SingleBlock { - Packed64SingleBlock6(int valueCount) { - super(valueCount, 6); - assert valuesPerBlock == 10; + Packed64SingleBlock9(int valueCount) { + super(valueCount, 9); } @Override - protected int blockOffset(int offset) { - return offset / 10; + public long get(int index) { + final int o = index / 7; + final int b = index % 7; + final int shift = b * 9; + return (blocks[o] >>> shift) & 511L; } @Override - protected int offsetInBlock(int offset) { - return offset % 10; + public void set(int index, long value) { + final int o = index / 7; + final int b = index % 7; + final int shift = b * 9; + blocks[o] = (blocks[o] & ~(511L << shift)) | (value << shift); } + } - static final class Packed64SingleBlock5 extends Packed64SingleBlock { + static class Packed64SingleBlock10 extends Packed64SingleBlock { - Packed64SingleBlock5(int valueCount) { - super(valueCount, 5); - assert valuesPerBlock == 12; + Packed64SingleBlock10(int valueCount) { + super(valueCount, 10); } @Override - protected int blockOffset(int offset) { - return offset / 12; + public long get(int index) { + final int o = index / 6; + final int b = index % 6; + final int shift = b * 10; + return (blocks[o] >>> shift) & 1023L; } @Override - protected int offsetInBlock(int offset) { - return offset % 12; + public void set(int index, long value) { + final int o = index / 6; + final int b = index % 6; + final int shift = b * 10; + blocks[o] = (blocks[o] & ~(1023L << shift)) | (value << shift); } + } - static final class Packed64SingleBlock4 extends Packed64SingleBlock { + static class Packed64SingleBlock12 extends Packed64SingleBlock { - Packed64SingleBlock4(int valueCount) { - super(valueCount, 4); - assert valuesPerBlock == 16; + Packed64SingleBlock12(int valueCount) { + super(valueCount, 12); } @Override - protected int blockOffset(int offset) { - return offset >> 4; + public long get(int index) { + final int o = index / 5; + final int b = index % 5; + final int shift = b * 12; + return (blocks[o] >>> shift) & 4095L; } @Override - protected int offsetInBlock(int offset) { - return offset & 15; + public void set(int index, long value) { + final int o = index / 5; + final int b = index % 5; + final int shift = b * 12; + blocks[o] = (blocks[o] & ~(4095L << shift)) | (value << shift); } + } - static final class Packed64SingleBlock3 extends Packed64SingleBlock { + static class Packed64SingleBlock16 extends Packed64SingleBlock { - Packed64SingleBlock3(int valueCount) { - super(valueCount, 3); - assert valuesPerBlock == 21; + Packed64SingleBlock16(int valueCount) { + super(valueCount, 16); } @Override - protected int blockOffset(int offset) { - return offset / 21; + public long get(int index) { + final int o = index >>> 2; + final int b = index & 3; + final int shift = b << 4; + return (blocks[o] >>> shift) & 65535L; } @Override - protected int offsetInBlock(int offset) { - return offset % 21; + public void set(int index, long value) { + final int o = index >>> 2; + final int b = index & 3; + final int shift = b << 4; + blocks[o] = (blocks[o] & ~(65535L << shift)) | (value << shift); } + } - static final class Packed64SingleBlock2 extends Packed64SingleBlock { + static class Packed64SingleBlock21 extends Packed64SingleBlock { - Packed64SingleBlock2(int valueCount) { - super(valueCount, 2); - assert valuesPerBlock == 32; + Packed64SingleBlock21(int valueCount) { + super(valueCount, 21); } @Override - protected int blockOffset(int offset) { - return offset >> 5; + public long get(int index) { + final int o = index / 3; + final int b = index % 3; + final int shift = b * 21; + return (blocks[o] >>> shift) & 2097151L; } @Override - protected int offsetInBlock(int offset) { - return offset & 31; + public void set(int index, long value) { + final int o = index / 3; + final int b = index % 3; + final int shift = b * 21; + blocks[o] = (blocks[o] & ~(2097151L << shift)) | (value << shift); } + } - static final class Packed64SingleBlock1 extends Packed64SingleBlock { + static class Packed64SingleBlock32 extends Packed64SingleBlock { - Packed64SingleBlock1(int valueCount) { - super(valueCount, 1); - assert valuesPerBlock == 64; + Packed64SingleBlock32(int valueCount) { + super(valueCount, 32); } @Override - protected int blockOffset(int offset) { - return offset >> 6; + public long get(int index) { + final int o = index >>> 1; + final int b = index & 1; + final int shift = b << 5; + return (blocks[o] >>> shift) & 4294967295L; } @Override - protected int offsetInBlock(int offset) { - return offset & 63; + public void set(int index, long value) { + final int o = index >>> 1; + final int b = index & 1; + final int shift = b << 5; + blocks[o] = (blocks[o] & ~(4294967295L << shift)) | (value << shift); } + } + } \ No newline at end of file diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlockReaderIterator.java b/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlockReaderIterator.java deleted file mode 100644 index a661771..0000000 --- a/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlockReaderIterator.java +++ /dev/null @@ -1,88 +0,0 @@ -package org.apache.lucene.util.packed; - -import java.io.IOException; - -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.packed.PackedInts.ReaderIteratorImpl; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -final class Packed64SingleBlockReaderIterator extends ReaderIteratorImpl { - - private long pending; - private int shift; - private final long mask; - private int position; - - Packed64SingleBlockReaderIterator(int valueCount, int bitsPerValue, IndexInput in) - throws IOException { - super(valueCount, bitsPerValue, in); - pending = 0; - shift = 64; - mask = ~(~0L << bitsPerValue); - position = -1; - } - - @Override - public long next() throws IOException { - if (shift + bitsPerValue > 64) { - pending = in.readLong(); - shift = 0; - } - final long next = (pending >>> shift) & mask; - shift += bitsPerValue; - ++position; - return next; - } - - @Override - public int ord() { - return position; - } - - @Override - public long advance(int ord) throws IOException { - assert ord < valueCount : "ord must be less than valueCount"; - assert ord > position : "ord must be greater than the current position"; - - final int valuesPerBlock = 64 / bitsPerValue; - final long nextBlock = (position + valuesPerBlock) / valuesPerBlock; - final long targetBlock = ord / valuesPerBlock; - final long blocksToSkip = targetBlock - nextBlock; - if (blocksToSkip > 0) { - final long skip = blocksToSkip << 3; - final long filePointer = in.getFilePointer(); - - in.seek(filePointer + skip); - shift = 64; - - final int offsetInBlock = ord % valuesPerBlock; - for (int i = 0; i < offsetInBlock; ++i) { - next(); - } - } else { - for (int i = position; i < ord - 1; ++i) { - next(); - } - } - - position = ord - 1; - return next(); - } - -} diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlockWriter.java b/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlockWriter.java deleted file mode 100644 index f01d34e..0000000 --- a/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlockWriter.java +++ /dev/null @@ -1,81 +0,0 @@ -package org.apache.lucene.util.packed; - -import java.io.IOException; - -import org.apache.lucene.store.DataOutput; -import org.apache.lucene.util.packed.PackedInts.Writer; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * A {@link Writer} for {@link Packed64SingleBlock} readers. - */ -final class Packed64SingleBlockWriter extends Writer { - - private long pending; - private int shift; - private int written; - - Packed64SingleBlockWriter(DataOutput out, int valueCount, - int bitsPerValue) throws IOException { - super(out, valueCount, bitsPerValue); - assert Packed64SingleBlock.isSupported(bitsPerValue) : bitsPerValue + " is not supported"; - pending = 0; - shift = 0; - written = 0; - } - - @Override - protected int getFormat() { - return PackedInts.PACKED_SINGLE_BLOCK; - } - - @Override - public void add(long v) throws IOException { - assert v <= PackedInts.maxValue(bitsPerValue) : "v=" + v - + " maxValue=" + PackedInts.maxValue(bitsPerValue); - assert v >= 0; - - if (shift + bitsPerValue > Long.SIZE) { - out.writeLong(pending); - pending = 0; - shift = 0; - } - pending |= v << shift; - shift += bitsPerValue; - ++written; - } - - @Override - public void finish() throws IOException { - while (written < valueCount) { - add(0L); // Auto flush - } - - if (shift > 0) { - // add was called at least once - out.writeLong(pending); - } - } - - @Override - public String toString() { - return "Packed64SingleBlockWriter(written " + written + "/" + valueCount + " with " - + bitsPerValue + " bits/value)"; - } -} \ No newline at end of file diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/Packed8ThreeBlocks.java b/lucene/core/src/java/org/apache/lucene/util/packed/Packed8ThreeBlocks.java index 75e912c..9ee624f 100644 --- a/lucene/core/src/java/org/apache/lucene/util/packed/Packed8ThreeBlocks.java +++ b/lucene/core/src/java/org/apache/lucene/util/packed/Packed8ThreeBlocks.java @@ -1,12 +1,8 @@ -package org.apache.lucene.util.packed; - -import java.io.IOException; -import java.util.Arrays; +// This file has been automatically generated, DO NOT EDIT -import org.apache.lucene.store.DataInput; -import org.apache.lucene.util.RamUsageEstimator; +package org.apache.lucene.util.packed; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,33 +19,38 @@ import org.apache.lucene.util.RamUsageEstimator; * limitations under the License. */ -/** 24 bitsPerValue backed by byte[] */ -final class Packed8ThreeBlocks extends PackedInts.ReaderImpl - implements PackedInts.Mutable { +import org.apache.lucene.store.DataInput; +import org.apache.lucene.util.RamUsageEstimator; - public static final int MAX_SIZE = Integer.MAX_VALUE / 3; +import java.io.IOException; +import java.util.Arrays; + +/** + * Packs integers into 3 bytes (24 bits per value). + * @lucene.internal + */ +final class Packed8ThreeBlocks extends PackedInts.MutableImpl { + final byte[] blocks; - private final byte[] blocks; + public static final int MAX_SIZE = Integer.MAX_VALUE / 3; Packed8ThreeBlocks(int valueCount) { super(valueCount, 24); if (valueCount > MAX_SIZE) { throw new ArrayIndexOutOfBoundsException("MAX_SIZE exceeded"); } - this.blocks = new byte[3 * valueCount]; + blocks = new byte[valueCount * 3]; } Packed8ThreeBlocks(DataInput in, int valueCount) throws IOException { this(valueCount); - for (int i = 0; i < blocks.length; i++) { + for (int i = 0; i < 3 * valueCount; ++i) { blocks[i] = in.readByte(); } final int mod = blocks.length % 8; if (mod != 0) { - final int pad = 8 - mod; - // round out long - for (int i = 0; i < pad; i++) { - in.readByte(); + for (int i = mod; i < 8; ++i) { + in.readByte(); } } } @@ -57,15 +58,56 @@ final class Packed8ThreeBlocks extends PackedInts.ReaderImpl @Override public long get(int index) { final int o = index * 3; - return (blocks[o] & 0xffL) << 16 | (blocks[o+1] & 0xffL) << 8 | (blocks[o+2] & 0xffL); + return (blocks[o] & 0xFFL) << 16 | (blocks[o+1] & 0xFFL) << 8 | (blocks[o+2] & 0xFFL); + } + + @Override + public int get(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + assert off + len <= arr.length; + + final int gets = Math.min(valueCount - index, len); + for (int i = index * 3, end = (index + gets) * 3; i < end; i+=3) { + arr[off++] = (blocks[i] & 0xFFL) << 16 | (blocks[i+1] & 0xFFL) << 8 | (blocks[i+2] & 0xFFL); + } + return gets; } @Override public void set(int index, long value) { final int o = index * 3; + blocks[o] = (byte) (value >>> 16); + blocks[o+1] = (byte) (value >>> 8); blocks[o+2] = (byte) value; - blocks[o+1] = (byte) (value >> 8); - blocks[o] = (byte) (value >> 16); + } + + @Override + public int set(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + assert off + len <= arr.length; + + final int sets = Math.min(valueCount - index, len); + for (int i = off, o = index * 3, end = off + sets; i < end; ++i) { + final long value = arr[i]; + blocks[o++] = (byte) (value >>> 16); + blocks[o++] = (byte) (value >>> 8); + blocks[o++] = (byte) value; + } + return sets; + } + + @Override + public void fill(int fromIndex, int toIndex, long val) { + final byte block1 = (byte) (val >>> 16); + final byte block2 = (byte) (val >>> 8); + final byte block3 = (byte) val; + for (int i = fromIndex * 3, end = toIndex * 3; i < end; i += 3) { + blocks[i] = block1; + blocks[i+1] = block2; + blocks[i+2] = block3; + } } @Override @@ -82,5 +124,4 @@ final class Packed8ThreeBlocks extends PackedInts.ReaderImpl return getClass().getSimpleName() + "(bitsPerValue=" + bitsPerValue + ", size=" + size() + ", elements.length=" + blocks.length + ")"; } - } diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java b/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java index 740f978..fd75946 100644 --- a/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java +++ b/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.packed; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,10 +19,11 @@ package org.apache.lucene.util.packed; import java.io.Closeable; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.store.DataInput; import org.apache.lucene.store.DataOutput; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.CodecUtil; +import org.apache.lucene.util.LongsRef; import java.io.IOException; @@ -34,7 +35,6 @@ import java.io.IOException; * * @lucene.internal */ - public class PackedInts { /** @@ -57,12 +57,189 @@ public class PackedInts { */ public static final float COMPACT = 0f; - private final static String CODEC_NAME = "PackedInts"; - private final static int VERSION_START = 0; - private final static int VERSION_CURRENT = VERSION_START; + /** + * Default amount of memory to use for bulk operations. + */ + public static final int DEFAULT_BUFFER_SIZE = 1024; // 1K - static final int PACKED = 0; - static final int PACKED_SINGLE_BLOCK = 1; + public final static String CODEC_NAME = "PackedInts"; + public final static int VERSION_START = 0; + public final static int VERSION_CURRENT = VERSION_START; + + /** + * A format to write packed ints. + * + * @lucene.internal + */ + public enum Format { + /** + * Compact format, all bits are written contiguously. + */ + PACKED(0) { + + @Override + public int nblocks(int bitsPerValue, int values) { + return (int) Math.ceil((double) values * bitsPerValue / 64); + } + + }, + + /** + * A format that may insert padding bits to improve encoding and decoding + * speed. Since this format doesn't support all possible bits per value, you + * should never use it directly, but rather use + * {@link PackedInts#fastestFormatAndBits(int, int, float)} to find the + * format that best suits your needs. + */ + PACKED_SINGLE_BLOCK(1) { + + @Override + public int nblocks(int bitsPerValue, int values) { + final int valuesPerBlock = 64 / bitsPerValue; + return (int) Math.ceil((double) values / valuesPerBlock); + } + + @Override + public boolean isSupported(int bitsPerValue) { + return Packed64SingleBlock.isSupported(bitsPerValue); + } + + @Override + public float overheadPerValue(int bitsPerValue) { + assert isSupported(bitsPerValue); + final int valuesPerBlock = 64 / bitsPerValue; + final int overhead = 64 % bitsPerValue; + return (float) overhead / valuesPerBlock; + } + + }; + + /** + * Get a format according to its ID. + */ + public static Format byId(int id) { + for (Format format : Format.values()) { + if (format.getId() == id) { + return format; + } + } + throw new IllegalArgumentException("Unknown format id: " + id); + } + + private Format(int id) { + this.id = id; + } + + public int id; + + /** + * Returns the ID of the format. + */ + public int getId() { + return id; + } + + /** + * Computes how many blocks are needed to store values values + * of size bitsPerValue. + */ + public abstract int nblocks(int bitsPerValue, int values); + + /** + * Tests whether the provided number of bits per value is supported by the + * format. + */ + public boolean isSupported(int bitsPerValue) { + return bitsPerValue >= 1 && bitsPerValue <= 64; + } + + /** + * Returns the overhead per value, in bits. + */ + public float overheadPerValue(int bitsPerValue) { + assert isSupported(bitsPerValue); + return 0f; + } + + /** + * Returns the overhead ratio (overhead per value / bits per value). + */ + public final float overheadRatio(int bitsPerValue) { + assert isSupported(bitsPerValue); + return overheadPerValue(bitsPerValue) / bitsPerValue; + } + } + + /** + * Simple class that holds a format and a number of bits per value. + */ + public static class FormatAndBits { + public final Format format; + public final int bitsPerValue; + public FormatAndBits(Format format, int bitsPerValue) { + this.format = format; + this.bitsPerValue = bitsPerValue; + } + } + + /** + * Try to find the {@link Format} and number of bits per value that would + * restore from disk the fastest reader whose overhead is less than + * acceptableOverheadRatio. + *

      + * The acceptableOverheadRatio parameter makes sense for + * random-access {@link Reader}s. In case you only plan to perform + * sequential access on this stream later on, you should probably use + * {@link PackedInts#COMPACT}. + *

      + * If you don't know how many values you are going to write, use + * valueCount = -1. + */ + public static FormatAndBits fastestFormatAndBits(int valueCount, int bitsPerValue, float acceptableOverheadRatio) { + if (valueCount == -1) { + valueCount = Integer.MAX_VALUE; + } + + acceptableOverheadRatio = Math.max(COMPACT, acceptableOverheadRatio); + acceptableOverheadRatio = Math.min(FASTEST, acceptableOverheadRatio); + float acceptableOverheadPerValue = acceptableOverheadRatio * bitsPerValue; // in bits + + int maxBitsPerValue = bitsPerValue + (int) acceptableOverheadPerValue; + + int actualBitsPerValue = -1; + Format format = Format.PACKED; + + if (bitsPerValue <= 8 && maxBitsPerValue >= 8) { + actualBitsPerValue = 8; + } else if (bitsPerValue <= 16 && maxBitsPerValue >= 16) { + actualBitsPerValue = 16; + } else if (bitsPerValue <= 32 && maxBitsPerValue >= 32) { + actualBitsPerValue = 32; + } else if (bitsPerValue <= 64 && maxBitsPerValue >= 64) { + actualBitsPerValue = 64; + } else if (valueCount <= Packed8ThreeBlocks.MAX_SIZE && bitsPerValue <= 24 && maxBitsPerValue >= 24) { + actualBitsPerValue = 24; + } else if (valueCount <= Packed16ThreeBlocks.MAX_SIZE && bitsPerValue <= 48 && maxBitsPerValue >= 48) { + actualBitsPerValue = 48; + } else { + for (int bpv = bitsPerValue; bpv <= maxBitsPerValue; ++bpv) { + if (Format.PACKED_SINGLE_BLOCK.isSupported(bpv)) { + float overhead = Format.PACKED_SINGLE_BLOCK.overheadPerValue(bpv); + float acceptableOverhead = acceptableOverheadPerValue + bitsPerValue - bpv; + if (overhead <= acceptableOverhead) { + actualBitsPerValue = bpv; + format = Format.PACKED_SINGLE_BLOCK; + break; + } + } + } + if (actualBitsPerValue < 0) { + actualBitsPerValue = bitsPerValue; + } + } + + return new FormatAndBits(format, actualBitsPerValue); + } /** * A read-only random access array of positive integers. @@ -76,6 +253,13 @@ public class PackedInts { long get(int index); /** + * Bulk get: read at least one and at most len longs starting + * from index into arr[off:off+len] and return + * the actual number of values that have been read. + */ + int get(int index, long[] arr, int off, int len); + + /** * @return the number of bits used to store any given value. * Note: This does not imply that memory usage is * {@code bitsPerValue * #values} as implementations are free to @@ -89,6 +273,11 @@ public class PackedInts { int size(); /** + * Return the in-memory size in bytes. + */ + long ramBytesUsed(); + + /** * Expert: if the bit-width of this reader matches one of * java's native types, returns the underlying array * (ie, byte[], short[], int[], long[]); else, returns @@ -106,6 +295,7 @@ public class PackedInts { * @see #getArray */ boolean hasArray(); + } /** @@ -114,31 +304,40 @@ public class PackedInts { public static interface ReaderIterator extends Closeable { /** Returns next value */ long next() throws IOException; + /** Returns at least 1 and at most count next values, + * the returned ref MUST NOT be modified */ + LongsRef next(int count) throws IOException; /** Returns number of bits per value */ int getBitsPerValue(); /** Returns number of values */ int size(); /** Returns the current position */ int ord(); - /** Skips to the given ordinal and returns its value. - * @return the value at the given position - * @throws IOException if reading the value throws an IOException*/ - long advance(int ord) throws IOException; } static abstract class ReaderIteratorImpl implements ReaderIterator { - protected final IndexInput in; + protected final DataInput in; protected final int bitsPerValue; protected final int valueCount; - protected ReaderIteratorImpl(int valueCount, int bitsPerValue, IndexInput in) { + protected ReaderIteratorImpl(int valueCount, int bitsPerValue, DataInput in) { this.in = in; this.bitsPerValue = bitsPerValue; this.valueCount = valueCount; } @Override + public long next() throws IOException { + LongsRef nextValues = next(1); + assert nextValues.length > 0; + final long result = nextValues.longs[nextValues.offset]; + ++nextValues.offset; + --nextValues.length; + return result; + } + + @Override public int getBitsPerValue() { return bitsPerValue; } @@ -150,7 +349,9 @@ public class PackedInts { @Override public void close() throws IOException { - in.close(); + if (in instanceof Closeable) { + ((Closeable) in).close(); + } } } @@ -159,6 +360,7 @@ public class PackedInts { * @lucene.internal */ public static interface Mutable extends Reader { + /** * Set the value at the given index in the array. * @param index where the value should be positioned. @@ -167,16 +369,38 @@ public class PackedInts { void set(int index, long value); /** + * Bulk set: set at least one and at most len longs starting + * at off in arr into this mutable, starting at + * index. Returns the actual number of values that have been + * set. + */ + int set(int index, long[] arr, int off, int len); + + /** + * Fill the mutable from fromIndex (inclusive) to + * toIndex (exclusive) with val. + */ + void fill(int fromIndex, int toIndex, long val); + + /** * Sets all values to 0. - */ + */ void clear(); + + /** + * Save this mutable into out. Instantiating a reader from + * the generated data will return a reader with the same number of bits + * per value. + */ + void save(DataOutput out) throws IOException; + } /** * A simple base for Readers that keeps track of valueCount and bitsPerValue. * @lucene.internal */ - public static abstract class ReaderImpl implements Reader { + static abstract class ReaderImpl implements Reader { protected final int bitsPerValue; protected final int valueCount; @@ -189,7 +413,7 @@ public class PackedInts { public int getBitsPerValue() { return bitsPerValue; } - + public int size() { return valueCount; } @@ -201,6 +425,61 @@ public class PackedInts { public boolean hasArray() { return false; } + + public int get(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + assert off + len <= arr.length; + + final int gets = Math.min(valueCount - index, len); + for (int i = index, o = off, end = index + gets; i < end; ++i, ++o) { + arr[o] = get(i); + } + return gets; + } + + } + + static abstract class MutableImpl extends ReaderImpl implements Mutable { + + protected MutableImpl(int valueCount, int bitsPerValue) { + super(valueCount, bitsPerValue); + } + + public int set(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + len = Math.min(len, valueCount - index); + assert off + len <= arr.length; + + for (int i = index, o = off, end = index + len; i < end; ++i, ++o) { + set(i, arr[o]); + } + return len; + } + + public void fill(int fromIndex, int toIndex, long val) { + assert val <= maxValue(bitsPerValue); + assert fromIndex <= toIndex; + for (int i = fromIndex; i < toIndex; ++i) { + set(i, val); + } + } + + protected Format getFormat() { + return Format.PACKED; + } + + @Override + public void save(DataOutput out) throws IOException { + Writer writer = getWriterNoHeader(out, getFormat(), + valueCount, bitsPerValue, DEFAULT_BUFFER_SIZE); + writer.writeHeader(); + for (int i = 0; i < valueCount; ++i) { + writer.add(get(i)); + } + writer.finish(); + } } /** A write-once Writer. @@ -208,121 +487,209 @@ public class PackedInts { */ public static abstract class Writer { protected final DataOutput out; - protected final int bitsPerValue; protected final int valueCount; + protected final int bitsPerValue; protected Writer(DataOutput out, int valueCount, int bitsPerValue) throws IOException { assert bitsPerValue <= 64; - + assert valueCount >= 0 || valueCount == -1; this.out = out; this.valueCount = valueCount; this.bitsPerValue = bitsPerValue; + } + + void writeHeader() throws IOException { + assert valueCount != -1; CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT); out.writeVInt(bitsPerValue); out.writeVInt(valueCount); - out.writeVInt(getFormat()); + out.writeVInt(getFormat().getId()); } - protected abstract int getFormat(); + /** The format used to serialize values. */ + protected abstract PackedInts.Format getFormat(); + + /** Add a value to the stream. */ public abstract void add(long v) throws IOException; + + /** The number of bits per value. */ + public final int bitsPerValue() { + return bitsPerValue; + } + + /** Perform end-of-stream operations. */ public abstract void finish() throws IOException; + + /** + * Returns the current ord in the stream (number of values that have been + * written so far minus one). + */ + public abstract int ord(); } /** - * Retrieve PackedInt data from the DataInput and return a packed int - * structure based on it. + * Expert: Restore a {@link Reader} from a stream without reading metadata at + * the beginning of the stream. This method is useful to restore data from + * streams which have been created using + * {@link PackedInts#getWriterNoHeader(DataOutput, Format, int, int, int)}. * - * @param in positioned at the beginning of a stored packed int structure. - * @return a read only random access capable array of positive integers. - * @throws IOException if the structure could not be retrieved. + * @param in the stream to read data from, positioned at the beginning of the packed values + * @param format the format used to serialize + * @param version the version used to serialize the data + * @param valueCount how many values the stream holds + * @param bitsPerValue the number of bits per value + * @return a Reader + * @throws IOException + * @see PackedInts#getWriterNoHeader(DataOutput, Format, int, int, int) * @lucene.internal */ - public static Reader getReader(DataInput in) throws IOException { - CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START); - final int bitsPerValue = in.readVInt(); - assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue; - final int valueCount = in.readVInt(); - final int format = in.readVInt(); - + public static Reader getReaderNoHeader(DataInput in, Format format, int version, + int valueCount, int bitsPerValue) throws IOException { switch (format) { + case PACKED_SINGLE_BLOCK: + return Packed64SingleBlock.create(in, valueCount, bitsPerValue); case PACKED: switch (bitsPerValue) { case 8: return new Direct8(in, valueCount); case 16: return new Direct16(in, valueCount); - case 24: - return new Packed8ThreeBlocks(in, valueCount); case 32: return new Direct32(in, valueCount); - case 48: - return new Packed16ThreeBlocks(in, valueCount); case 64: return new Direct64(in, valueCount); - default: - return new Packed64(in, valueCount, bitsPerValue); + case 24: + if (valueCount <= Packed8ThreeBlocks.MAX_SIZE) { + return new Packed8ThreeBlocks(in, valueCount); + } + break; + case 48: + if (valueCount <= Packed16ThreeBlocks.MAX_SIZE) { + return new Packed16ThreeBlocks(in, valueCount); + } + break; } - case PACKED_SINGLE_BLOCK: - return Packed64SingleBlock.create(in, valueCount, bitsPerValue); + return new Packed64(in, valueCount, bitsPerValue); default: throw new AssertionError("Unknwown Writer format: " + format); } } /** + * Restore a {@link Reader} from a stream. + * + * @param in the stream to read data from + * @return a Reader + * @throws IOException + * @lucene.internal + */ + public static Reader getReader(DataInput in) throws IOException { + final int version = CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START); + final int bitsPerValue = in.readVInt(); + assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue; + final int valueCount = in.readVInt(); + final Format format = Format.byId(in.readVInt()); + + return getReaderNoHeader(in, format, version, valueCount, bitsPerValue); + } + + /** + * Expert: Restore a {@link ReaderIterator} from a stream without reading + * metadata at the beginning of the stream. This method is useful to restore + * data from streams which have been created using + * {@link PackedInts#getWriterNoHeader(DataOutput, Format, int, int, int)}. + * + * @param in the stream to read data from, positioned at the beginning of the packed values + * @param format the format used to serialize + * @param version the version used to serialize the data + * @param valueCount how many values the stream holds + * @param bitsPerValue the number of bits per value + * @param mem how much memory the iterator is allowed to use to read-ahead (likely to speed up iteration) + * @return a ReaderIterator + * @throws IOException + * @see PackedInts#getWriterNoHeader(DataOutput, Format, int, int, int) + * @lucene.internal + */ + public static ReaderIterator getReaderIteratorNoHeader(DataInput in, Format format, int version, + int valueCount, int bitsPerValue, int mem) throws IOException { + return new PackedReaderIterator(format, valueCount, bitsPerValue, in, mem); + } + + /** * Retrieve PackedInts as a {@link ReaderIterator} * @param in positioned at the beginning of a stored packed int structure. + * @param mem how much memory the iterator is allowed to use to read-ahead (likely to speed up iteration) * @return an iterator to access the values * @throws IOException if the structure could not be retrieved. * @lucene.internal */ - public static ReaderIterator getReaderIterator(IndexInput in) throws IOException { - CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START); + public static ReaderIterator getReaderIterator(DataInput in, int mem) throws IOException { + final int version = CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START); final int bitsPerValue = in.readVInt(); assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue; final int valueCount = in.readVInt(); - final int format = in.readVInt(); + final Format format = Format.byId(in.readVInt()); + return getReaderIteratorNoHeader(in, format, version, valueCount, bitsPerValue, mem); + } + + /** + * Expert: Construct a direct {@link Reader} from a stream without reading + * metadata at the beginning of the stream. This method is useful to restore + * data from streams which have been created using + * {@link PackedInts#getWriterNoHeader(DataOutput, Format, int, int, int)}. + *

      + * The returned reader will have very little memory overhead, but every call + * to {@link Reader#get(int)} is likely to perform a disk seek. + * + * @param in the stream to read data from + * @param format the format used to serialize + * @param version the version used to serialize the data + * @param valueCount how many values the stream holds + * @param bitsPerValue the number of bits per value + * @return a direct Reader + * @throws IOException + * @lucene.internal + */ + public static Reader getDirectReaderNoHeader(IndexInput in, Format format, + int version, int valueCount, int bitsPerValue) throws IOException { switch (format) { case PACKED: - return new PackedReaderIterator(valueCount, bitsPerValue, in); + return new DirectPackedReader(bitsPerValue, valueCount, in); case PACKED_SINGLE_BLOCK: - return new Packed64SingleBlockReaderIterator(valueCount, bitsPerValue, in); + return new DirectPacked64SingleBlockReader(bitsPerValue, valueCount, in); default: - throw new AssertionError("Unknwown Writer format: " + format); + throw new AssertionError("Unknwown format: " + format); } } - + /** - * Retrieve PackedInts.Reader that does not load values - * into RAM but rather accesses all values via the - * provided IndexInput. - * @param in positioned at the beginning of a stored packed int structure. - * @return an Reader to access the values - * @throws IOException if the structure could not be retrieved. + * Construct a direct {@link Reader} from an {@link IndexInput}. This method + * is useful to restore data from streams which have been created using + * {@link PackedInts#getWriter(DataOutput, int, int, float)}. + *

      + * The returned reader will have very little memory overhead, but every call + * to {@link Reader#get(int)} is likely to perform a disk seek. + * + * @param in the stream to read data from + * @return a direct Reader + * @throws IOException * @lucene.internal */ public static Reader getDirectReader(IndexInput in) throws IOException { - CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START); + final int version = CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START); final int bitsPerValue = in.readVInt(); assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue; final int valueCount = in.readVInt(); - final int format = in.readVInt(); - switch (format) { - case PACKED: - return new DirectPackedReader(bitsPerValue, valueCount, in); - case PACKED_SINGLE_BLOCK: - return new DirectPacked64SingleBlockReader(bitsPerValue, valueCount, in); - default: - throw new AssertionError("Unknwown Writer format: " + format); - } + final Format format = Format.byId(in.readVInt()); + return getDirectReaderNoHeader(in, format, version, valueCount, bitsPerValue); } /** * Create a packed integer array with the given amount of values initialized * to 0. the valueCount and the bitsPerValue cannot be changed after creation. * All Mutables known by this factory are kept fully in RAM. - * + *

      * Positive values of acceptableOverheadRatio will trade space * for speed by selecting a faster but potentially less memory-efficient * implementation. An acceptableOverheadRatio of @@ -335,98 +702,134 @@ public class PackedInts { * @param acceptableOverheadRatio an acceptable overhead * ratio per value * @return a mutable packed integer array - * @throws java.io.IOException if the Mutable could not be created. With the - * current implementations, this never happens, but the method - * signature allows for future persistence-backed Mutables. * @lucene.internal */ public static Mutable getMutable(int valueCount, int bitsPerValue, float acceptableOverheadRatio) { - acceptableOverheadRatio = Math.max(COMPACT, acceptableOverheadRatio); - acceptableOverheadRatio = Math.min(FASTEST, acceptableOverheadRatio); - float acceptableOverheadPerValue = acceptableOverheadRatio * bitsPerValue; // in bits + assert valueCount >= 0; - int maxBitsPerValue = bitsPerValue + (int) acceptableOverheadPerValue; - - if (bitsPerValue <= 8 && maxBitsPerValue >= 8) { - return new Direct8(valueCount); - } else if (bitsPerValue <= 16 && maxBitsPerValue >= 16) { - return new Direct16(valueCount); - } else if (bitsPerValue <= 32 && maxBitsPerValue >= 32) { - return new Direct32(valueCount); - } else if (bitsPerValue <= 64 && maxBitsPerValue >= 64) { - return new Direct64(valueCount); - } else if (valueCount <= Packed8ThreeBlocks.MAX_SIZE && bitsPerValue <= 24 && maxBitsPerValue >= 24) { - return new Packed8ThreeBlocks(valueCount); - } else if (valueCount <= Packed16ThreeBlocks.MAX_SIZE && bitsPerValue <= 48 && maxBitsPerValue >= 48) { - return new Packed16ThreeBlocks(valueCount); - } else { - for (int bpv = bitsPerValue; bpv <= maxBitsPerValue; ++bpv) { - if (Packed64SingleBlock.isSupported(bpv)) { - float overhead = Packed64SingleBlock.overheadPerValue(bpv); - float acceptableOverhead = acceptableOverheadPerValue + bitsPerValue - bpv; - if (overhead <= acceptableOverhead) { - return Packed64SingleBlock.create(valueCount, bpv); - } + final FormatAndBits formatAndBits = fastestFormatAndBits(valueCount, bitsPerValue, acceptableOverheadRatio); + switch (formatAndBits.format) { + case PACKED_SINGLE_BLOCK: + return Packed64SingleBlock.create(valueCount, formatAndBits.bitsPerValue); + case PACKED: + switch (formatAndBits.bitsPerValue) { + case 8: + return new Direct8(valueCount); + case 16: + return new Direct16(valueCount); + case 32: + return new Direct32(valueCount); + case 64: + return new Direct64(valueCount); + case 24: + if (valueCount <= Packed8ThreeBlocks.MAX_SIZE) { + return new Packed8ThreeBlocks(valueCount); + } + break; + case 48: + if (valueCount <= Packed16ThreeBlocks.MAX_SIZE) { + return new Packed16ThreeBlocks(valueCount); + } + break; } - } - return new Packed64(valueCount, bitsPerValue); + return new Packed64(valueCount, formatAndBits.bitsPerValue); + default: + throw new AssertionError(); } } /** - * Create a packed integer array writer for the given number of values at the - * given bits/value. Writers append to the given IndexOutput and has very - * low memory overhead. + * Expert: Create a packed integer array writer for the given output, format, + * value count, and number of bits per value. + *

      + * The resulting stream will be long-aligned. This means that depending on + * the format which is used, up to 63 bits will be wasted. An easy way to + * make sure that no space is lost is to always use a valueCount + * that is a multiple of 64. + *

      + * This method does not write any metadata to the stream, meaning that it is + * your responsibility to store it somewhere else in order to be able to + * recover data from the stream later on: + *

        + *
      • format (using {@link Format#getId()}),
        • + *
      • valueCount,
        • + *
      • bitsPerValue,
        • + *
      • {@link #VERSION_CURRENT}.
        • + *
      + *

      + * It is possible to start writing values without knowing how many of them you + * are actually going to write. To do this, just pass -1 as + * valueCount. On the other hand, for any positive value of + * valueCount, the returned writer will make sure that you don't + * write more values than expected and pad the end of stream with zeros in + * case you have written less than valueCount when calling + * {@link Writer#finish()}. + *

      + * The mem parameter lets you control how much memory can be used + * to buffer changes in memory before flushing to disk. High values of + * mem are likely to improve throughput. On the other hand, if + * speed is not that important to you, a value of 0 will use as + * little memory as possible and should already offer reasonable throughput. * - * Positive values of acceptableOverheadRatio will trade space + * @param out the data output + * @param format the format to use to serialize the values + * @param valueCount the number of values + * @param bitsPerValue the number of bits per value + * @param mem how much memory (in bytes) can be used to speed up serialization + * @return a Writer + * @throws IOException + * @see PackedInts#getReaderIteratorNoHeader(DataInput, Format, int, int, int, int) + * @see PackedInts#getReaderNoHeader(DataInput, Format, int, int, int) + * @lucene.internal + */ + public static Writer getWriterNoHeader( + DataOutput out, Format format, int valueCount, int bitsPerValue, int mem) throws IOException { + return new PackedWriter(format, out, valueCount, bitsPerValue, mem); + } + + /** + * Create a packed integer array writer for the given output, format, value + * count, and number of bits per value. + *

      + * The resulting stream will be long-aligned. This means that depending on + * the format which is used under the hoods, up to 63 bits will be wasted. + * An easy way to make sure that no space is lost is to always use a + * valueCount that is a multiple of 64. + *

      + * This method writes metadata to the stream, so that the resulting stream is + * sufficient to restore a {@link Reader} from it. You don't need to track + * valueCount or bitsPerValue by yourself. In case + * this is a problem, you should probably look at + * {@link #getWriterNoHeader(DataOutput, Format, int, int, int)}. + *

      + * The acceptableOverheadRatio parameter controls how + * readers that will be restored from this stream trade space * for speed by selecting a faster but potentially less memory-efficient * implementation. An acceptableOverheadRatio of * {@link PackedInts#COMPACT} will make sure that the most memory-efficient * implementation is selected whereas {@link PackedInts#FASTEST} will make sure - * that the fastest implementation is selected. + * that the fastest implementation is selected. In case you are only interested + * in reading this stream sequentially later on, you should probably use + * {@link PackedInts#COMPACT}. * - * @param out the destination for the produced bits. - * @param valueCount the number of elements. - * @param bitsPerValue the number of bits available for any given value. + * @param out the data output + * @param valueCount the number of values + * @param bitsPerValue the number of bits per value * @param acceptableOverheadRatio an acceptable overhead ratio per value - * @return a Writer ready for receiving values. - * @throws IOException if bits could not be written to out. + * @return a Writer + * @throws IOException * @lucene.internal */ public static Writer getWriter(DataOutput out, int valueCount, int bitsPerValue, float acceptableOverheadRatio) throws IOException { - acceptableOverheadRatio = Math.max(COMPACT, acceptableOverheadRatio); - acceptableOverheadRatio = Math.min(FASTEST, acceptableOverheadRatio); - float acceptableOverheadPerValue = acceptableOverheadRatio * bitsPerValue; // in bits - - int maxBitsPerValue = bitsPerValue + (int) acceptableOverheadPerValue; + assert valueCount >= 0; - if (bitsPerValue <= 8 && maxBitsPerValue >= 8) { - return new PackedWriter(out, valueCount, 8); - } else if (bitsPerValue <= 16 && maxBitsPerValue >= 16) { - return new PackedWriter(out, valueCount, 16); - } else if (bitsPerValue <= 32 && maxBitsPerValue >= 32) { - return new PackedWriter(out, valueCount, 32); - } else if (bitsPerValue <= 64 && maxBitsPerValue >= 64) { - return new PackedWriter(out, valueCount, 64); - } else if (valueCount <= Packed8ThreeBlocks.MAX_SIZE && bitsPerValue <= 24 && maxBitsPerValue >= 24) { - return new PackedWriter(out, valueCount, 24); - } else if (valueCount <= Packed16ThreeBlocks.MAX_SIZE && bitsPerValue <= 48 && maxBitsPerValue >= 48) { - return new PackedWriter(out, valueCount, bitsPerValue); - } else { - for (int bpv = bitsPerValue; bpv <= maxBitsPerValue; ++bpv) { - if (Packed64SingleBlock.isSupported(bpv)) { - float overhead = Packed64SingleBlock.overheadPerValue(bpv); - float acceptableOverhead = acceptableOverheadPerValue + bitsPerValue - bpv; - if (overhead <= acceptableOverhead) { - return new Packed64SingleBlockWriter(out, valueCount, bpv); - } - } - } - return new PackedWriter(out, valueCount, bitsPerValue); - } + final FormatAndBits formatAndBits = fastestFormatAndBits(valueCount, bitsPerValue, acceptableOverheadRatio); + final Writer writer = getWriterNoHeader(out, formatAndBits.format, valueCount, formatAndBits.bitsPerValue, DEFAULT_BUFFER_SIZE); + writer.writeHeader(); + return writer; } /** Returns how many bits are required to hold values up @@ -452,4 +855,43 @@ public class PackedInts { public static long maxValue(int bitsPerValue) { return bitsPerValue == 64 ? Long.MAX_VALUE : ~(~0L << bitsPerValue); } + + /** + * Copy src[srcPos:srcPos+len] into + * dest[destPos:destPos+len] using at most mem + * bytes. + */ + public static void copy(Reader src, int srcPos, Mutable dest, int destPos, int len, int mem) { + assert srcPos + len <= src.size(); + assert destPos + len <= dest.size(); + final int capacity = mem >>> 3; + if (capacity == 0) { + for (int i = 0; i < len; ++i) { + dest.set(destPos++, src.get(srcPos++)); + } + } else { + // use bulk operations + long[] buf = new long[Math.min(capacity, len)]; + int remaining = 0; + while (len > 0) { + final int read = src.get(srcPos, buf, remaining, Math.min(len, buf.length - remaining)); + assert read > 0; + srcPos += read; + len -= read; + remaining += read; + final int written = dest.set(destPos, buf, 0, remaining); + assert written > 0; + destPos += written; + if (written < remaining) { + System.arraycopy(buf, written, buf, 0, remaining - written); + } + remaining -= written; + } + while (remaining > 0) { + final int written = dest.set(destPos, buf, 0, remaining); + remaining -= written; + } + } + } + } diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java b/lucene/core/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java index 49f3e6d..a6ba222 100644 --- a/lucene/core/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java +++ b/lucene/core/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.packed; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,77 +17,71 @@ package org.apache.lucene.util.packed; * limitations under the License. */ -import org.apache.lucene.store.IndexInput; - +import java.io.EOFException; import java.io.IOException; +import org.apache.lucene.store.DataInput; +import org.apache.lucene.util.LongsRef; + final class PackedReaderIterator extends PackedInts.ReaderIteratorImpl { - private long pending; - private int pendingBitsLeft; - private int position = -1; - // masks[n-1] masks for bottom n bits - private final long[] masks; + final PackedInts.Format format; + final BulkOperation bulkOperation; + final long[] nextBlocks; + final LongsRef nextValues; + final int iterations; + int position; - public PackedReaderIterator(int valueCount, int bitsPerValue, IndexInput in) - throws IOException { + PackedReaderIterator(PackedInts.Format format, int valueCount, int bitsPerValue, DataInput in, int mem) { super(valueCount, bitsPerValue, in); + this.format = format; + bulkOperation = BulkOperation.of(format, bitsPerValue); + iterations = bulkOperation.computeIterations(valueCount, mem); + assert iterations > 0; + nextBlocks = new long[iterations * bulkOperation.blocks()]; + nextValues = new LongsRef(new long[iterations * bulkOperation.values()], 0, 0); + assert iterations * bulkOperation.values() == nextValues.longs.length; + assert iterations * bulkOperation.blocks() == nextBlocks.length; + nextValues.offset = nextValues.longs.length; + position = -1; + } - masks = new long[bitsPerValue]; + @Override + public LongsRef next(int count) throws IOException { + assert nextValues.length >= 0; + assert count > 0; + assert nextValues.offset + nextValues.length <= nextValues.longs.length; + + nextValues.offset += nextValues.length; - long v = 1; - for (int i = 0; i < bitsPerValue; i++) { - v *= 2; - masks[i] = v - 1; + final int remaining = valueCount - position - 1; + if (remaining <= 0) { + throw new EOFException(); } - } + count = Math.min(remaining, count); - public long next() throws IOException { - if (pendingBitsLeft == 0) { - pending = in.readLong(); - pendingBitsLeft = 64; - } - - final long result; - if (pendingBitsLeft >= bitsPerValue) { // not split - result = (pending >> (pendingBitsLeft - bitsPerValue)) & masks[bitsPerValue-1]; - pendingBitsLeft -= bitsPerValue; - } else { // split - final int bits1 = bitsPerValue - pendingBitsLeft; - final long result1 = (pending & masks[pendingBitsLeft-1]) << bits1; - pending = in.readLong(); - final long result2 = (pending >> (64 - bits1)) & masks[bits1-1]; - pendingBitsLeft = 64 + pendingBitsLeft - bitsPerValue; - result = result1 | result2; + if (nextValues.offset == nextValues.longs.length) { + final int remainingBlocks = format.nblocks(bitsPerValue, remaining); + final int blocksToRead = Math.min(remainingBlocks, nextBlocks.length); + for (int i = 0; i < blocksToRead; ++i) { + nextBlocks[i] = in.readLong(); + } + for (int i = blocksToRead; i < nextBlocks.length; ++i) { + nextBlocks[i] = 0L; + } + + bulkOperation.get(nextBlocks, 0, nextValues.longs, 0, iterations); + nextValues.offset = 0; } - - ++position; - return result; + + nextValues.length = Math.min(nextValues.longs.length - nextValues.offset, count); + position += nextValues.length; + return nextValues; } + @Override public int ord() { return position; } - public long advance(final int ord) throws IOException{ - assert ord < valueCount : "ord must be less than valueCount"; - assert ord > position : "ord must be greater than the current position"; - final long bits = (long) bitsPerValue; - final int posToSkip = ord - 1 - position; - final long bitsToSkip = (bits * (long)posToSkip); - if (bitsToSkip < pendingBitsLeft) { // enough bits left - no seek required - pendingBitsLeft -= bitsToSkip; - } else { - final long skip = bitsToSkip-pendingBitsLeft; - final long closestByte = (skip >> 6) << 3; - if (closestByte != 0) { // need to seek - final long filePointer = in.getFilePointer(); - in.seek(filePointer + closestByte); - } - pending = in.readLong(); - pendingBitsLeft = 64 - (int)(skip % 64); - } - position = ord-1; - return next(); - } } diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java b/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java index ae324b7..169f1e5 100644 --- a/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.packed; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,101 +19,79 @@ package org.apache.lucene.util.packed; import org.apache.lucene.store.DataOutput; +import java.io.EOFException; import java.io.IOException; // Packs high order byte first, to match // IndexOutput.writeInt/Long/Short byte order -/** - * Generic writer for space-optimal packed values. The resulting bits can be - * used directly by Packed32, Packed64 and PackedDirect* and will always be - * long-aligned. - */ - -class PackedWriter extends PackedInts.Writer { - private long pending; - private int pendingBitPos; +final class PackedWriter extends PackedInts.Writer { - // masks[n-1] masks for bottom n bits - private final long[] masks; - private int written = 0; + boolean finished; + final PackedInts.Format format; + final BulkOperation bulkOperation; + final long[] nextBlocks; + final long[] nextValues; + final int iterations; + int off; + int written; - public PackedWriter(DataOutput out, int valueCount, int bitsPerValue) - throws IOException { + PackedWriter(PackedInts.Format format, DataOutput out, int valueCount, int bitsPerValue, int mem) + throws IOException { super(out, valueCount, bitsPerValue); - - pendingBitPos = 64; - masks = new long[bitsPerValue - 1]; - - long v = 1; - for (int i = 0; i < bitsPerValue - 1; i++) { - v *= 2; - masks[i] = v - 1; - } + this.format = format; + bulkOperation = BulkOperation.of(format, bitsPerValue); + iterations = bulkOperation.computeIterations(valueCount, mem); + nextBlocks = new long[iterations * bulkOperation.blocks()]; + nextValues = new long[iterations * bulkOperation.values()]; + off = 0; + written = 0; + finished = false; } @Override - protected int getFormat() { - return PackedInts.PACKED; + protected PackedInts.Format getFormat() { + return format; } - /** - * Do not call this after finish - */ @Override public void add(long v) throws IOException { - assert v <= PackedInts.maxValue(bitsPerValue) : "v=" + v - + " maxValue=" + PackedInts.maxValue(bitsPerValue); - assert v >= 0; - //System.out.println(" packedw add v=" + v + " pendingBitPos=" + pendingBitPos); - - // TODO - if (pendingBitPos >= bitsPerValue) { - // not split - - // write-once, so we can |= w/o first masking to 0s - pending |= v << (pendingBitPos - bitsPerValue); - if (pendingBitPos == bitsPerValue) { - // flush - out.writeLong(pending); - pending = 0; - pendingBitPos = 64; - } else { - pendingBitPos -= bitsPerValue; - } - - } else { - // split - - // write top pendingBitPos bits of value into bottom bits of pending - pending |= (v >> (bitsPerValue - pendingBitPos)) & masks[pendingBitPos - 1]; - //System.out.println(" part1 (v >> " + (bitsPerValue - pendingBitPos) + ") & " + masks[pendingBitPos-1]); - - // flush - out.writeLong(pending); - - // write bottom (bitsPerValue - pendingBitPos) bits of value into top bits of pending - pendingBitPos = 64 - bitsPerValue + pendingBitPos; - //System.out.println(" part2 v << " + pendingBitPos); - pending = (v << pendingBitPos); + assert v >= 0 && v <= PackedInts.maxValue(bitsPerValue); + assert !finished; + if (valueCount != -1 && written >= valueCount) { + throw new EOFException("Writing past end of stream"); } - written++; + nextValues[off++] = v; + if (off == nextValues.length) { + flush(nextValues.length); + off = 0; + } + ++written; } @Override public void finish() throws IOException { - while (written < valueCount) { - add(0L); // Auto flush + assert !finished; + if (valueCount != -1) { + while (written < valueCount) { + add(0L); + } } + flush(off); + finished = true; + } - if (pendingBitPos != 64) { - out.writeLong(pending); + private void flush(int nvalues) throws IOException { + bulkOperation.set(nextBlocks, 0, nextValues, 0, iterations); + final int blocks = format.nblocks(bitsPerValue, nvalues); + for (int i = 0; i < blocks; ++i) { + out.writeLong(nextBlocks[i]); } + off = 0; } @Override - public String toString() { - return "PackedWriter(written " + written + "/" + valueCount + " with " - + bitsPerValue + " bits/value)"; + public int ord() { + return written - 1; } } diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/gen_BulkOperation.py b/lucene/core/src/java/org/apache/lucene/util/packed/gen_BulkOperation.py new file mode 100644 index 0000000..09b7886 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/util/packed/gen_BulkOperation.py @@ -0,0 +1,264 @@ +#! /usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from fractions import gcd + +"""Code generation for bulk operations""" + +PACKED_64_SINGLE_BLOCK_BPV = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 16, 21, 32] +OUTPUT_FILE = "BulkOperation.java" +HEADER = """// This file has been automatically generated, DO NOT EDIT + +package org.apache.lucene.util.packed; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.EnumMap; + +/** + * Efficient sequential read/write of packed integers. + */ +abstract class BulkOperation { + + static final EnumMap BULK_OPERATIONS = new EnumMap(PackedInts.Format.class); + + public static BulkOperation of(PackedInts.Format format, int bitsPerValue) { + assert bitsPerValue > 0 && bitsPerValue <= 64; + BulkOperation[] ops = BULK_OPERATIONS.get(format); + if (ops == null || ops[bitsPerValue] == null) { + throw new IllegalArgumentException("format: " + format + ", bitsPerValue: " + bitsPerValue); + } + return ops[bitsPerValue]; + } + + /** + * For every number of bits per value, there is a minimum number of + * blocks (b) / values (v) you need to write in order to reach the next block + * boundary: + * - 16 bits per value -> b=1, v=4 + * - 24 bits per value -> b=3, v=8 + * - 50 bits per value -> b=25, v=32 + * - 63 bits per value -> b=63, v = 64 + * - ... + * + * A bulk read consists in copying iterations*v values that are + * contained in iterations*b blocks into a long[] + * (higher values of iterations are likely to yield a better + * throughput) => this requires n * (b + v) longs in memory. + * + * This method computes iterations as + * ramBudget / (8 * (b + v)) (since a long is 8 bytes). + */ + public final int computeIterations(int valueCount, int ramBudget) { + final int iterations = (ramBudget >>> 3) / (blocks() + values()); + if (iterations == 0) { + // at least 1 + return 1; + } else if ((iterations - 1) * blocks() >= valueCount) { + // don't allocate for more than the size of the reader + return (int) Math.ceil((double) valueCount / values()); + } else { + return iterations; + } + } + + /** + * The minimum number of blocks required to perform a bulk get/set. + */ + public abstract int blocks(); + + /** + * The number of values that can be stored in blocks() blocks. + */ + public abstract int values(); + + /** + * Get n * values() values from n * blocks() blocks. + */ + public abstract void get(long[] blocks, int blockIndex, long[] values, int valuesIndex, int iterations); + + /** + * Set n * values() values into n * blocks() blocks. + */ + public abstract void set(long[] blocks, int blockIndex, long[] values, int valuesIndex, int iterations); + +""" + +FOOTER = "}" + +def packed64singleblock(bpv, f): + values = 64 / bpv + f.write("\n static final class Packed64SingleBlockBulkOperation%d extends BulkOperation {\n\n" %bpv) + f.write(" public int blocks() {\n") + f.write(" return 1;\n") + f.write(" }\n\n") + f.write(" public int values() {\n") + f.write(" return %d;\n" %values) + f.write(" }\n\n") + + f.write(" public void get(long[] blocks, int bi, long[] values, int vi, int iterations) {\n") + f.write(" assert bi + iterations * blocks() <= blocks.length;\n") + f.write(" assert vi + iterations * values() <= values.length;\n") + f.write(" for (int i = 0; i < iterations; ++i) {\n") + f.write(" final long block = blocks[bi++];\n") + mask = (1 << bpv) - 1 + for i in xrange(values): + block_offset = i / values + offset_in_block = i % values + if i == 0: + f.write(" values[vi++] = block & %dL;\n" %mask) + elif i == values - 1: + f.write(" values[vi++] = block >>> %d;\n" %(i * bpv)) + else: + f.write(" values[vi++] = (block >>> %d) & %dL;\n" %(i * bpv, mask)) + f.write(" }\n") + f.write(" }\n\n") + + f.write(" public void set(long[] blocks, int bi, long[] values, int vi, int iterations) {\n") + f.write(" assert bi + iterations * blocks() <= blocks.length;\n") + f.write(" assert vi + iterations * values() <= values.length;\n") + f.write(" for (int i = 0; i < iterations; ++i) {\n") + for i in xrange(values): + block_offset = i / values + offset_in_block = i % values + if i == 0: + f.write(" blocks[bi++] = values[vi++]") + else: + f.write(" | (values[vi++] << %d)" %(i * bpv)) + if i == values - 1: + f.write(";\n") + f.write(" }\n") + f.write(" }\n") + + f.write(" }\n") + +def packed64(bpv, f): + blocks = bpv + values = blocks * 64 / bpv + while blocks % 2 == 0 and values % 2 == 0: + blocks /= 2 + values /= 2 + assert values * bpv == 64 * blocks, "%d values, %d blocks, %d bits per value" %(values, blocks, bpv) + mask = (1 << bpv) - 1 + f.write(" static final class Packed64BulkOperation%d extends BulkOperation {\n\n" %bpv) + f.write(" public int blocks() {\n") + f.write(" return %d;\n" %blocks) + f.write(" }\n\n") + f.write(" public int values() {\n") + f.write(" return %d;\n" %values) + f.write(" }\n\n") + + if bpv == 64: + f.write(""" public void get(long[] blocks, int bi, long[] values, int vi, int iterations) { + System.arraycopy(blocks, bi, values, vi, iterations); + } + + public void set(long[] blocks, int bi, long[] values, int vi, int iterations) { + System.arraycopy(values, bi, blocks, vi, iterations); + } + } +""") + return + + f.write(" public void get(long[] blocks, int bi, long[] values, int vi, int iterations) {\n") + f.write(" assert bi + iterations * blocks() <= blocks.length;\n") + f.write(" assert vi + iterations * values() <= values.length;\n") + f.write(" for (int i = 0; i < iterations; ++i) {\n") + for i in xrange(0, values): + block_offset = i * bpv / 64 + bit_offset = (i * bpv) % 64 + if bit_offset == 0: + # start of block + f.write(" final long block%d = blocks[bi++];\n" %block_offset); + f.write(" values[vi++] = block%d >>> %d;\n" %(block_offset, 64 - bpv)) + elif bit_offset + bpv == 64: + # end of block + f.write(" values[vi++] = block%d & %dL;\n" %(block_offset, mask)) + elif bit_offset + bpv < 64: + # middle of block + f.write(" values[vi++] = (block%d >>> %d) & %dL;\n" %(block_offset, 64 - bit_offset - bpv, mask)) + else: + # value spans across 2 blocks + mask1 = (1 << (64 - bit_offset)) -1 + shift1 = bit_offset + bpv - 64 + shift2 = 64 - shift1 + f.write(" final long block%d = blocks[bi++];\n" %(block_offset + 1)); + f.write(" values[vi++] = ((block%d & %dL) << %d) | (block%d >>> %d);\n" %(block_offset, mask1, shift1, block_offset + 1, shift2)) + f.write(" }\n") + f.write(" }\n\n") + + f.write(" public void set(long[] blocks, int bi, long[] values, int vi, int iterations) {\n") + f.write(" assert bi + iterations * blocks() <= blocks.length;\n") + f.write(" assert vi + iterations * values() <= values.length;\n") + f.write(" for (int i = 0; i < iterations; ++i) {\n") + for i in xrange(0, values): + block_offset = i * bpv / 64 + bit_offset = (i * bpv) % 64 + if bit_offset == 0: + # start of block + f.write(" blocks[bi++] = (values[vi++] << %d)" %(64 - bpv)) + elif bit_offset + bpv == 64: + # end of block + f.write(" | values[vi++];\n") + elif bit_offset + bpv < 64: + # inside a block + f.write(" | (values[vi++] << %d)" %(64 - bit_offset - bpv)) + else: + # value spans across 2 blocks + right_bits = bit_offset + bpv - 64 + f.write(" | (values[vi] >>> %d);\n" %right_bits) + f.write(" blocks[bi++] = (values[vi++] << %d)" %(64 - right_bits)) + f.write(" }\n") + f.write(" }\n") + + f.write(" }\n\n") + + + +if __name__ == '__main__': + p64_bpv = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 16, 21, 32] + f = open(OUTPUT_FILE, 'w') + f.write(HEADER) + f.write(" static {\n") + f.write(" BULK_OPERATIONS.put(PackedInts.Format.PACKED, new BulkOperation[65]);") + for bpv in xrange(1, 65): + f.write(" BULK_OPERATIONS.get(PackedInts.Format.PACKED)[%d] = new Packed64BulkOperation%d();\n" %(bpv, bpv)) + f.write(" BULK_OPERATIONS.put(PackedInts.Format.PACKED_SINGLE_BLOCK, new BulkOperation[65]);\n") + for bpv in PACKED_64_SINGLE_BLOCK_BPV: + f.write(" BULK_OPERATIONS.get(PackedInts.Format.PACKED_SINGLE_BLOCK)[%d] = new Packed64SingleBlockBulkOperation%d();\n" %(bpv, bpv)) + f.write(" }\n") + for bpv in xrange(1, 65): + packed64(bpv, f) + for bpv in PACKED_64_SINGLE_BLOCK_BPV: + packed64singleblock(bpv,f) + f.write(FOOTER) + f.close() diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/gen_Direct.py b/lucene/core/src/java/org/apache/lucene/util/packed/gen_Direct.py new file mode 100644 index 0000000..c53dc15 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/util/packed/gen_Direct.py @@ -0,0 +1,175 @@ +#! /usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HEADER="""// This file has been automatically generated, DO NOT EDIT + +package org.apache.lucene.util.packed; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.store.DataInput; +import org.apache.lucene.util.RamUsageEstimator; + +import java.io.IOException; +import java.util.Arrays; + +""" + +TYPES = {8: "byte", 16: "short", 32: "int", 64: "long"} +MASKS = {8: " & 0xFFL", 16: " & 0xFFFFL", 32: " & 0xFFFFFFFFL", 64: ""} +CASTS = {8: "(byte) ", 16: "(short) ", 32: "(int) ", 64: ""} + +if __name__ == '__main__': + for bpv in TYPES.keys(): + type + f = open("Direct%d.java" %bpv, 'w') + f.write(HEADER) + f.write("""/** + * Direct wrapping of %d-bits values to a backing array. + * @lucene.internal + */\n""" %bpv) + f.write("final class Direct%d extends PackedInts.MutableImpl {\n" %bpv) + f.write(" final %s[] values;\n\n" %TYPES[bpv]) + + f.write(" Direct%d(int valueCount) {\n" %bpv) + f.write(" super(valueCount, %d);\n" %bpv) + f.write(" values = new %s[valueCount];\n" %TYPES[bpv]) + f.write(" }\n\n") + + f.write(" Direct%d(DataInput in, int valueCount) throws IOException {\n" %bpv) + f.write(" this(valueCount);\n") + f.write(" for (int i = 0; i < valueCount; ++i) {\n") + f.write(" values[i] = in.read%s();\n" %TYPES[bpv].title()) + f.write(" }\n") + if bpv != 64: + f.write(" final int mod = valueCount %% %d;\n" %(64 / bpv)) + f.write(" if (mod != 0) {\n") + f.write(" for (int i = mod; i < %d; ++i) {\n" %(64 / bpv)) + f.write(" in.read%s();\n" %TYPES[bpv].title()) + f.write(" }\n") + f.write(" }\n") + f.write(" }\n") + + f.write(""" + @Override + public long get(final int index) { + return values[index]%s; + } + + public void set(final int index, final long value) { + values[index] = %s(value); + } + + public long ramBytesUsed() { + return RamUsageEstimator.sizeOf(values); + } + + public void clear() { + Arrays.fill(values, %s0L); + } + + @Override + public Object getArray() { + return values; + } + + @Override + public boolean hasArray() { + return true; + } +""" %(MASKS[bpv], CASTS[bpv], CASTS[bpv])) + + if bpv == 64: + f.write(""" + @Override + public int get(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + assert off + len <= arr.length; + + final int gets = Math.min(valueCount - index, len); + System.arraycopy(values, index, arr, off, gets); + return gets; + } + + public int set(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + assert off + len <= arr.length; + + final int sets = Math.min(valueCount - index, len); + System.arraycopy(arr, off, values, index, sets); + return sets; + } + + @Override + public void fill(int fromIndex, int toIndex, long val) { + Arrays.fill(values, fromIndex, toIndex, val); + } +""") + else: + f.write(""" + @Override + public int get(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + assert off + len <= arr.length; + + final int gets = Math.min(valueCount - index, len); + for (int i = index, o = off, end = index + gets; i < end; ++i, ++o) { + arr[o] = values[i]%s; + } + return gets; + } + + public int set(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + assert off + len <= arr.length; + + final int sets = Math.min(valueCount - index, len); + for (int i = index, o = off, end = index + sets; i < end; ++i, ++o) { + values[i] = %sarr[o]; + } + return sets; + } + + @Override + public void fill(int fromIndex, int toIndex, long val) { + assert val == (val%s); + Arrays.fill(values, fromIndex, toIndex, %sval); + } +""" %(MASKS[bpv], CASTS[bpv], MASKS[bpv], CASTS[bpv])) + + f.write("}\n") + + f.close() diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/gen_Packed64SingleBlock.py b/lucene/core/src/java/org/apache/lucene/util/packed/gen_Packed64SingleBlock.py new file mode 100644 index 0000000..cd4cbba --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/util/packed/gen_Packed64SingleBlock.py @@ -0,0 +1,291 @@ +#! /usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SUPPORTED_BITS_PER_VALUE = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 16, 21, 32] + +HEADER="""// This file has been automatically generated, DO NOT EDIT + +package org.apache.lucene.util.packed; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.store.DataInput; +import org.apache.lucene.util.RamUsageEstimator; + +/** + * This class is similar to {@link Packed64} except that it trades space for + * speed by ensuring that a single block needs to be read/written in order to + * read/write a value. + */ +abstract class Packed64SingleBlock extends PackedInts.MutableImpl { + + public static final int MAX_SUPPORTED_BITS_PER_VALUE = %d; + private static final int[] SUPPORTED_BITS_PER_VALUE = new int[] {%s}; + + public static boolean isSupported(int bitsPerValue) { + return Arrays.binarySearch(SUPPORTED_BITS_PER_VALUE, bitsPerValue) >= 0; + } + + private static int requiredCapacity(int valueCount, int valuesPerBlock) { + return valueCount / valuesPerBlock + + (valueCount %% valuesPerBlock == 0 ? 0 : 1); + } + + final long[] blocks; + + Packed64SingleBlock(int valueCount, int bitsPerValue) { + super(valueCount, bitsPerValue); + assert isSupported(bitsPerValue); + final int valuesPerBlock = 64 / bitsPerValue; + blocks = new long[requiredCapacity(valueCount, valuesPerBlock)]; + } + + @Override + public void clear() { + Arrays.fill(blocks, 0L); + } + + public long ramBytesUsed() { + return RamUsageEstimator.sizeOf(blocks); + } + + @Override + public int get(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + len = Math.min(len, valueCount - index); + assert off + len <= arr.length; + + final int originalIndex = index; + + // go to the next block boundary + final int valuesPerBlock = 64 / bitsPerValue; + final int offsetInBlock = index %% valuesPerBlock; + if (offsetInBlock != 0) { + for (int i = offsetInBlock; i < valuesPerBlock && len > 0; ++i) { + arr[off++] = get(index++); + --len; + } + if (len == 0) { + return index - originalIndex; + } + } + + // bulk get + assert index %% valuesPerBlock == 0; + final BulkOperation op = BulkOperation.of(PackedInts.Format.PACKED_SINGLE_BLOCK, bitsPerValue); + assert op.blocks() == 1; + assert op.values() == valuesPerBlock; + final int blockIndex = index / valuesPerBlock; + final int nblocks = (index + len) / valuesPerBlock - blockIndex; + op.get(blocks, blockIndex, arr, off, nblocks); + final int diff = nblocks * valuesPerBlock; + index += diff; len -= diff; + + if (index > originalIndex) { + // stay at the block boundary + return index - originalIndex; + } else { + // no progress so far => already at a block boundary but no full block to + // get + assert index == originalIndex; + return super.get(index, arr, off, len); + } + } + + @Override + public int set(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + len = Math.min(len, valueCount - index); + assert off + len <= arr.length; + + final int originalIndex = index; + + // go to the next block boundary + final int valuesPerBlock = 64 / bitsPerValue; + final int offsetInBlock = index %% valuesPerBlock; + if (offsetInBlock != 0) { + for (int i = offsetInBlock; i < valuesPerBlock && len > 0; ++i) { + set(index++, arr[off++]); + --len; + } + if (len == 0) { + return index - originalIndex; + } + } + + // bulk set + assert index %% valuesPerBlock == 0; + final BulkOperation op = BulkOperation.of(PackedInts.Format.PACKED_SINGLE_BLOCK, bitsPerValue); + assert op.blocks() == 1; + assert op.values() == valuesPerBlock; + final int blockIndex = index / valuesPerBlock; + final int nblocks = (index + len) / valuesPerBlock - blockIndex; + op.set(blocks, blockIndex, arr, off, nblocks); + final int diff = nblocks * valuesPerBlock; + index += diff; len -= diff; + + if (index > originalIndex) { + // stay at the block boundary + return index - originalIndex; + } else { + // no progress so far => already at a block boundary but no full block to + // set + assert index == originalIndex; + return super.set(index, arr, off, len); + } + } + + @Override + public void fill(int fromIndex, int toIndex, long val) { + assert fromIndex >= 0; + assert fromIndex <= toIndex; + assert PackedInts.bitsRequired(val) <= bitsPerValue; + + final int valuesPerBlock = 64 / bitsPerValue; + if (toIndex - fromIndex <= valuesPerBlock << 1) { + // there needs to be at least one full block to set for the block + // approach to be worth trying + super.fill(fromIndex, toIndex, val); + return; + } + + // set values naively until the next block start + int fromOffsetInBlock = fromIndex %% valuesPerBlock; + if (fromOffsetInBlock != 0) { + for (int i = fromOffsetInBlock; i < valuesPerBlock; ++i) { + set(fromIndex++, val); + } + assert fromIndex %% valuesPerBlock == 0; + } + + // bulk set of the inner blocks + final int fromBlock = fromIndex / valuesPerBlock; + final int toBlock = toIndex / valuesPerBlock; + assert fromBlock * valuesPerBlock == fromIndex; + + long blockValue = 0L; + for (int i = 0; i < valuesPerBlock; ++i) { + blockValue = blockValue | (val << (i * bitsPerValue)); + } + Arrays.fill(blocks, fromBlock, toBlock, blockValue); + + // fill the gap + for (int i = valuesPerBlock * toBlock; i < toIndex; ++i) { + set(i, val); + } + } + + @Override + protected PackedInts.Format getFormat() { + return PackedInts.Format.PACKED_SINGLE_BLOCK; + } + + @Override + public String toString() { + return getClass().getSimpleName() + "(bitsPerValue=" + bitsPerValue + + ", size=" + size() + ", elements.length=" + blocks.length + ")"; + } + + public static Packed64SingleBlock create(DataInput in, + int valueCount, int bitsPerValue) throws IOException { + Packed64SingleBlock reader = create(valueCount, bitsPerValue); + for (int i = 0; i < reader.blocks.length; ++i) { + reader.blocks[i] = in.readLong(); + } + return reader; + } + +""" %(SUPPORTED_BITS_PER_VALUE[-1], ", ".join(map(str, SUPPORTED_BITS_PER_VALUE))) + +FOOTER = "}" + +if __name__ == '__main__': + + f = open("Packed64SingleBlock.java", 'w') + f.write(HEADER) + f.write(" public static Packed64SingleBlock create(int valueCount, int bitsPerValue) {\n") + f.write(" switch (bitsPerValue) {\n") + for bpv in SUPPORTED_BITS_PER_VALUE: + f.write(" case %d:\n" %bpv) + f.write(" return new Packed64SingleBlock%d(valueCount);\n" %bpv) + f.write(" default:\n") + f.write(" throw new IllegalArgumentException(\"Unsupported number of bits per value: \" + %d);\n" %bpv) + f.write(" }\n") + f.write(" }\n\n") + + for bpv in SUPPORTED_BITS_PER_VALUE: + log_2 = 0 + while (1 << log_2) < bpv: + log_2 = log_2 + 1 + if (1 << log_2) != bpv: + log_2 = None + + f.write(" static class Packed64SingleBlock%d extends Packed64SingleBlock {\n\n" %bpv) + + f.write(" Packed64SingleBlock%d(int valueCount) {\n" %bpv) + f.write(" super(valueCount, %d);\n" %bpv) + f.write(" }\n\n") + + f.write(" @Override\n") + f.write(" public long get(int index) {\n") + if log_2 is not None: + f.write(" final int o = index >>> %d;\n" %(6 - log_2)) + f.write(" final int b = index & %d;\n" %((1 << (6 - log_2)) - 1)) + f.write(" final int shift = b << %d;\n" %log_2) + else: + f.write(" final int o = index / %d;\n" %(64 / bpv)) + f.write(" final int b = index %% %d;\n" %(64 / bpv)) + f.write(" final int shift = b * %d;\n" %bpv) + f.write(" return (blocks[o] >>> shift) & %dL;\n" %((1 << bpv) - 1)) + f.write(" }\n\n") + + f.write(" @Override\n") + f.write(" public void set(int index, long value) {\n") + if log_2 is not None: + f.write(" final int o = index >>> %d;\n" %(6 - log_2)) + f.write(" final int b = index & %d;\n" %((1 << (6 - log_2)) - 1)) + f.write(" final int shift = b << %d;\n" %log_2) + else: + f.write(" final int o = index / %d;\n" %(64 / bpv)) + f.write(" final int b = index %% %d;\n" %(64 / bpv)) + f.write(" final int shift = b * %d;\n" %bpv) + f.write(" blocks[o] = (blocks[o] & ~(%dL << shift)) | (value << shift);\n" % ((1 << bpv) - 1)) + f.write(" }\n\n") + f.write(" }\n\n") + + f.write(FOOTER) + f.close() diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/gen_PackedThreeBlocks.py b/lucene/core/src/java/org/apache/lucene/util/packed/gen_PackedThreeBlocks.py new file mode 100644 index 0000000..2c6b282 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/util/packed/gen_PackedThreeBlocks.py @@ -0,0 +1,161 @@ +#! /usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HEADER="""// This file has been automatically generated, DO NOT EDIT + +package org.apache.lucene.util.packed; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.store.DataInput; +import org.apache.lucene.util.RamUsageEstimator; + +import java.io.IOException; +import java.util.Arrays; + +""" + +TYPES = {8: "byte", 16: "short"} +MASKS = {8: " & 0xFFL", 16: " & 0xFFFFL", 32: " & 0xFFFFFFFFL", 64: ""} +CASTS = {8: "(byte) ", 16: "(short) ", 32: "(int) ", 64: ""} + +if __name__ == '__main__': + for bpv in TYPES.keys(): + type + f = open("Packed%dThreeBlocks.java" %bpv, 'w') + f.write(HEADER) + f.write("""/** + * Packs integers into 3 %ss (%d bits per value). + * @lucene.internal + */\n""" %(TYPES[bpv], bpv*3)) + f.write("final class Packed%dThreeBlocks extends PackedInts.MutableImpl {\n" %bpv) + f.write(" final %s[] blocks;\n\n" %TYPES[bpv]) + + f.write(" public static final int MAX_SIZE = Integer.MAX_VALUE / 3;\n\n") + + f.write(" Packed%dThreeBlocks(int valueCount) {\n" %bpv) + f.write(" super(valueCount, %d);\n" %(bpv*3)) + f.write(" if (valueCount > MAX_SIZE) {\n") + f.write(" throw new ArrayIndexOutOfBoundsException(\"MAX_SIZE exceeded\");\n") + f.write(" }\n") + f.write(" blocks = new %s[valueCount * 3];\n" %TYPES[bpv]) + f.write(" }\n\n") + + f.write(" Packed%dThreeBlocks(DataInput in, int valueCount) throws IOException {\n" %bpv) + f.write(" this(valueCount);\n") + f.write(" for (int i = 0; i < 3 * valueCount; ++i) {\n") + f.write(" blocks[i] = in.read%s();\n" %TYPES[bpv].title()) + f.write(" }\n") + f.write(" final int mod = blocks.length %% %d;\n" %(64 / bpv)) + f.write(" if (mod != 0) {\n") + f.write(" for (int i = mod; i < %d; ++i) {\n" %(64 / bpv)) + f.write(" in.read%s();\n" %TYPES[bpv].title()) + f.write(" }\n") + f.write(" }\n") + f.write(" }\n") + + f.write(""" + @Override + public long get(int index) { + final int o = index * 3; + return (blocks[o]%s) << %d | (blocks[o+1]%s) << %d | (blocks[o+2]%s); + } + + @Override + public int get(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + assert off + len <= arr.length; + + final int gets = Math.min(valueCount - index, len); + for (int i = index * 3, end = (index + gets) * 3; i < end; i+=3) { + arr[off++] = (blocks[i]%s) << %d | (blocks[i+1]%s) << %d | (blocks[i+2]%s); + } + return gets; + } + + @Override + public void set(int index, long value) { + final int o = index * 3; + blocks[o] = %s(value >>> %d); + blocks[o+1] = %s(value >>> %d); + blocks[o+2] = %svalue; + } + + @Override + public int set(int index, long[] arr, int off, int len) { + assert len > 0 : "len must be > 0 (got " + len + ")"; + assert index >= 0 && index < valueCount; + assert off + len <= arr.length; + + final int sets = Math.min(valueCount - index, len); + for (int i = off, o = index * 3, end = off + sets; i < end; ++i) { + final long value = arr[i]; + blocks[o++] = %s(value >>> %d); + blocks[o++] = %s(value >>> %d); + blocks[o++] = %svalue; + } + return sets; + } + + @Override + public void fill(int fromIndex, int toIndex, long val) { + final %s block1 = %s(val >>> %d); + final %s block2 = %s(val >>> %d); + final %s block3 = %sval; + for (int i = fromIndex * 3, end = toIndex * 3; i < end; i += 3) { + blocks[i] = block1; + blocks[i+1] = block2; + blocks[i+2] = block3; + } + } + + @Override + public void clear() { + Arrays.fill(blocks, %s0); + } + + public long ramBytesUsed() { + return RamUsageEstimator.sizeOf(blocks); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "(bitsPerValue=" + bitsPerValue + + ", size=" + size() + ", elements.length=" + blocks.length + ")"; + } +} +""" %(MASKS[bpv], 2*bpv, MASKS[bpv], bpv, MASKS[bpv], MASKS[bpv], 2*bpv, MASKS[bpv], bpv, MASKS[bpv], CASTS[bpv], 2*bpv, CASTS[bpv], bpv, CASTS[bpv], CASTS[bpv], + 2*bpv, CASTS[bpv], bpv, CASTS[bpv], TYPES[bpv], CASTS[bpv], 2*bpv, TYPES[bpv], + CASTS[bpv], bpv, TYPES[bpv], CASTS[bpv], CASTS[bpv])) + + f.close() diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/package.html b/lucene/core/src/java/org/apache/lucene/util/packed/package.html index d1d0e29..6d1d6fe 100644 --- a/lucene/core/src/java/org/apache/lucene/util/packed/package.html +++ b/lucene/core/src/java/org/apache/lucene/util/packed/package.html @@ -19,9 +19,16 @@ +

      Packed integer arrays and streams.

      +

      - The packed package provides random access capable arrays of positive longs. - The implementations provides different trade offs between memory usage and + The packed package provides +

        +
      • sequential and random access capable arrays of positive longs,
        • +
      • routines for efficient serialization and deserialization of streams of packed integers.
        • +
      + + The implementations provide different trade-offs between memory usage and access speed. The standard usage scenario is replacing large int or long arrays in order to reduce the memory footprint.

      diff --git a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec index 34d1c57..eaa1d33 100644 --- a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -14,6 +14,5 @@ # limitations under the License. org.apache.lucene.codecs.lucene40.Lucene40Codec -org.apache.lucene.codecs.lucene3x.Lucene3xCodec org.apache.lucene.codecs.simpletext.SimpleTextCodec org.apache.lucene.codecs.appending.AppendingCodec diff --git a/lucene/core/src/test/org/apache/lucene/TestAssertions.java b/lucene/core/src/test/org/apache/lucene/TestAssertions.java index eceae88..5ad3a80 100644 --- a/lucene/core/src/test/org/apache/lucene/TestAssertions.java +++ b/lucene/core/src/test/org/apache/lucene/TestAssertions.java @@ -1,6 +1,6 @@ package org.apache.lucene; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/TestDemo.java b/lucene/core/src/test/org/apache/lucene/TestDemo.java index ebc608c..9f36dfe 100644 --- a/lucene/core/src/test/org/apache/lucene/TestDemo.java +++ b/lucene/core/src/test/org/apache/lucene/TestDemo.java @@ -1,6 +1,6 @@ package org.apache.lucene; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,7 +22,8 @@ import java.io.IOException; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.index.RandomIndexWriter; @@ -49,12 +50,12 @@ public class TestDemo extends LuceneTestCase { Document doc = new Document(); String longTerm = "longtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongterm"; String text = "This is the text to be indexed. " + longTerm; - doc.add(newField("fieldname", text, TextField.TYPE_STORED)); + doc.add(newTextField("fieldname", text, Field.Store.YES)); iwriter.addDocument(doc); iwriter.close(); // Now search the index: - IndexReader ireader = IndexReader.open(directory); // read-only=true + IndexReader ireader = DirectoryReader.open(directory); // read-only=true IndexSearcher isearcher = new IndexSearcher(ireader); assertEquals(1, isearcher.search(new TermQuery(new Term("fieldname", longTerm)), 1).totalHits); diff --git a/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java b/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java index 6f1d6be..6371bc7 100644 --- a/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java +++ b/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java @@ -1,6 +1,6 @@ package org.apache.lucene; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -69,12 +69,12 @@ public class TestExternalCodecs extends LuceneTestCase { ); Document doc = new Document(); // uses default codec: - doc.add(newField("field1", "this field uses the standard codec as the test", TextField.TYPE_UNSTORED)); + doc.add(newTextField("field1", "this field uses the standard codec as the test", Field.Store.NO)); // uses pulsing codec: - Field field2 = newField("field2", "this field uses the pulsing codec as the test", TextField.TYPE_UNSTORED); + Field field2 = newTextField("field2", "this field uses the pulsing codec as the test", Field.Store.NO); doc.add(field2); - Field idField = newField("id", "", StringField.TYPE_UNSTORED); + Field idField = newStringField("id", "", Field.Store.NO); doc.add(idField); for(int i=0;i { - private static final Comparator legacyComparator = - BytesRef.getUTF8SortedAsUTF16Comparator(); - - public int compare(Term term1, Term term2) { - if (term1.field().equals(term2.field())) { - return legacyComparator.compare(term1.bytes(), term2.bytes()); - } else { - return term1.field().compareTo(term2.field()); - } - } - } - - private static final SortTermAsUTF16Comparator termAsUTF16Comparator = new SortTermAsUTF16Comparator(); - - // single straight enum - private void doTestStraightEnum(List fieldTerms, IndexReader reader, int uniqueTermCount) throws IOException { - - if (VERBOSE) { - System.out.println("\nTEST: top now enum reader=" + reader); - } - FieldsEnum fieldsEnum = MultiFields.getFields(reader).iterator(); - - { - // Test straight enum: - String field; - int termCount = 0; - while((field = fieldsEnum.next()) != null) { - Terms terms = fieldsEnum.terms(); - assertNotNull(terms); - TermsEnum termsEnum = terms.iterator(null); - BytesRef text; - BytesRef lastText = null; - while((text = termsEnum.next()) != null) { - Term exp = fieldTerms.get(termCount); - if (VERBOSE) { - System.out.println(" got term=" + field + ":" + UnicodeUtil.toHexString(text.utf8ToString())); - System.out.println(" exp=" + exp.field() + ":" + UnicodeUtil.toHexString(exp.text().toString())); - System.out.println(); - } - if (lastText == null) { - lastText = BytesRef.deepCopyOf(text); - } else { - assertTrue(lastText.compareTo(text) < 0); - lastText.copyBytes(text); - } - assertEquals(exp.field(), field); - assertEquals(exp.bytes(), text); - termCount++; - } - if (VERBOSE) { - System.out.println(" no more terms for field=" + field); - } - } - assertEquals(uniqueTermCount, termCount); - } - } - - // randomly seeks to term that we know exists, then next's - // from there - private void doTestSeekExists(Random r, List fieldTerms, IndexReader reader) throws IOException { - - final Map tes = new HashMap(); - - // Test random seek to existing term, then enum: - if (VERBOSE) { - System.out.println("\nTEST: top now seek"); - } - - int num = atLeast(100); - for (int iter = 0; iter < num; iter++) { - - // pick random field+term - int spot = r.nextInt(fieldTerms.size()); - Term term = fieldTerms.get(spot); - String field = term.field(); - - if (VERBOSE) { - System.out.println("TEST: exist seek field=" + field + " term=" + UnicodeUtil.toHexString(term.text())); - } - - // seek to it - TermsEnum te = tes.get(field); - if (te == null) { - te = MultiFields.getTerms(reader, field).iterator(null); - tes.put(field, te); - } - - if (VERBOSE) { - System.out.println(" done get enum"); - } - - // seek should find the term - assertEquals(TermsEnum.SeekStatus.FOUND, - te.seekCeil(term.bytes())); - - // now .next() this many times: - int ct = _TestUtil.nextInt(r, 5, 100); - for(int i=0;i= fieldTerms.size()) { - break; - } - term = fieldTerms.get(1+spot+i); - if (!term.field().equals(field)) { - assertNull(te.next()); - break; - } else { - BytesRef t = te.next(); - - if (VERBOSE) { - System.out.println(" got term=" + (t == null ? null : UnicodeUtil.toHexString(t.utf8ToString()))); - System.out.println(" exp=" + UnicodeUtil.toHexString(term.text().toString())); - } - - assertEquals(term.bytes(), t); - } - } - } - } - - private void doTestSeekDoesNotExist(Random r, int numField, List fieldTerms, Term[] fieldTermsArray, IndexReader reader) throws IOException { - - final Map tes = new HashMap(); - - if (VERBOSE) { - System.out.println("TEST: top random seeks"); - } - - { - int num = atLeast(100); - for (int iter = 0; iter < num; iter++) { - - // seek to random spot - String field = ("f" + r.nextInt(numField)).intern(); - Term tx = new Term(field, getRandomString(r)); - - int spot = Arrays.binarySearch(fieldTermsArray, tx); - - if (spot < 0) { - if (VERBOSE) { - System.out.println("TEST: non-exist seek to " + field + ":" + UnicodeUtil.toHexString(tx.text())); - } - - // term does not exist: - TermsEnum te = tes.get(field); - if (te == null) { - te = MultiFields.getTerms(reader, field).iterator(null); - tes.put(field, te); - } - - if (VERBOSE) { - System.out.println(" got enum"); - } - - spot = -spot - 1; - - if (spot == fieldTerms.size() || !fieldTerms.get(spot).field().equals(field)) { - assertEquals(TermsEnum.SeekStatus.END, te.seekCeil(tx.bytes())); - } else { - assertEquals(TermsEnum.SeekStatus.NOT_FOUND, te.seekCeil(tx.bytes())); - - if (VERBOSE) { - System.out.println(" got term=" + UnicodeUtil.toHexString(te.term().utf8ToString())); - System.out.println(" exp term=" + UnicodeUtil.toHexString(fieldTerms.get(spot).text())); - } - - assertEquals(fieldTerms.get(spot).bytes(), - te.term()); - - // now .next() this many times: - int ct = _TestUtil.nextInt(r, 5, 100); - for(int i=0;i= fieldTerms.size()) { - break; - } - Term term = fieldTerms.get(1+spot+i); - if (!term.field().equals(field)) { - assertNull(te.next()); - break; - } else { - BytesRef t = te.next(); - - if (VERBOSE) { - System.out.println(" got term=" + (t == null ? null : UnicodeUtil.toHexString(t.utf8ToString()))); - System.out.println(" exp=" + UnicodeUtil.toHexString(term.text().toString())); - } - - assertEquals(term.bytes(), t); - } - } - - } - } - } - } - } - - - @Test - public void testSurrogatesOrder() throws Exception { - Directory dir = newDirectory(); - RandomIndexWriter w = new RandomIndexWriter(random(), - dir, - newIndexWriterConfig( TEST_VERSION_CURRENT, - new MockAnalyzer(random())).setCodec(new PreFlexRWCodec())); - - final int numField = _TestUtil.nextInt(random(), 2, 5); - - int uniqueTermCount = 0; - - int tc = 0; - - List fieldTerms = new ArrayList(); - - for(int f=0;f uniqueTerms = new HashSet(); - - for(int i=0;i sampleTerms; - - /** we will manually instantiate preflex-rw here */ - @BeforeClass - public static void beforeClass() throws Exception { - LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE = true; - IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)); - - termIndexInterval = config.getTermIndexInterval(); - indexDivisor = _TestUtil.nextInt(random(), 1, 10); - NUMBER_OF_DOCUMENTS = atLeast(100); - NUMBER_OF_FIELDS = atLeast(Math.max(10, 3*termIndexInterval*indexDivisor/NUMBER_OF_DOCUMENTS)); - - directory = newDirectory(); - - config.setCodec(new PreFlexRWCodec()); - LogMergePolicy mp = newLogMergePolicy(); - // turn off compound file, this test will open some index files directly. - mp.setUseCompoundFile(false); - config.setMergePolicy(mp); - - - populate(directory, config); - - DirectoryReader r0 = IndexReader.open(directory); - SegmentReader r = LuceneTestCase.getOnlySegmentReader(r0); - String segment = r.getSegmentName(); - r.close(); - - FieldInfosReader infosReader = new PreFlexRWCodec().fieldInfosFormat().getFieldInfosReader(); - FieldInfos fieldInfos = infosReader.read(directory, segment, IOContext.READONCE); - String segmentFileName = IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION); - long tiiFileLength = directory.fileLength(segmentFileName); - IndexInput input = directory.openInput(segmentFileName, newIOContext(random())); - termEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_EXTENSION), newIOContext(random())), fieldInfos, false); - int totalIndexInterval = termEnum.indexInterval * indexDivisor; - - SegmentTermEnum indexEnum = new SegmentTermEnum(input, fieldInfos, true); - index = new TermInfosReaderIndex(indexEnum, indexDivisor, tiiFileLength, totalIndexInterval); - indexEnum.close(); - input.close(); - - reader = IndexReader.open(directory); - sampleTerms = sample(reader,1000); - } - - @AfterClass - public static void afterClass() throws Exception { - termEnum.close(); - reader.close(); - directory.close(); - termEnum = null; - reader = null; - directory = null; - index = null; - sampleTerms = null; - } - - public void testSeekEnum() throws CorruptIndexException, IOException { - int indexPosition = 3; - SegmentTermEnum clone = termEnum.clone(); - Term term = findTermThatWouldBeAtIndex(clone, indexPosition); - SegmentTermEnum enumerator = clone; - index.seekEnum(enumerator, indexPosition); - assertEquals(term, enumerator.term()); - clone.close(); - } - - public void testCompareTo() throws IOException { - Term term = new Term("field" + random().nextInt(NUMBER_OF_FIELDS) ,getText()); - for (int i = 0; i < index.length(); i++) { - Term t = index.getTerm(i); - int compareTo = term.compareTo(t); - assertEquals(compareTo, index.compareTo(term, i)); - } - } - - public void testRandomSearchPerformance() throws CorruptIndexException, IOException { - IndexSearcher searcher = new IndexSearcher(reader); - for (Term t : sampleTerms) { - TermQuery query = new TermQuery(t); - TopDocs topDocs = searcher.search(query, 10); - assertTrue(topDocs.totalHits > 0); - } - } - - private static List sample(IndexReader reader, int size) throws IOException { - List sample = new ArrayList(); - Random random = new Random(); - FieldsEnum fieldsEnum = MultiFields.getFields(reader).iterator(); - String field; - while((field = fieldsEnum.next()) != null) { - Terms terms = fieldsEnum.terms(); - assertNotNull(terms); - TermsEnum termsEnum = terms.iterator(null); - while (termsEnum.next() != null) { - if (sample.size() >= size) { - int pos = random.nextInt(size); - sample.set(pos, new Term(field, termsEnum.term())); - } else { - sample.add(new Term(field, termsEnum.term())); - } - } - } - Collections.shuffle(sample); - return sample; - } - - private Term findTermThatWouldBeAtIndex(SegmentTermEnum termEnum, int index) throws IOException { - int termPosition = index * termIndexInterval * indexDivisor; - for (int i = 0; i < termPosition; i++) { - // TODO: this test just uses random terms, so this is always possible - assumeTrue("ran out of terms", termEnum.next()); - } - final Term term = termEnum.term(); - // An indexed term is only written when the term after - // it exists, so, if the number of terms is 0 mod - // termIndexInterval, the last index term will not be - // written; so we require a term after this term - // as well: - assumeTrue("ran out of terms", termEnum.next()); - return term; - } - - private static void populate(Directory directory, IndexWriterConfig config) throws CorruptIndexException, LockObtainFailedException, IOException { - RandomIndexWriter writer = new RandomIndexWriter(random(), directory, config); - for (int i = 0; i < NUMBER_OF_DOCUMENTS; i++) { - Document document = new Document(); - for (int f = 0; f < NUMBER_OF_FIELDS; f++) { - document.add(newField("field" + f, getText(), StringField.TYPE_UNSTORED)); - } - writer.addDocument(document); - } - writer.forceMerge(1); - writer.close(); - } - - private static String getText() { - return Long.toString(random().nextLong(),Character.MAX_RADIX); - } -} diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestAllFilesHaveCodecHeader.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestAllFilesHaveCodecHeader.java new file mode 100644 index 0000000..d6535df --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestAllFilesHaveCodecHeader.java @@ -0,0 +1,92 @@ +package org.apache.lucene.codecs.lucene40; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.store.CompoundFileDirectory; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; + +/** + * Test that a plain Lucene40Codec puts codec headers in all files. + */ +public class TestAllFilesHaveCodecHeader extends LuceneTestCase { + public void test() throws Exception { + Directory dir = newDirectory(); + IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); + conf.setCodec(Codec.forName("Lucene40")); + // riw should sometimes create docvalues fields, etc + RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf); + Document doc = new Document(); + // these fields should sometimes get term vectors, etc + Field idField = newStringField("id", "", Field.Store.NO); + Field bodyField = newTextField("body", "", Field.Store.NO); + doc.add(idField); + doc.add(bodyField); + for (int i = 0; i < 100; i++) { + idField.setStringValue(Integer.toString(i)); + bodyField.setStringValue(_TestUtil.randomUnicodeString(random())); + riw.addDocument(doc); + if (random().nextInt(7) == 0) { + riw.commit(); + } + } + riw.close(); + checkHeaders(dir); + dir.close(); + } + + private void checkHeaders(Directory dir) throws IOException { + for (String file : dir.listAll()) { + if (file.equals(IndexFileNames.SEGMENTS_GEN)) { + continue; // segments.gen has no header, thats ok + } + if (file.endsWith(IndexFileNames.COMPOUND_FILE_EXTENSION)) { + CompoundFileDirectory cfsDir = new CompoundFileDirectory(dir, file, newIOContext(random()), false); + checkHeaders(cfsDir); // recurse into cfs + cfsDir.close(); + } + IndexInput in = null; + boolean success = false; + try { + in = dir.openInput(file, newIOContext(random())); + int val = in.readInt(); + assertEquals(file + " has no codec header, instead found: " + val, CodecUtil.CODEC_MAGIC, val); + success = true; + } finally { + if (success) { + IOUtils.close(in); + } else { + IOUtils.closeWhileHandlingException(in); + } + } + } + } +} diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestBitVector.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestBitVector.java index 251c43e..32080c1 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestBitVector.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestBitVector.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestLucene40PostingsReader.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestLucene40PostingsReader.java new file mode 100644 index 0000000..f81223d --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestLucene40PostingsReader.java @@ -0,0 +1,126 @@ +package org.apache.lucene.codecs.lucene40; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.ArrayList; +import java.util.Collections; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; + +public class TestLucene40PostingsReader extends LuceneTestCase { + static final String terms[] = new String[100]; + static { + for (int i = 0; i < terms.length; i++) { + terms[i] = Integer.toString(i+1); + } + } + + /** tests terms with different probabilities of being in the document. + * depends heavily on term vectors cross-check at checkIndex + */ + public void testPostings() throws Exception { + Directory dir = newFSDirectory(_TestUtil.getTempDir("postings")); + IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); + iwc.setCodec(Codec.forName("Lucene40")); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc); + + Document doc = new Document(); + + // id field + FieldType idType = new FieldType(StringField.TYPE_NOT_STORED); + idType.setStoreTermVectors(true); + Field idField = new Field("id", "", idType); + doc.add(idField); + + // title field: short text field + FieldType titleType = new FieldType(TextField.TYPE_NOT_STORED); + titleType.setStoreTermVectors(true); + titleType.setStoreTermVectorPositions(true); + titleType.setStoreTermVectorOffsets(true); + titleType.setIndexOptions(indexOptions()); + Field titleField = new Field("title", "", titleType); + doc.add(titleField); + + // body field: long text field + FieldType bodyType = new FieldType(TextField.TYPE_NOT_STORED); + bodyType.setStoreTermVectors(true); + bodyType.setStoreTermVectorPositions(true); + bodyType.setStoreTermVectorOffsets(true); + bodyType.setIndexOptions(indexOptions()); + Field bodyField = new Field("body", "", bodyType); + doc.add(bodyField); + + int numDocs = atLeast(1000); + for (int i = 0; i < numDocs; i++) { + idField.setStringValue(Integer.toString(i)); + titleField.setStringValue(fieldValue(1)); + bodyField.setStringValue(fieldValue(3)); + iw.addDocument(doc); + if (random().nextInt(20) == 0) { + iw.deleteDocuments(new Term("id", Integer.toString(i))); + } + } + if (random().nextBoolean()) { + // delete 1-100% of docs + iw.deleteDocuments(new Term("title", terms[random().nextInt(terms.length)])); + } + iw.close(); + dir.close(); // checkindex + } + + IndexOptions indexOptions() { + switch(random().nextInt(4)) { + case 0: return IndexOptions.DOCS_ONLY; + case 1: return IndexOptions.DOCS_AND_FREQS; + case 2: return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; + default: return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS; + } + } + + String fieldValue(int maxTF) { + ArrayList shuffled = new ArrayList(); + StringBuilder sb = new StringBuilder(); + int i = random().nextInt(terms.length); + while (i < terms.length) { + int tf = _TestUtil.nextInt(random(), 1, maxTF); + for (int j = 0; j < tf; j++) { + shuffled.add(terms[i]); + } + i++; + } + Collections.shuffle(shuffled, random()); + for (String term : shuffled) { + sb.append(term); + sb.append(' '); + } + return sb.toString(); + } +} diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestReuseDocsEnum.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestReuseDocsEnum.java index ff6ac7d..7c71327 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestReuseDocsEnum.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestReuseDocsEnum.java @@ -1,5 +1,5 @@ package org.apache.lucene.codecs.lucene40; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,6 +17,7 @@ package org.apache.lucene.codecs.lucene40; */ import java.io.IOException; import java.util.IdentityHashMap; +import java.util.List; import java.util.Random; import org.apache.lucene.analysis.MockAnalyzer; @@ -35,7 +36,6 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.ReaderUtil; import org.apache.lucene.util._TestUtil; public class TestReuseDocsEnum extends LuceneTestCase { @@ -50,21 +50,18 @@ public class TestReuseDocsEnum extends LuceneTestCase { writer.commit(); DirectoryReader open = DirectoryReader.open(dir); - new ReaderUtil.Gather(open) { - @Override - protected void add(int base, AtomicReader r) throws IOException { - Terms terms = r.terms("body"); - TermsEnum iterator = terms.iterator(null); - IdentityHashMap enums = new IdentityHashMap(); - MatchNoBits bits = new Bits.MatchNoBits(r.maxDoc()); - while ((iterator.next()) != null) { - DocsEnum docs = iterator.docs(random().nextBoolean() ? bits : new Bits.MatchNoBits(r.maxDoc()), null, random().nextBoolean()); - enums.put(docs, true); - } - - assertEquals(terms.size(), enums.size()); + for (AtomicReader indexReader : open.getSequentialSubReaders()) { + Terms terms = indexReader.terms("body"); + TermsEnum iterator = terms.iterator(null); + IdentityHashMap enums = new IdentityHashMap(); + MatchNoBits bits = new Bits.MatchNoBits(indexReader.maxDoc()); + while ((iterator.next()) != null) { + DocsEnum docs = iterator.docs(random().nextBoolean() ? bits : new Bits.MatchNoBits(indexReader.maxDoc()), null, random().nextBoolean()); + enums.put(docs, true); } - }.run(); + + assertEquals(terms.size(), enums.size()); + } IOUtils.close(writer, open, dir); } @@ -79,9 +76,8 @@ public class TestReuseDocsEnum extends LuceneTestCase { writer.commit(); DirectoryReader open = DirectoryReader.open(dir); - IndexReader[] sequentialSubReaders = open.getSequentialSubReaders(); - for (IndexReader indexReader : sequentialSubReaders) { - Terms terms = ((AtomicReader) indexReader).terms("body"); + for (AtomicReader indexReader : open.getSequentialSubReaders()) { + Terms terms = indexReader.terms("body"); TermsEnum iterator = terms.iterator(null); IdentityHashMap enums = new IdentityHashMap(); MatchNoBits bits = new Bits.MatchNoBits(open.maxDoc()); @@ -125,8 +121,8 @@ public class TestReuseDocsEnum extends LuceneTestCase { DirectoryReader firstReader = DirectoryReader.open(dir); DirectoryReader secondReader = DirectoryReader.open(dir); - IndexReader[] sequentialSubReaders = firstReader.getSequentialSubReaders(); - IndexReader[] sequentialSubReaders2 = secondReader.getSequentialSubReaders(); + List sequentialSubReaders = firstReader.getSequentialSubReaders(); + List sequentialSubReaders2 = secondReader.getSequentialSubReaders(); for (IndexReader indexReader : sequentialSubReaders) { Terms terms = ((AtomicReader) indexReader).terms("body"); @@ -154,11 +150,11 @@ public class TestReuseDocsEnum extends LuceneTestCase { IOUtils.close(writer, firstReader, secondReader, dir); } - public DocsEnum randomDocsEnum(String field, BytesRef term, IndexReader[] readers, Bits bits) throws IOException { + public DocsEnum randomDocsEnum(String field, BytesRef term, List readers, Bits bits) throws IOException { if (random().nextInt(10) == 0) { return null; } - AtomicReader indexReader = (AtomicReader) readers[random().nextInt(readers.length)]; + AtomicReader indexReader = (AtomicReader) readers.get(random().nextInt(readers.size())); return indexReader.termDocsEnum(bits, field, term, random().nextBoolean()); } diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene40/values/TestDocValues.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene40/values/TestDocValues.java index 7d871ed..c8fcd9e 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/lucene40/values/TestDocValues.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene40/values/TestDocValues.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40.values; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -360,7 +360,7 @@ public class TestDocValues extends LuceneTestCase { final Counter trackBytes = Counter.newCounter(); DocValuesConsumer w = Ints.getWriter(dir, "test", trackBytes, type, newIOContext(random())); for (int i = 0; i < NUM_VALUES; i++) { - final long v = random().nextLong() % (1 + maxV); + final long v = _TestUtil.nextLong(random(), -maxV, maxV); valueHolder.numberValue = values[i] = v; w.add(i, valueHolder); } diff --git a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat.java index 76c7fff..05d43e7 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.perfield; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,21 +19,23 @@ package org.apache.lucene.codecs.perfield; import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.lucene40.Lucene40Codec; import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.codecs.mocksep.MockSepPostingsFormat; +import org.apache.lucene.codecs.pulsing.Pulsing40PostingsFormat; import org.apache.lucene.codecs.simpletext.SimpleTextPostingsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LogDocMergePolicy; +import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.search.IndexSearcher; @@ -67,7 +69,7 @@ public class TestPerFieldPostingsFormat extends LuceneTestCase { private void addDocs(IndexWriter writer, int numDocs) throws IOException { for (int i = 0; i < numDocs; i++) { Document doc = new Document(); - doc.add(newField("content", "aaa", TextField.TYPE_UNSTORED)); + doc.add(newTextField("content", "aaa", Field.Store.NO)); writer.addDocument(doc); } } @@ -75,7 +77,7 @@ public class TestPerFieldPostingsFormat extends LuceneTestCase { private void addDocs2(IndexWriter writer, int numDocs) throws IOException { for (int i = 0; i < numDocs; i++) { Document doc = new Document(); - doc.add(newField("content", "bbb", TextField.TYPE_UNSTORED)); + doc.add(newTextField("content", "bbb", Field.Store.NO)); writer.addDocument(doc); } } @@ -83,8 +85,8 @@ public class TestPerFieldPostingsFormat extends LuceneTestCase { private void addDocs3(IndexWriter writer, int numDocs) throws IOException { for (int i = 0; i < numDocs; i++) { Document doc = new Document(); - doc.add(newField("content", "ccc", TextField.TYPE_UNSTORED)); - doc.add(newField("id", "" + i, StringField.TYPE_STORED)); + doc.add(newTextField("content", "ccc", Field.Store.NO)); + doc.add(newStringField("id", "" + i, Field.Store.YES)); writer.addDocument(doc); } } @@ -186,11 +188,11 @@ public class TestPerFieldPostingsFormat extends LuceneTestCase { } public void assertQuery(Term t, Directory dir, int num) - throws CorruptIndexException, IOException { + throws IOException { if (VERBOSE) { System.out.println("\nTEST: assertQuery " + t); } - IndexReader reader = IndexReader.open(dir, 1); + IndexReader reader = DirectoryReader.open(dir, 1); IndexSearcher searcher = newSearcher(reader); TopDocs search = searcher.search(new TermQuery(t), num + 10); assertEquals(num, search.totalHits); @@ -246,7 +248,7 @@ public class TestPerFieldPostingsFormat extends LuceneTestCase { for (int j = 0; j < docsPerRound; j++) { final Document doc = new Document(); for (int k = 0; k < num; k++) { - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setTokenized(random().nextBoolean()); customType.setOmitNorms(random().nextBoolean()); Field field = newField("" + k, _TestUtil @@ -264,4 +266,60 @@ public class TestPerFieldPostingsFormat extends LuceneTestCase { } dir.close(); } + + public void testSameCodecDifferentInstance() throws Exception { + Codec codec = new Lucene40Codec() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + if ("id".equals(field)) { + return new Pulsing40PostingsFormat(1); + } else if ("date".equals(field)) { + return new Pulsing40PostingsFormat(1); + } else { + return super.getPostingsFormatForField(field); + } + } + }; + doTestMixedPostings(codec); + } + + public void testSameCodecDifferentParams() throws Exception { + Codec codec = new Lucene40Codec() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + if ("id".equals(field)) { + return new Pulsing40PostingsFormat(1); + } else if ("date".equals(field)) { + return new Pulsing40PostingsFormat(2); + } else { + return super.getPostingsFormatForField(field); + } + } + }; + doTestMixedPostings(codec); + } + + private void doTestMixedPostings(Codec codec) throws Exception { + Directory dir = newDirectory(); + IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); + iwc.setCodec(codec); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc); + Document doc = new Document(); + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); + // turn on vectors for the checkindex cross-check + ft.setStoreTermVectors(true); + ft.setStoreTermVectorOffsets(true); + ft.setStoreTermVectorPositions(true); + Field idField = new Field("id", "", ft); + Field dateField = new Field("date", "", ft); + doc.add(idField); + doc.add(dateField); + for (int i = 0; i < 100; i++) { + idField.setStringValue(Integer.toString(random().nextInt(50))); + dateField.setStringValue(Integer.toString(random().nextInt(100))); + iw.addDocument(doc); + } + iw.close(); + dir.close(); // checkindex + } } diff --git a/lucene/core/src/test/org/apache/lucene/codecs/pulsing/Test10KPulsings.java b/lucene/core/src/test/org/apache/lucene/codecs/pulsing/Test10KPulsings.java index c48201c..0bf3ad5 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/pulsing/Test10KPulsings.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/pulsing/Test10KPulsings.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.pulsing; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/codecs/pulsing/TestPulsingReuse.java b/lucene/core/src/test/org/apache/lucene/codecs/pulsing/TestPulsingReuse.java index 5c2e92f..12529fe 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/pulsing/TestPulsingReuse.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/pulsing/TestPulsingReuse.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.pulsing; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -31,7 +31,6 @@ import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.store.Directory; @@ -51,7 +50,7 @@ public class TestPulsingReuse extends LuceneTestCase { RandomIndexWriter iw = new RandomIndexWriter(random(), dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setCodec(cp)); Document doc = new Document(); - doc.add(new Field("foo", "a b b c c c d e f g g h i i j j k", TextField.TYPE_UNSTORED)); + doc.add(new TextField("foo", "a b b c c c d e f g g h i i j j k", Field.Store.NO)); iw.addDocument(doc); DirectoryReader ir = iw.getReader(); iw.close(); @@ -90,7 +89,7 @@ public class TestPulsingReuse extends LuceneTestCase { RandomIndexWriter iw = new RandomIndexWriter(random(), dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setCodec(cp)); Document doc = new Document(); - doc.add(new Field("foo", "a b b c c c d e f g g g h i i j j k l l m m m", TextField.TYPE_UNSTORED)); + doc.add(new TextField("foo", "a b b c c c d e f g g g h i i j j k l l m m m", Field.Store.NO)); // note: the reuse is imperfect, here we would have 4 enums (lost reuse when we get an enum for 'm') // this is because we only track the 'last' enum we reused (not all). // but this seems 'good enough' for now. diff --git a/lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.java b/lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.java index 7b2a671..45c72cf 100644 --- a/lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.java +++ b/lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.java @@ -7,7 +7,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/document/TestDateTools.java b/lucene/core/src/test/org/apache/lucene/document/TestDateTools.java index 83d40fb..a60d155 100644 --- a/lucene/core/src/test/org/apache/lucene/document/TestDateTools.java +++ b/lucene/core/src/test/org/apache/lucene/document/TestDateTools.java @@ -11,7 +11,7 @@ import org.junit.rules.TestRule; import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/document/TestDocument.java b/lucene/core/src/test/org/apache/lucene/document/TestDocument.java index 2e54a6a..d33116b 100644 --- a/lucene/core/src/test/org/apache/lucene/document/TestDocument.java +++ b/lucene/core/src/test/org/apache/lucene/document/TestDocument.java @@ -1,6 +1,6 @@ package org.apache.lucene.document; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,16 +22,12 @@ import java.io.StringReader; import org.apache.lucene.analysis.EmptyTokenizer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.DocsAndPositionsEnum; -import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; @@ -55,8 +51,8 @@ public class TestDocument extends LuceneTestCase { FieldType ft = new FieldType(); ft.setStored(true); IndexableField stringFld = new Field("string", binaryVal, ft); - IndexableField binaryFld = new StoredField("binary", binaryVal.getBytes()); - IndexableField binaryFld2 = new StoredField("binary", binaryVal2.getBytes()); + IndexableField binaryFld = new StoredField("binary", binaryVal.getBytes("UTF-8")); + IndexableField binaryFld2 = new StoredField("binary", binaryVal2.getBytes("UTF-8")); doc.add(stringFld); doc.add(binaryFld); @@ -66,7 +62,6 @@ public class TestDocument extends LuceneTestCase { assertTrue(binaryFld.binaryValue() != null); assertTrue(binaryFld.fieldType().stored()); assertFalse(binaryFld.fieldType().indexed()); - assertFalse(binaryFld.fieldType().tokenized()); String binaryTest = doc.getBinaryValue("binary").utf8ToString(); assertTrue(binaryTest.equals(binaryVal)); @@ -134,7 +129,7 @@ public class TestDocument extends LuceneTestCase { FieldType ft = new FieldType(); ft.setStored(true); new Field("name", "value", ft); // okay - new StringField("name", "value"); // okay + new StringField("name", "value", Field.Store.NO); // okay try { new Field("name", "value", new FieldType()); fail(); @@ -206,16 +201,16 @@ public class TestDocument extends LuceneTestCase { Document doc = new Document(); FieldType stored = new FieldType(); stored.setStored(true); - doc.add(new Field("keyword", "test1", StringField.TYPE_STORED)); - doc.add(new Field("keyword", "test2", StringField.TYPE_STORED)); - doc.add(new Field("text", "test1", TextField.TYPE_STORED)); - doc.add(new Field("text", "test2", TextField.TYPE_STORED)); + doc.add(new StringField("keyword", "test1", Field.Store.YES)); + doc.add(new StringField("keyword", "test2", Field.Store.YES)); + doc.add(new TextField("text", "test1", Field.Store.YES)); + doc.add(new TextField("text", "test2", Field.Store.YES)); doc.add(new Field("unindexed", "test1", stored)); doc.add(new Field("unindexed", "test2", stored)); doc - .add(new TextField("unstored", "test1")); + .add(new TextField("unstored", "test1", Field.Store.NO)); doc - .add(new TextField("unstored", "test2")); + .add(new TextField("unstored", "test2", Field.Store.NO)); return doc; } @@ -250,10 +245,10 @@ public class TestDocument extends LuceneTestCase { public void testFieldSetValue() throws Exception { - Field field = new Field("id", "id1", StringField.TYPE_STORED); + Field field = new StringField("id", "id1", Field.Store.YES); Document doc = new Document(); doc.add(field); - doc.add(new Field("keyword", "test", StringField.TYPE_STORED)); + doc.add(new StringField("keyword", "test", Field.Store.YES)); Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); @@ -296,85 +291,20 @@ public class TestDocument extends LuceneTestCase { } } - // LUCENE-3682 - public void testTransitionAPI() throws Exception { - Directory dir = newDirectory(); - RandomIndexWriter w = new RandomIndexWriter(random(), dir); - - Document doc = new Document(); - doc.add(new Field("stored", "abc", Field.Store.YES, Field.Index.NO)); - doc.add(new Field("stored_indexed", "abc xyz", Field.Store.YES, Field.Index.NOT_ANALYZED)); - doc.add(new Field("stored_tokenized", "abc xyz", Field.Store.YES, Field.Index.ANALYZED)); - doc.add(new Field("indexed", "abc xyz", Field.Store.NO, Field.Index.NOT_ANALYZED)); - doc.add(new Field("tokenized", "abc xyz", Field.Store.NO, Field.Index.ANALYZED)); - doc.add(new Field("tokenized_reader", new StringReader("abc xyz"))); - doc.add(new Field("tokenized_tokenstream", w.w.getAnalyzer().tokenStream("tokenized_tokenstream", new StringReader("abc xyz")))); - doc.add(new Field("binary", new byte[10])); - doc.add(new Field("tv", "abc xyz", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.YES)); - doc.add(new Field("tv_pos", "abc xyz", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS)); - doc.add(new Field("tv_off", "abc xyz", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_OFFSETS)); - doc.add(new Field("tv_pos_off", "abc xyz", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); - w.addDocument(doc); - IndexReader r = w.getReader(); - w.close(); - - doc = r.document(0); - // 4 stored fields - assertEquals(4, doc.getFields().size()); - assertEquals("abc", doc.get("stored")); - assertEquals("abc xyz", doc.get("stored_indexed")); - assertEquals("abc xyz", doc.get("stored_tokenized")); - final BytesRef br = doc.getBinaryValue("binary"); - assertNotNull(br); - assertEquals(10, br.length); - - IndexSearcher s = new IndexSearcher(r); - assertEquals(1, s.search(new TermQuery(new Term("stored_indexed", "abc xyz")), 1).totalHits); - assertEquals(1, s.search(new TermQuery(new Term("stored_tokenized", "abc")), 1).totalHits); - assertEquals(1, s.search(new TermQuery(new Term("stored_tokenized", "xyz")), 1).totalHits); - assertEquals(1, s.search(new TermQuery(new Term("indexed", "abc xyz")), 1).totalHits); - assertEquals(1, s.search(new TermQuery(new Term("tokenized", "abc")), 1).totalHits); - assertEquals(1, s.search(new TermQuery(new Term("tokenized", "xyz")), 1).totalHits); - assertEquals(1, s.search(new TermQuery(new Term("tokenized_reader", "abc")), 1).totalHits); - assertEquals(1, s.search(new TermQuery(new Term("tokenized_reader", "xyz")), 1).totalHits); - assertEquals(1, s.search(new TermQuery(new Term("tokenized_tokenstream", "abc")), 1).totalHits); - assertEquals(1, s.search(new TermQuery(new Term("tokenized_tokenstream", "xyz")), 1).totalHits); - - for(String field : new String[] {"tv", "tv_pos", "tv_off", "tv_pos_off"}) { - Fields tvFields = r.getTermVectors(0); - Terms tvs = tvFields.terms(field); - assertNotNull(tvs); - assertEquals(2, tvs.size()); - TermsEnum tvsEnum = tvs.iterator(null); - assertEquals(new BytesRef("abc"), tvsEnum.next()); - final DocsAndPositionsEnum dpEnum = tvsEnum.docsAndPositions(null, null, false); - if (field.equals("tv")) { - assertNull(dpEnum); - } else { - assertNotNull(dpEnum); - } - assertEquals(new BytesRef("xyz"), tvsEnum.next()); - assertNull(tvsEnum.next()); - } - - r.close(); - dir.close(); - } - public void testBoost() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); iwc.setMergePolicy(newLogMergePolicy()); IndexWriter iw = new IndexWriter(dir, iwc); Document doc = new Document(); - doc.add(new Field("field1", "sometext", StringField.TYPE_STORED)); - doc.add(new TextField("field2", "sometext")); - doc.add(new StringField("foo", "bar")); + doc.add(new StringField("field1", "sometext", Field.Store.YES)); + doc.add(new TextField("field2", "sometext", Field.Store.NO)); + doc.add(new StringField("foo", "bar", Field.Store.NO)); iw.addDocument(doc); // add an 'ok' document try { doc = new Document(); // try to boost with norms omitted - StringField field = new StringField("foo", "baz"); + StringField field = new StringField("foo", "baz", Field.Store.NO); field.setBoost(5.0f); doc.add(field); iw.addDocument(doc); diff --git a/lucene/core/src/test/org/apache/lucene/index/BinaryTokenStream.java b/lucene/core/src/test/org/apache/lucene/index/BinaryTokenStream.java index 4ff5fbc..21ff12b 100644 --- a/lucene/core/src/test/org/apache/lucene/index/BinaryTokenStream.java +++ b/lucene/core/src/test/org/apache/lucene/index/BinaryTokenStream.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,8 +17,6 @@ package org.apache.lucene.index; * limitations under the License. */ -import java.io.IOException; - import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; import org.apache.lucene.util.AttributeImpl; @@ -36,7 +34,7 @@ public final class BinaryTokenStream extends TokenStream { } @Override - public boolean incrementToken() throws IOException { + public boolean incrementToken() { if (available) { available = false; return true; @@ -45,7 +43,7 @@ public final class BinaryTokenStream extends TokenStream { } @Override - public void reset() throws IOException { + public void reset() { available = true; } diff --git a/lucene/core/src/test/org/apache/lucene/index/Test2BDocs.java b/lucene/core/src/test/org/apache/lucene/index/Test2BDocs.java new file mode 100644 index 0000000..e529a05 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/index/Test2BDocs.java @@ -0,0 +1,83 @@ +package org.apache.lucene.index; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Arrays; + +import org.apache.lucene.document.Document; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +public class Test2BDocs extends LuceneTestCase { + static Directory dir; + + @BeforeClass + public static void beforeClass() throws Exception { + dir = newFSDirectory(_TestUtil.getTempDir("2Bdocs")); + IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null)); + Document doc = new Document(); + for (int i = 0; i < 262144; i++) { + iw.addDocument(doc); + } + iw.forceMerge(1); + iw.close(); + } + + @AfterClass + public static void afterClass() throws Exception { + dir.close(); + dir = null; + } + + public void testOverflow() throws Exception { + DirectoryReader ir = DirectoryReader.open(dir); + IndexReader subReaders[] = new IndexReader[8192]; + Arrays.fill(subReaders, ir); + try { + new MultiReader(subReaders); + fail(); + } catch (IllegalArgumentException expected) { + // expected + } + ir.close(); + } + + public void testExactlyAtLimit() throws Exception { + Directory dir2 = newFSDirectory(_TestUtil.getTempDir("2BDocs2")); + IndexWriter iw = new IndexWriter(dir2, new IndexWriterConfig(TEST_VERSION_CURRENT, null)); + Document doc = new Document(); + for (int i = 0; i < 262143; i++) { + iw.addDocument(doc); + } + iw.close(); + DirectoryReader ir = DirectoryReader.open(dir); + DirectoryReader ir2 = DirectoryReader.open(dir2); + IndexReader subReaders[] = new IndexReader[8192]; + Arrays.fill(subReaders, ir); + subReaders[subReaders.length-1] = ir2; + MultiReader mr = new MultiReader(subReaders); + assertEquals(Integer.MAX_VALUE, mr.maxDoc()); + assertEquals(Integer.MAX_VALUE, mr.numDocs()); + ir.close(); + ir2.close(); + dir2.close(); + } +} diff --git a/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java b/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java index c9d0c7d..ed3e83a 100644 --- a/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java +++ b/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,8 +17,6 @@ package org.apache.lucene.index; * limitations under the License. */ -import java.io.IOException; - import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; @@ -60,7 +58,7 @@ public class Test2BPostings extends LuceneTestCase { } Document doc = new Document(); - FieldType ft = new FieldType(TextField.TYPE_UNSTORED); + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); ft.setOmitNorms(true); ft.setIndexOptions(IndexOptions.DOCS_ONLY); Field field = new Field("field", new MyTokenStream(), ft); @@ -94,7 +92,7 @@ public class Test2BPostings extends LuceneTestCase { } @Override - public boolean incrementToken() throws IOException { + public boolean incrementToken() { if (index <= 'z') { buffer[0] = (char) index++; return true; @@ -103,7 +101,7 @@ public class Test2BPostings extends LuceneTestCase { } @Override - public void reset() throws IOException { + public void reset() { index = 'a'; } } diff --git a/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java b/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java index 431773b..b4f282e 100644 --- a/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java +++ b/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,24 +23,19 @@ import org.apache.lucene.store.*; import org.apache.lucene.search.*; import org.apache.lucene.analysis.*; import org.apache.lucene.analysis.tokenattributes.*; -import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.*; import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.junit.Ignore; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Random; -import org.junit.Ignore; -// NOTE: this test will fail w/ PreFlexRW codec! (Because -// this test uses full binary term space, but PreFlex cannot -// handle this since it requires the terms are UTF8 bytes). -// -// Also, SimpleText codec will consume very large amounts of +// NOTE: SimpleText codec will consume very large amounts of // disk (but, should run successfully). Best to run w/ -// -Dtests.codec=Standard, and w/ plenty of RAM, eg: +// -Dtests.codec=, and w/ plenty of RAM, eg: // // ant test -Dtest.slow=true -Dtests.heapsize=8g // @@ -141,12 +136,9 @@ public class Test2BTerms extends LuceneTestCase { } } - @Slow + @Ignore("Very slow. Enable manually by removing @Ignore.") public void test2BTerms() throws IOException { - if ("Lucene3x".equals(Codec.getDefault().getName())) { - throw new RuntimeException("this test cannot run with PreFlex codec"); - } System.out.println("Starting Test2B"); final long TERM_COUNT = ((long) Integer.MAX_VALUE) + 100000000; @@ -178,7 +170,7 @@ public class Test2BTerms extends LuceneTestCase { Document doc = new Document(); final MyTokenStream ts = new MyTokenStream(random(), TERMS_PER_DOC); - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setIndexOptions(IndexOptions.DOCS_ONLY); customType.setOmitNorms(true); Field field = new Field("field", ts, customType); @@ -203,7 +195,7 @@ public class Test2BTerms extends LuceneTestCase { } System.out.println("TEST: open reader"); - final IndexReader r = IndexReader.open(dir); + final IndexReader r = DirectoryReader.open(dir); if (savedTerms == null) { savedTerms = findTerms(r); } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java b/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java index 6c261dd..7d394ec 100755 --- a/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -45,7 +45,6 @@ import org.apache.lucene.codecs.pulsing.Pulsing40PostingsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.PackedLongDocValuesField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; @@ -179,8 +178,8 @@ public class TestAddIndexes extends LuceneTestCase { // docs, so 10 pending deletes: for (int i = 0; i < 20; i++) { Document doc = new Document(); - doc.add(newField("id", "" + (i % 10), StringField.TYPE_UNSTORED)); - doc.add(newField("content", "bbb " + i, TextField.TYPE_UNSTORED)); + doc.add(newStringField("id", "" + (i % 10), Field.Store.NO)); + doc.add(newTextField("content", "bbb " + i, Field.Store.NO)); writer.updateDocument(new Term("id", "" + (i%10)), doc); } // Deletes one of the 10 added docs, leaving 9: @@ -214,8 +213,8 @@ public class TestAddIndexes extends LuceneTestCase { // docs, so 10 pending deletes: for (int i = 0; i < 20; i++) { Document doc = new Document(); - doc.add(newField("id", "" + (i % 10), StringField.TYPE_UNSTORED)); - doc.add(newField("content", "bbb " + i, TextField.TYPE_UNSTORED)); + doc.add(newStringField("id", "" + (i % 10), Field.Store.NO)); + doc.add(newTextField("content", "bbb " + i, Field.Store.NO)); writer.updateDocument(new Term("id", "" + (i%10)), doc); } @@ -252,8 +251,8 @@ public class TestAddIndexes extends LuceneTestCase { // docs, so 10 pending deletes: for (int i = 0; i < 20; i++) { Document doc = new Document(); - doc.add(newField("id", "" + (i % 10), StringField.TYPE_UNSTORED)); - doc.add(newField("content", "bbb " + i, TextField.TYPE_UNSTORED)); + doc.add(newStringField("id", "" + (i % 10), Field.Store.NO)); + doc.add(newTextField("content", "bbb " + i, Field.Store.NO)); writer.updateDocument(new Term("id", "" + (i%10)), doc); } @@ -433,7 +432,7 @@ public class TestAddIndexes extends LuceneTestCase { writer.deleteDocuments(new Term("id", "" + i)); } writer.close(); - IndexReader reader = IndexReader.open(aux); + IndexReader reader = DirectoryReader.open(aux); assertEquals(10, reader.numDocs()); reader.close(); @@ -485,7 +484,7 @@ public class TestAddIndexes extends LuceneTestCase { writer.deleteDocuments(new Term("id", "" + i)); } writer.close(); - IndexReader reader = IndexReader.open(aux); + IndexReader reader = DirectoryReader.open(aux); assertEquals(3, reader.numDocs()); reader.close(); @@ -496,7 +495,7 @@ public class TestAddIndexes extends LuceneTestCase { writer.deleteDocuments(new Term("id", "" + i)); } writer.close(); - reader = IndexReader.open(aux2); + reader = DirectoryReader.open(aux2); assertEquals(22, reader.numDocs()); reader.close(); @@ -527,7 +526,7 @@ public class TestAddIndexes extends LuceneTestCase { private void addDocs(IndexWriter writer, int numDocs) throws IOException { for (int i = 0; i < numDocs; i++) { Document doc = new Document(); - doc.add(newField("content", "aaa", TextField.TYPE_UNSTORED)); + doc.add(newTextField("content", "aaa", Field.Store.NO)); writer.addDocument(doc); } } @@ -535,13 +534,13 @@ public class TestAddIndexes extends LuceneTestCase { private void addDocs2(IndexWriter writer, int numDocs) throws IOException { for (int i = 0; i < numDocs; i++) { Document doc = new Document(); - doc.add(newField("content", "bbb", TextField.TYPE_UNSTORED)); + doc.add(newTextField("content", "bbb", Field.Store.NO)); writer.addDocument(doc); } } private void verifyNumDocs(Directory dir, int numDocs) throws IOException { - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); assertEquals(numDocs, reader.maxDoc()); assertEquals(numDocs, reader.numDocs()); reader.close(); @@ -549,7 +548,7 @@ public class TestAddIndexes extends LuceneTestCase { private void verifyTermDocs(Directory dir, Term term, int numDocs) throws IOException { - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); DocsEnum docsEnum = _TestUtil.docs(random(), reader, term.field, term.bytes, null, null, false); int count = 0; while (docsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) @@ -654,7 +653,7 @@ public class TestAddIndexes extends LuceneTestCase { private void addDoc(IndexWriter writer) throws IOException { Document doc = new Document(); - doc.add(newField("content", "aaa", TextField.TYPE_UNSTORED)); + doc.add(newTextField("content", "aaa", Field.Store.NO)); writer.addDocument(doc); } @@ -687,7 +686,7 @@ public class TestAddIndexes extends LuceneTestCase { readers = new IndexReader[NUM_COPY]; for(int i=0;i oldIndexDirs; @@ -168,8 +184,8 @@ public class TestBackwardsCompatibility extends LuceneTestCase { IndexReader reader = null; IndexWriter writer = null; try { - reader = IndexReader.open(dir); - fail("IndexReader.open should not pass for "+unsupportedNames[i]); + reader = DirectoryReader.open(dir); + fail("DirectoryReader.open should not pass for "+unsupportedNames[i]); } catch (IndexFormatTooOldException e) { // pass } finally { @@ -248,7 +264,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase { public void testAddOldIndexesReader() throws IOException { for (String name : oldNames) { - IndexReader reader = IndexReader.open(oldIndexDirs.get(name)); + IndexReader reader = DirectoryReader.open(oldIndexDirs.get(name)); Directory targetDir = newDirectory(); IndexWriter w = new IndexWriter(targetDir, newIndexWriterConfig( @@ -299,10 +315,14 @@ public class TestBackwardsCompatibility extends LuceneTestCase { //QueryParser parser = new QueryParser("contents", new MockAnalyzer(random)); //Query query = parser.parse("handle:1"); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); _TestUtil.checkIndex(dir); + + // true if this is a 4.0+ index + final boolean is40Index = MultiFields.getMergedFieldInfos(reader).fieldInfo("content5") != null; + assert is40Index; // NOTE: currently we can only do this on trunk! final Bits liveDocs = MultiFields.getLiveDocs(reader); @@ -310,8 +330,9 @@ public class TestBackwardsCompatibility extends LuceneTestCase { if (liveDocs.get(i)) { Document d = reader.document(i); List fields = d.getFields(); - if (d.getField("content3") == null) { - final int numFields = 5; + boolean isProxDoc = d.getField("content3") == null; + if (isProxDoc) { + final int numFields = is40Index ? 7 : 5; assertEquals(numFields, fields.size()); IndexableField f = d.getField("id"); assertEquals(""+i, f.stringValue()); @@ -339,18 +360,69 @@ public class TestBackwardsCompatibility extends LuceneTestCase { } } + if (is40Index) { + // check docvalues fields + Source dvByte = MultiDocValues.getDocValues(reader, "dvByte").getSource(); + Source dvBytesDerefFixed = MultiDocValues.getDocValues(reader, "dvBytesDerefFixed").getSource(); + Source dvBytesDerefVar = MultiDocValues.getDocValues(reader, "dvBytesDerefVar").getSource(); + Source dvBytesSortedFixed = MultiDocValues.getDocValues(reader, "dvBytesSortedFixed").getSource(); + Source dvBytesSortedVar = MultiDocValues.getDocValues(reader, "dvBytesSortedVar").getSource(); + Source dvBytesStraightFixed = MultiDocValues.getDocValues(reader, "dvBytesStraightFixed").getSource(); + Source dvBytesStraightVar = MultiDocValues.getDocValues(reader, "dvBytesStraightVar").getSource(); + Source dvDouble = MultiDocValues.getDocValues(reader, "dvDouble").getSource(); + Source dvFloat = MultiDocValues.getDocValues(reader, "dvFloat").getSource(); + Source dvInt = MultiDocValues.getDocValues(reader, "dvInt").getSource(); + Source dvLong = MultiDocValues.getDocValues(reader, "dvLong").getSource(); + Source dvPacked = MultiDocValues.getDocValues(reader, "dvPacked").getSource(); + Source dvShort = MultiDocValues.getDocValues(reader, "dvShort").getSource(); + + for (int i=0;i<35;i++) { + int id = Integer.parseInt(reader.document(i).get("id")); + assertEquals((byte)id, dvByte.getInt(i)); + + byte bytes[] = new byte[] { + (byte)(id >>> 24), (byte)(id >>> 16),(byte)(id >>> 8),(byte)id + }; + BytesRef expectedRef = new BytesRef(bytes); + BytesRef scratch = new BytesRef(); + + assertEquals(expectedRef, dvBytesDerefFixed.getBytes(i, scratch)); + assertEquals(expectedRef, dvBytesDerefVar.getBytes(i, scratch)); + assertEquals(expectedRef, dvBytesSortedFixed.getBytes(i, scratch)); + assertEquals(expectedRef, dvBytesSortedVar.getBytes(i, scratch)); + assertEquals(expectedRef, dvBytesStraightFixed.getBytes(i, scratch)); + assertEquals(expectedRef, dvBytesStraightVar.getBytes(i, scratch)); + + assertEquals((double)id, dvDouble.getFloat(i), 0D); + assertEquals((float)id, dvFloat.getFloat(i), 0F); + assertEquals(id, dvInt.getInt(i)); + assertEquals(id, dvLong.getInt(i)); + assertEquals(id, dvPacked.getInt(i)); + assertEquals(id, dvShort.getInt(i)); + } + } + ScoreDoc[] hits = searcher.search(new TermQuery(new Term(new String("content"), "aaa")), null, 1000).scoreDocs; - // First document should be #21 since it's norm was - // increased: + // First document should be #0 Document d = searcher.getIndexReader().document(hits[0].doc); - assertEquals("didn't get the right document first", "21", d.get("id")); + assertEquals("didn't get the right document first", "0", d.get("id")); doTestHits(hits, 34, searcher.getIndexReader()); + + if (is40Index) { + hits = searcher.search(new TermQuery(new Term(new String("content5"), "aaa")), null, 1000).scoreDocs; + + doTestHits(hits, 34, searcher.getIndexReader()); + + hits = searcher.search(new TermQuery(new Term(new String("content6"), "aaa")), null, 1000).scoreDocs; + + doTestHits(hits, 34, searcher.getIndexReader()); + } hits = searcher.search(new TermQuery(new Term("utf8", "\u0000")), null, 1000).scoreDocs; assertEquals(34, hits.length); - hits = searcher.search(new TermQuery(new Term(new String("utf8"), "Lu\uD834\uDD1Ece\uD834\uDD60ne")), null, 1000).scoreDocs; + hits = searcher.search(new TermQuery(new Term(new String("utf8"), "lu\uD834\uDD1Ece\uD834\uDD60ne")), null, 1000).scoreDocs; assertEquals(34, hits.length); hits = searcher.search(new TermQuery(new Term("utf8", "ab\ud917\udc17cd")), null, 1000).scoreDocs; assertEquals(34, hits.length); @@ -366,7 +438,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase { public void changeIndexWithAdds(Random random, Directory dir, String origOldName) throws IOException { // open writer - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND)); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy())); // add 10 docs for(int i=0;i<10;i++) { addDoc(writer, 35+i); @@ -383,37 +455,37 @@ public class TestBackwardsCompatibility extends LuceneTestCase { writer.close(); // make sure searching sees right # hits - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs; Document d = searcher.getIndexReader().document(hits[0].doc); - assertEquals("wrong first document", "21", d.get("id")); + assertEquals("wrong first document", "0", d.get("id")); doTestHits(hits, 44, searcher.getIndexReader()); reader.close(); // fully merge - writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND)); + writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy())); writer.forceMerge(1); writer.close(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); searcher = new IndexSearcher(reader); hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs; assertEquals("wrong number of hits", 44, hits.length); d = searcher.doc(hits[0].doc); doTestHits(hits, 44, searcher.getIndexReader()); - assertEquals("wrong first document", "21", d.get("id")); + assertEquals("wrong first document", "0", d.get("id")); reader.close(); } public void changeIndexNoAdds(Random random, Directory dir) throws IOException { // make sure searching sees right # hits - IndexReader reader = IndexReader.open(dir); + DirectoryReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs; assertEquals("wrong number of hits", 34, hits.length); Document d = searcher.doc(hits[0].doc); - assertEquals("wrong first document", "21", d.get("id")); + assertEquals("wrong first document", "0", d.get("id")); reader.close(); // fully merge @@ -421,7 +493,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase { writer.forceMerge(1); writer.close(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); searcher = new IndexSearcher(reader); hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs; assertEquals("wrong number of hits", 34, hits.length); @@ -431,7 +503,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase { public File createIndex(String dirName, boolean doCFS, boolean fullyMerged) throws IOException { // we use a real directory name that is not cleaned up, because this method is only used to create backwards indexes: - File indexDir = new File(LuceneTestCase.TEMP_DIR, dirName); + File indexDir = new File("/tmp/4x", dirName); _TestUtil.rmDir(indexDir); Directory dir = newFSDirectory(indexDir); LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy(); @@ -555,8 +627,8 @@ public class TestBackwardsCompatibility extends LuceneTestCase { private void addDoc(IndexWriter writer, int id) throws IOException { Document doc = new Document(); - doc.add(new TextField("content", "aaa")); - doc.add(new Field("id", Integer.toString(id), StringField.TYPE_STORED)); + doc.add(new TextField("content", "aaa", Field.Store.NO)); + doc.add(new StringField("id", Integer.toString(id), Field.Store.YES)); FieldType customType2 = new FieldType(TextField.TYPE_STORED); customType2.setStoreTermVectors(true); customType2.setStoreTermVectorPositions(true); @@ -566,8 +638,43 @@ public class TestBackwardsCompatibility extends LuceneTestCase { doc.add(new Field("content2", "here is more content with aaa aaa aaa", customType2)); doc.add(new Field("fie\u2C77ld", "field with non-ascii name", customType2)); // add numeric fields, to test if flex preserves encoding - doc.add(new IntField("trieInt", id)); - doc.add(new LongField("trieLong", (long) id)); + doc.add(new IntField("trieInt", id, Field.Store.NO)); + doc.add(new LongField("trieLong", (long) id, Field.Store.NO)); + // add docvalues fields + doc.add(new ByteDocValuesField("dvByte", (byte) id)); + byte bytes[] = new byte[] { + (byte)(id >>> 24), (byte)(id >>> 16),(byte)(id >>> 8),(byte)id + }; + BytesRef ref = new BytesRef(bytes); + doc.add(new DerefBytesDocValuesField("dvBytesDerefFixed", ref, true)); + doc.add(new DerefBytesDocValuesField("dvBytesDerefVar", ref, false)); + doc.add(new SortedBytesDocValuesField("dvBytesSortedFixed", ref, true)); + doc.add(new SortedBytesDocValuesField("dvBytesSortedVar", ref, false)); + doc.add(new StraightBytesDocValuesField("dvBytesStraightFixed", ref, true)); + doc.add(new StraightBytesDocValuesField("dvBytesStraightVar", ref, false)); + doc.add(new DoubleDocValuesField("dvDouble", (double)id)); + doc.add(new FloatDocValuesField("dvFloat", (float)id)); + doc.add(new IntDocValuesField("dvInt", id)); + doc.add(new LongDocValuesField("dvLong", id)); + doc.add(new PackedLongDocValuesField("dvPacked", id)); + doc.add(new ShortDocValuesField("dvShort", (short)id)); + // a field with both offsets and term vectors for a cross-check + FieldType customType3 = new FieldType(TextField.TYPE_STORED); + customType3.setStoreTermVectors(true); + customType3.setStoreTermVectorPositions(true); + customType3.setStoreTermVectorOffsets(true); + customType3.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); + doc.add(new Field("content5", "here is more content with aaa aaa aaa", customType3)); + // a field that omits only positions + FieldType customType4 = new FieldType(TextField.TYPE_STORED); + customType4.setStoreTermVectors(true); + customType4.setStoreTermVectorPositions(false); + customType4.setStoreTermVectorOffsets(true); + customType4.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + doc.add(new Field("content6", "here is more content with aaa aaa aaa", customType4)); + // TODO: + // index different norms types via similarity (we use a random one currently?!) + // remove any analyzer randomness, explicitly add payloads for certain fields. writer.addDocument(doc); } @@ -597,7 +704,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase { public void testNextIntoWrongField() throws Exception { for (String name : oldNames) { Directory dir = oldIndexDirs.get(name); - IndexReader r = IndexReader.open(dir); + IndexReader r = DirectoryReader.open(dir); TermsEnum terms = MultiFields.getFields(r).terms("content").iterator(null); BytesRef t = terms.next(); assertNotNull(t); @@ -639,7 +746,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase { for (String name : oldNames) { Directory dir = oldIndexDirs.get(name); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); for (int id=10; id<15; id++) { @@ -760,16 +867,4 @@ public class TestBackwardsCompatibility extends LuceneTestCase { dir.close(); } } - - public static final String surrogatesIndexName = "index.36.surrogates.zip"; - - public void testSurrogates() throws Exception { - File oldIndexDir = _TestUtil.getTempDir("surrogates"); - _TestUtil.unzip(getDataFile(surrogatesIndexName), oldIndexDir); - Directory dir = newFSDirectory(oldIndexDir); - // TODO: more tests - _TestUtil.checkIndex(dir); - dir.close(); - } - } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestBinaryTerms.java b/lucene/core/src/test/org/apache/lucene/index/TestBinaryTerms.java index e0938b1..3930b29 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestBinaryTerms.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestBinaryTerms.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -29,12 +29,10 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; /** * Test indexing and searching some byte[] terms */ -@SuppressCodecs("Lucene3x") public class TestBinaryTerms extends LuceneTestCase { public void testBinary() throws IOException { Directory dir = newDirectory(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java b/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java index 4438b74..421f188 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -101,7 +101,7 @@ public class TestCheckIndex extends LuceneTestCase { assertTrue("Invalid version: "+version, version.equals(Constants.LUCENE_MAIN_VERSION+"-SNAPSHOT") || version.equals(Constants.LUCENE_MAIN_VERSION)); - assertTrue(version + " should start with: "+Constants.LUCENE_VERSION, + assertTrue(Constants.LUCENE_VERSION + " should start with: "+version, Constants.LUCENE_VERSION.startsWith(version)); } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java index e891b2d..3f55f23 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -28,7 +28,6 @@ import org.apache.lucene.codecs.FieldsProducer; import org.apache.lucene.codecs.PostingsConsumer; import org.apache.lucene.codecs.TermStats; import org.apache.lucene.codecs.TermsConsumer; -import org.apache.lucene.codecs.lucene3x.Lucene3xCodec; import org.apache.lucene.codecs.mocksep.MockSepPostingsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldType; @@ -45,7 +44,6 @@ import org.apache.lucene.util.Constants; import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.OpenBitSet; -import org.apache.lucene.util.Version; import org.apache.lucene.util._TestUtil; import org.junit.BeforeClass; @@ -78,7 +76,7 @@ public class TestCodecs extends LuceneTestCase { private final static int TERM_DOC_FREQ_RAND = 20; @BeforeClass - public static void beforeClass() throws Exception { + public static void beforeClass() { NUM_TEST_ITER = atLeast(20); } @@ -254,7 +252,7 @@ public class TestCodecs extends LuceneTestCase { final FieldData[] fields = new FieldData[] {field}; final FieldInfos fieldInfos = builder.finish(); final Directory dir = newDirectory(); - this.write(fieldInfos, dir, fields, true); + this.write(fieldInfos, dir, fields); Codec codec = Codec.getDefault(); final SegmentInfo si = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, SEGMENT, 10000, false, codec, null, null); @@ -310,7 +308,7 @@ public class TestCodecs extends LuceneTestCase { System.out.println("TEST: now write postings"); } - this.write(fieldInfos, dir, fields, false); + this.write(fieldInfos, dir, fields); Codec codec = Codec.getDefault(); final SegmentInfo si = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, SEGMENT, 10000, false, codec, null, null); @@ -340,8 +338,9 @@ public class TestCodecs extends LuceneTestCase { public void testSepPositionAfterMerge() throws IOException { final Directory dir = newDirectory(); - final IndexWriterConfig config = newIndexWriterConfig(Version.LUCENE_31, + final IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); + config.setMergePolicy(newLogMergePolicy()); config.setCodec(_TestUtil.alwaysPostingsFormat(new MockSepPostingsFormat())); final IndexWriter writer = new IndexWriter(dir, config); @@ -351,7 +350,7 @@ public class TestCodecs extends LuceneTestCase { pq.add(new Term("content", "ccc")); final Document doc = new Document(); - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setOmitNorms(true); doc.add(newField("content", "aaa bbb ccc ddd", customType)); @@ -454,10 +453,6 @@ public class TestCodecs extends LuceneTestCase { for(int iter=0;iter= 157); reader.close(); dir.close(); @@ -129,7 +129,7 @@ public class TestCrash extends LuceneTestCase { System.out.println("file " + i + " = " + l[i] + " " + dir.fileLength(l[i]) + " bytes"); */ - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); assertEquals(157, reader.numDocs()); reader.close(); dir.close(); @@ -150,7 +150,7 @@ public class TestCrash extends LuceneTestCase { for(int i=0;i commits) throws IOException { + private void verifyCommitOrder(List commits) { final IndexCommit firstCommit = commits.get(0); long last = SegmentInfos.generationFromSegmentsFileName(firstCommit.getSegmentsFileName()); assertEquals(last, firstCommit.getGeneration()); @@ -68,7 +68,7 @@ public class TestDeletionPolicy extends LuceneTestCase { public void onCommit(List commits) throws IOException { IndexCommit lastCommit = commits.get(commits.size()-1); DirectoryReader r = DirectoryReader.open(dir); - assertEquals("lastCommit.segmentCount()=" + lastCommit.getSegmentCount() + " vs IndexReader.segmentCount=" + r.getSequentialSubReaders().length, r.getSequentialSubReaders().length, lastCommit.getSegmentCount()); + assertEquals("lastCommit.segmentCount()=" + lastCommit.getSegmentCount() + " vs IndexReader.segmentCount=" + r.getSequentialSubReaders().size(), r.getSequentialSubReaders().size(), lastCommit.getSegmentCount()); r.close(); verifyCommitOrder(commits); numOnCommit++; @@ -215,10 +215,9 @@ public class TestDeletionPolicy extends LuceneTestCase { writer.commit(commitData); writer.close(); - final int ITER = 9; - long lastDeleteTime = 0; - for(int i=0;i 0); - // Then simplistic check: just verify that the // segments_N's that still exist are in fact within SECONDS // seconds of the last one's mod time, and, that I can @@ -263,7 +255,7 @@ public class TestDeletionPolicy extends LuceneTestCase { while(gen > 0) { try { - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); reader.close(); fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", @@ -326,7 +318,7 @@ public class TestDeletionPolicy extends LuceneTestCase { final boolean needsMerging; { DirectoryReader r = DirectoryReader.open(dir); - needsMerging = r.getSequentialSubReaders().length != 1; + needsMerging = r.getSequentialSubReaders().size() != 1; r.close(); } if (needsMerging) { @@ -357,7 +349,7 @@ public class TestDeletionPolicy extends LuceneTestCase { // Make sure we can open a reader on each commit: for (final IndexCommit commit : commits) { - IndexReader r = IndexReader.open(commit); + IndexReader r = DirectoryReader.open(commit); r.close(); } @@ -366,7 +358,7 @@ public class TestDeletionPolicy extends LuceneTestCase { dir.deleteFile(IndexFileNames.SEGMENTS_GEN); long gen = SegmentInfos.getLastCommitGeneration(dir); while(gen > 0) { - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); reader.close(); dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)); gen--; @@ -443,7 +435,7 @@ public class TestDeletionPolicy extends LuceneTestCase { DirectoryReader r = DirectoryReader.open(dir); // Still merged, still 11 docs - assertEquals(1, r.getSequentialSubReaders().length); + assertEquals(1, r.getSequentialSubReaders().size()); assertEquals(11, r.numDocs()); r.close(); @@ -459,7 +451,7 @@ public class TestDeletionPolicy extends LuceneTestCase { r = DirectoryReader.open(dir); // Not fully merged because we rolled it back, and now only // 10 docs - assertTrue(r.getSequentialSubReaders().length > 1); + assertTrue(r.getSequentialSubReaders().size() > 1); assertEquals(10, r.numDocs()); r.close(); @@ -468,8 +460,8 @@ public class TestDeletionPolicy extends LuceneTestCase { writer.forceMerge(1); writer.close(); - r = IndexReader.open(dir); - assertEquals(1, r.getSequentialSubReaders().length); + r = DirectoryReader.open(dir); + assertEquals(1, r.getSequentialSubReaders().size()); assertEquals(10, r.numDocs()); r.close(); @@ -480,16 +472,16 @@ public class TestDeletionPolicy extends LuceneTestCase { // Reader still sees fully merged index, because writer // opened on the prior commit has not yet committed: - r = IndexReader.open(dir); - assertEquals(1, r.getSequentialSubReaders().length); + r = DirectoryReader.open(dir); + assertEquals(1, r.getSequentialSubReaders().size()); assertEquals(10, r.numDocs()); r.close(); writer.close(); // Now reader sees not-fully-merged index: - r = IndexReader.open(dir); - assertTrue(r.getSequentialSubReaders().length > 1); + r = DirectoryReader.open(dir); + assertTrue(r.getSequentialSubReaders().size() > 1); assertEquals(10, r.numDocs()); r.close(); @@ -541,7 +533,7 @@ public class TestDeletionPolicy extends LuceneTestCase { // Simplistic check: just verify the index is in fact // readable: - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); reader.close(); dir.close(); @@ -589,7 +581,7 @@ public class TestDeletionPolicy extends LuceneTestCase { long gen = SegmentInfos.getLastCommitGeneration(dir); for(int i=0;i subs = r2.getSequentialSubReaders(); + for(AtomicReader s : subs) { + assertEquals(36, s.getUniqueTermCount()); } r2.close(); writer.close(); @@ -828,8 +827,8 @@ public void testFilesOpenClose() throws IOException { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()))); Document doc = new Document(); - doc.add(newField("field", "a b c d e f g h i j k l m n o p q r s t u v w x y z", TextField.TYPE_UNSTORED)); - doc.add(newField("number", "0 1 2 3 4 5 6 7 8 9", TextField.TYPE_UNSTORED)); + doc.add(newTextField("field", "a b c d e f g h i j k l m n o p q r s t u v w x y z", Field.Store.NO)); + doc.add(newTextField("number", "0 1 2 3 4 5 6 7 8 9", Field.Store.NO)); writer.addDocument(doc); writer.addDocument(doc); writer.close(); @@ -842,7 +841,7 @@ public void testFilesOpenClose() throws IOException { // expected } - assertEquals(-1, ((SegmentReader) r.getSequentialSubReaders()[0]).getTermInfosIndexDivisor()); + assertEquals(-1, ((SegmentReader) r.getSequentialSubReaders().get(0)).getTermInfosIndexDivisor()); writer = new IndexWriter( dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())). @@ -857,11 +856,11 @@ public void testFilesOpenClose() throws IOException { assertNotNull(r2); assertNull(DirectoryReader.openIfChanged(r2)); r.close(); - IndexReader[] subReaders = r2.getSequentialSubReaders(); - assertEquals(2, subReaders.length); - for(int i=0;i<2;i++) { + List subReaders = r2.getSequentialSubReaders(); + assertEquals(2, subReaders.size()); + for(AtomicReader s : subReaders) { try { - subReaders[i].docFreq(new Term("field", "f")); + s.docFreq(new Term("field", "f")); fail("did not hit expected exception"); } catch (IllegalStateException ise) { // expected @@ -936,7 +935,7 @@ public void testFilesOpenClose() throws IOException { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); Document d = new Document(); - d.add(newField("f", "a a b", TextField.TYPE_UNSTORED)); + d.add(newTextField("f", "a a b", Field.Store.NO)); writer.addDocument(d); DirectoryReader r = writer.getReader(); writer.close(); @@ -1003,7 +1002,7 @@ public void testFilesOpenClose() throws IOException { dir.close(); } - public void testTryIncRef() throws CorruptIndexException, LockObtainFailedException, IOException { + public void testTryIncRef() throws IOException { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); writer.addDocument(new Document()); @@ -1017,7 +1016,7 @@ public void testFilesOpenClose() throws IOException { dir.close(); } - public void testStressTryIncRef() throws CorruptIndexException, LockObtainFailedException, IOException, InterruptedException { + public void testStressTryIncRef() throws IOException, InterruptedException { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); writer.addDocument(new Document()); @@ -1073,8 +1072,8 @@ public void testFilesOpenClose() throws IOException { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); - doc.add(newField("field1", "foobar", StringField.TYPE_STORED)); - doc.add(newField("field2", "foobaz", StringField.TYPE_STORED)); + doc.add(newStringField("field1", "foobar", Field.Store.YES)); + doc.add(newStringField("field2", "foobaz", Field.Store.YES)); writer.addDocument(doc); DirectoryReader r = writer.getReader(); writer.close(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java index a4a9efb..8e6f3ea 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -32,7 +32,6 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.search.IndexSearcher; @@ -470,9 +469,9 @@ public class TestDirectoryReaderReopen extends LuceneTestCase { DirectoryReader r = DirectoryReader.open(dir); if (multiSegment) { - assertTrue(r.getSequentialSubReaders().length > 1); + assertTrue(r.getSequentialSubReaders().size() > 1); } else { - assertTrue(r.getSequentialSubReaders().length == 1); + assertTrue(r.getSequentialSubReaders().size() == 1); } r.close(); } @@ -487,13 +486,13 @@ public class TestDirectoryReaderReopen extends LuceneTestCase { customType2.setOmitNorms(true); FieldType customType3 = new FieldType(); customType3.setStored(true); - doc.add(new Field("field1", sb.toString(), TextField.TYPE_STORED)); + doc.add(new TextField("field1", sb.toString(), Field.Store.YES)); doc.add(new Field("fielda", sb.toString(), customType2)); doc.add(new Field("fieldb", sb.toString(), customType3)); sb.append(" b"); sb.append(n); for (int i = 1; i < numFields; i++) { - doc.add(new Field("field" + (i+1), sb.toString(), TextField.TYPE_STORED)); + doc.add(new TextField("field" + (i+1), sb.toString(), Field.Store.YES)); } return doc; } @@ -543,9 +542,9 @@ public class TestDirectoryReaderReopen extends LuceneTestCase { } if (checkSubReaders && reader instanceof CompositeReader) { - IndexReader[] subReaders = ((CompositeReader) reader).getSequentialSubReaders(); - for (int i = 0; i < subReaders.length; i++) { - assertReaderClosed(subReaders[i], checkSubReaders, checkNormsClosed); + List subReaders = ((CompositeReader) reader).getSequentialSubReaders(); + for (IndexReader r : subReaders) { + assertReaderClosed(r, checkSubReaders, checkNormsClosed); } } } @@ -591,7 +590,7 @@ public class TestDirectoryReaderReopen extends LuceneTestCase { ); for(int i=0;i<4;i++) { Document doc = new Document(); - doc.add(newField("id", ""+i, StringField.TYPE_UNSTORED)); + doc.add(newStringField("id", ""+i, Field.Store.NO)); writer.addDocument(doc); Map data = new HashMap(); data.put("index", i+""); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java index 04bfd96..b345f82 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -25,11 +25,11 @@ import java.io.StringWriter; import java.util.Collection; import java.util.HashSet; import java.util.LinkedList; -import java.util.Set; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.search.DocIdSetIterator; @@ -182,7 +182,7 @@ public class TestDoc extends LuceneTestCase { { File file = new File(workDir, fileName); Document doc = new Document(); - doc.add(new TextField("contents", new FileReader(file))); + doc.add(new TextField("contents", new FileReader(file), Field.Store.NO)); writer.addDocument(doc); writer.commit(); return writer.newestSegment(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocCount.java b/lucene/core/src/test/org/apache/lucene/index/TestDocCount.java index 88baa46..47fb0e6 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDocCount.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDocCount.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,18 +18,16 @@ package org.apache.lucene.index; */ import org.apache.lucene.document.Document; -import org.apache.lucene.document.StringField; +import org.apache.lucene.document.Field; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.Directory; import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; -import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; /** * Tests the Terms.docCount statistic */ -@SuppressCodecs("Lucene3x") public class TestDocCount extends LuceneTestCase { public void testSimple() throws Exception { Directory dir = newDirectory(); @@ -53,7 +51,7 @@ public class TestDocCount extends LuceneTestCase { Document doc = new Document(); int numFields = _TestUtil.nextInt(random(), 1, 10); for (int i = 0; i < numFields; i++) { - doc.add(newField("" + _TestUtil.nextInt(random(), 'a', 'z'), "" + _TestUtil.nextInt(random(), 'a', 'z'), StringField.TYPE_UNSTORED)); + doc.add(newStringField("" + _TestUtil.nextInt(random(), 'a', 'z'), "" + _TestUtil.nextInt(random(), 'a', 'z'), Field.Store.NO)); } return doc; } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocTermOrds.java b/lucene/core/src/test/org/apache/lucene/index/TestDocTermOrds.java index 69fb1d5..7e55009 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDocTermOrds.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDocTermOrds.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -29,8 +29,6 @@ import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.IntField; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; import org.apache.lucene.index.DocTermOrds.TermOrdsIterator; import org.apache.lucene.search.FieldCache; import org.apache.lucene.store.Directory; @@ -52,7 +50,7 @@ public class TestDocTermOrds extends LuceneTestCase { Directory dir = newDirectory(); final RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); Document doc = new Document(); - Field field = newField("field", "", TextField.TYPE_UNSTORED); + Field field = newTextField("field", "", Field.Store.NO); doc.add(field); field.setStringValue("a b c"); w.addDocument(doc); @@ -124,7 +122,7 @@ public class TestDocTermOrds extends LuceneTestCase { for(int id=0;id byteVariantList = new ArrayList(BYTES); @@ -447,7 +457,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { d.close(); } - public void testGetArrayNumerics() throws CorruptIndexException, IOException { + public void testGetArrayNumerics() throws IOException { Directory d = newDirectory(); IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); IndexWriter w = new IndexWriter(d, cfg); @@ -529,7 +539,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { d.close(); } - public void testGetArrayBytes() throws CorruptIndexException, IOException { + public void testGetArrayBytes() throws IOException { Directory d = newDirectory(); IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); @@ -600,7 +610,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { private FixedBitSet indexValues(IndexWriter w, int numValues, Type valueType, List valueVarList, boolean withDeletions, int bytesSize) - throws CorruptIndexException, IOException { + throws IOException { final boolean isNumeric = NUMERICS.contains(valueType); FixedBitSet deleted = new FixedBitSet(numValues); Document doc = new Document(); @@ -704,7 +714,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { } } doc.removeFields("id"); - doc.add(new Field("id", idBase + i, StringField.TYPE_STORED)); + doc.add(new StringField("id", idBase + i, Field.Store.YES)); w.addDocument(doc); if (i % 7 == 0) { @@ -754,7 +764,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { w.forceMerge(1); DirectoryReader r = w.getReader(); w.close(); - assertEquals(17, r.getSequentialSubReaders()[0].docValues("field").load().getInt(0)); + assertEquals(17, getOnlySegmentReader(r).docValues("field").load().getInt(0)); r.close(); d.close(); } @@ -800,7 +810,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { int len = 1 + random().nextInt(50); for (int i = 0; i < numDocs; i++) { Document doc = new Document(); - doc.add(newField("id", "" + i, TextField.TYPE_STORED)); + doc.add(newTextField("id", "" + i, Field.Store.YES)); String string = fixed ? _TestUtil.randomFixedByteLengthUnicodeString(random(), len) : _TestUtil.randomRealisticUnicodeString(random(), 1, len); BytesRef br = new BytesRef(string); @@ -815,7 +825,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { int numDocsNoValue = atLeast(10); for (int i = 0; i < numDocsNoValue; i++) { Document doc = new Document(); - doc.add(newField("id", "noValue", TextField.TYPE_STORED)); + doc.add(newTextField("id", "noValue", Field.Store.YES)); w.addDocument(doc); } BytesRef bytesRef = new BytesRef(fixed ? len : 0); @@ -828,7 +838,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { for (int i = 0; i < numDocs; i++) { Document doc = new Document(); String id = "" + i + numDocs; - doc.add(newField("id", id, TextField.TYPE_STORED)); + doc.add(newTextField("id", id, Field.Store.YES)); String string = fixed ? _TestUtil.randomFixedByteLengthUnicodeString(random(), len) : _TestUtil.randomRealisticUnicodeString(random(), 1, len); BytesRef br = new BytesRef(string); @@ -858,7 +868,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { for (Entry entry : entrySet) { int docId = docId(slowR, new Term("id", entry.getKey())); - expected.copyChars(entry.getValue()); + expected = new BytesRef(entry.getValue()); assertEquals(expected, asSortedSource.getBytes(docId, actual)); } @@ -998,8 +1008,9 @@ public class TestDocValuesIndexing extends LuceneTestCase { w.addDocument(doc); bytes[0] = 1; w.addDocument(doc); + w.forceMerge(1); DirectoryReader r = w.getReader(); - Source s = r.getSequentialSubReaders()[0].docValues("field").getSource(); + Source s = getOnlySegmentReader(r).docValues("field").getSource(); BytesRef bytes1 = s.getBytes(0, new BytesRef()); assertEquals(bytes.length, bytes1.length); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocValuesTypeCompatibility.java b/lucene/core/src/test/org/apache/lucene/index/TestDocValuesTypeCompatibility.java new file mode 100644 index 0000000..618e057 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/index/TestDocValuesTypeCompatibility.java @@ -0,0 +1,312 @@ +package org.apache.lucene.index; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.ByteDocValuesField; +import org.apache.lucene.document.DerefBytesDocValuesField; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoubleDocValuesField; +import org.apache.lucene.document.FloatDocValuesField; +import org.apache.lucene.document.IntDocValuesField; +import org.apache.lucene.document.LongDocValuesField; +import org.apache.lucene.document.PackedLongDocValuesField; +import org.apache.lucene.document.ShortDocValuesField; +import org.apache.lucene.document.SortedBytesDocValuesField; +import org.apache.lucene.document.StraightBytesDocValuesField; +import org.apache.lucene.index.DocValues.Type; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; + +/** + * Tests compatibility of {@link DocValues.Type} during indexing + */ +public class TestDocValuesTypeCompatibility extends LuceneTestCase { + + public void testAddCompatibleIntTypes() throws IOException { + int numIter = atLeast(10); + for (int i = 0; i < numIter; i++) { + Directory dir = newDirectory(); + IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer(random())); + int numDocs = atLeast(100); + + iwc.setMaxBufferedDocs(2 * numDocs); // make sure we hit the same DWPT + // here + iwc.setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); + iwc.setRAMPerThreadHardLimitMB(2000); + IndexWriter writer = new IndexWriter(dir, iwc); + Type[] types = new Type[] {Type.VAR_INTS, Type.FIXED_INTS_16, + Type.FIXED_INTS_64, Type.FIXED_INTS_16, Type.FIXED_INTS_8}; + Type maxType = types[random().nextInt(types.length)]; + for (int j = 0; j < numDocs; j++) { + addDoc(writer, getRandomIntsField(maxType, j == 0)); + } + writer.close(); + dir.close(); + } + + } + + @SuppressWarnings("fallthrough") + public IndexableField getRandomIntsField(Type maxType, boolean force) { + switch (maxType) { + + case VAR_INTS: + if (random().nextInt(5) == 0 || force) { + return new PackedLongDocValuesField("f", 1); + } + case FIXED_INTS_64: + if (random().nextInt(4) == 0 || force) { + return new LongDocValuesField("f", 1); + } + case FIXED_INTS_32: + if (random().nextInt(3) == 0 || force) { + return new IntDocValuesField("f", 1); + } + case FIXED_INTS_16: + if (random().nextInt(2) == 0 || force) { + return new ShortDocValuesField("f", (short) 1); + } + case FIXED_INTS_8: + return new ByteDocValuesField("f", (byte) 1); + + default: + throw new IllegalArgumentException(); + + } + } + + public void testAddCompatibleDoubleTypes() throws IOException { + int numIter = atLeast(10); + for (int i = 0; i < numIter; i++) { + Directory dir = newDirectory(); + IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer(random())); + int numDocs = atLeast(100); + + iwc.setMaxBufferedDocs(2 * numDocs); // make sure we hit the same DWPT + // here + iwc.setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); + iwc.setRAMPerThreadHardLimitMB(2000); + IndexWriter writer = new IndexWriter(dir, iwc); + + Type[] types = new Type[] {Type.FLOAT_64, Type.FLOAT_32}; + Type maxType = types[random().nextInt(types.length)]; + for (int j = 0; j < numDocs; j++) { + addDoc(writer, getRandomFloatField(maxType, j == 0)); + } + writer.close(); + dir.close(); + } + + } + @SuppressWarnings("fallthrough") + public IndexableField getRandomFloatField(Type maxType, boolean force) { + switch (maxType) { + + case FLOAT_64: + if (random().nextInt(5) == 0 || force) { + return new PackedLongDocValuesField("f", 1); + } + case FIXED_INTS_32: + if (random().nextInt(4) == 0 || force) { + return new LongDocValuesField("f", 1); + } + case FLOAT_32: + if (random().nextInt(3) == 0 || force) { + return new IntDocValuesField("f", 1); + } + case FIXED_INTS_16: + if (random().nextInt(2) == 0 || force) { + return new ShortDocValuesField("f", (short) 1); + } + case FIXED_INTS_8: + return new ByteDocValuesField("f", (byte) 1); + + default: + throw new IllegalArgumentException(); + + } + } + + public void testAddCompatibleDoubleTypes2() throws IOException { + int numIter = atLeast(10); + for (int i = 0; i < numIter; i++) { + Directory dir = newDirectory(); + IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer(random())); + int numDocs = atLeast(100); + + iwc.setMaxBufferedDocs(2 * numDocs); // make sure we hit the same DWPT + // here + iwc.setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); + iwc.setRAMPerThreadHardLimitMB(2000); + IndexWriter writer = new IndexWriter(dir, iwc); + IndexableField[] fields = new IndexableField[] { + new DoubleDocValuesField("f", 1.0), new IntDocValuesField("f", 1), + new ShortDocValuesField("f", (short) 1), + new ByteDocValuesField("f", (byte) 1)}; + int base = random().nextInt(fields.length - 1); + + addDoc(writer, fields[base]); + + for (int j = 0; j < numDocs; j++) { + int f = base + random().nextInt(fields.length - base); + addDoc(writer, fields[f]); + } + writer.close(); + dir.close(); + } + + } + + public void testAddCompatibleByteTypes() throws IOException { + int numIter = atLeast(10); + for (int i = 0; i < numIter; i++) { + Directory dir = newDirectory(); + IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer(random())); + int numDocs = atLeast(100); + + iwc.setMaxBufferedDocs(2 * numDocs); // make sure we hit the same DWPT + // here + iwc.setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); + iwc.setRAMPerThreadHardLimitMB(2000); + IndexWriter writer = new IndexWriter(dir, iwc); + boolean mustBeFixed = random().nextBoolean(); + int maxSize = 2 + random().nextInt(15); + IndexableField bytesField = getRandomBytesField(mustBeFixed, maxSize, + true); + addDoc(writer, bytesField); + for (int j = 0; j < numDocs; j++) { + bytesField = getRandomBytesField(mustBeFixed, maxSize, false); + addDoc(writer, bytesField); + + } + writer.close(); + dir.close(); + } + } + + public IndexableField getRandomBytesField(boolean mustBeFixed, int maxSize, + boolean mustBeVariableIfNotFixed) { + int size = mustBeFixed ? maxSize : random().nextInt(maxSize) + 1; + StringBuilder s = new StringBuilder(); + for (int i = 0; i < size; i++) { + s.append("a"); + } + BytesRef bytesRef = new BytesRef(s); + boolean fixed = mustBeFixed ? true : mustBeVariableIfNotFixed ? false + : random().nextBoolean(); + switch (random().nextInt(3)) { + case 0: + return new SortedBytesDocValuesField("f", bytesRef, fixed); + case 1: + return new DerefBytesDocValuesField("f", bytesRef, fixed); + default: + return new StraightBytesDocValuesField("f", bytesRef, fixed); + } + } + + public void testIncompatibleTypesBytes() throws IOException { + Directory dir = newDirectory(); + IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer(random())); + int numDocs = atLeast(100); + + iwc.setMaxBufferedDocs(numDocs); // make sure we hit the same DWPT + iwc.setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); + iwc.setRAMPerThreadHardLimitMB(2000); + IndexWriter writer = new IndexWriter(dir, iwc); + + int numDocsIndexed = 0; + for (int j = 1; j < numDocs; j++) { + try { + addDoc(writer, getRandomIndexableDVField()); + numDocsIndexed++; + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().startsWith("Incompatible DocValues type:")); + } + } + + writer.commit(); + DirectoryReader open = DirectoryReader.open(dir); + assertEquals(numDocsIndexed, open.numDocs()); + open.close(); + writer.close(); + dir.close(); + } + + private void addDoc(IndexWriter writer, IndexableField... fields) + throws IOException { + Document doc = new Document(); + for (IndexableField indexableField : fields) { + doc.add(indexableField); + } + writer.addDocument(doc); + } + + public IndexableField getRandomIndexableDVField() { + int size = random().nextInt(100) + 1; + StringBuilder s = new StringBuilder(); + for (int i = 0; i < size; i++) { + s.append("a"); + } + BytesRef bytesRef = new BytesRef(s); + + Type[] values = Type.values(); + Type t = values[random().nextInt(values.length)]; + switch (t) { + case BYTES_FIXED_DEREF: + return new DerefBytesDocValuesField("f", bytesRef, true); + case BYTES_FIXED_SORTED: + return new SortedBytesDocValuesField("f", bytesRef, true); + case BYTES_FIXED_STRAIGHT: + return new StraightBytesDocValuesField("f", bytesRef, true); + case BYTES_VAR_DEREF: + return new DerefBytesDocValuesField("f", bytesRef, false); + case BYTES_VAR_SORTED: + return new SortedBytesDocValuesField("f", bytesRef, false); + case BYTES_VAR_STRAIGHT: + return new StraightBytesDocValuesField("f", bytesRef, false); + case FIXED_INTS_16: + return new ShortDocValuesField("f", (short) 1); + case FIXED_INTS_32: + return new IntDocValuesField("f", 1); + case FIXED_INTS_64: + return new LongDocValuesField("f", 1); + case FIXED_INTS_8: + return new ByteDocValuesField("f", (byte) 1); + case FLOAT_32: + return new FloatDocValuesField("f", 1.0f); + case FLOAT_64: + return new DoubleDocValuesField("f", 1.0f); + case VAR_INTS: + return new PackedLongDocValuesField("f", 1); + default: + throw new IllegalArgumentException(); + + } + + } + +} diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java b/lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java index 3dfaac2..48d0dc8 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,8 +22,8 @@ import java.util.Arrays; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.Directory; @@ -50,7 +50,7 @@ public class TestDocsAndPositions extends LuceneTestCase { newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); for (int i = 0; i < 39; i++) { Document doc = new Document(); - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setOmitNorms(true); doc.add(newField(fieldName, "1 2 3 4 5 6 7 8 9 10 " + "1 2 3 4 5 6 7 8 9 10 " + "1 2 3 4 5 6 7 8 9 10 " @@ -64,8 +64,7 @@ public class TestDocsAndPositions extends LuceneTestCase { for (int i = 0; i < num; i++) { BytesRef bytes = new BytesRef("1"); IndexReaderContext topReaderContext = reader.getTopReaderContext(); - AtomicReaderContext[] leaves = topReaderContext.leaves(); - for (AtomicReaderContext atomicReaderContext : leaves) { + for (AtomicReaderContext atomicReaderContext : topReaderContext.leaves()) { DocsAndPositionsEnum docsAndPosEnum = getDocsAndPositions( atomicReaderContext.reader(), bytes, null); assertNotNull(docsAndPosEnum); @@ -110,7 +109,7 @@ public class TestDocsAndPositions extends LuceneTestCase { int max = 1051; int term = random().nextInt(max); Integer[][] positionsInDoc = new Integer[numDocs][]; - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setOmitNorms(true); for (int i = 0; i < numDocs; i++) { Document doc = new Document(); @@ -140,8 +139,7 @@ public class TestDocsAndPositions extends LuceneTestCase { for (int i = 0; i < num; i++) { BytesRef bytes = new BytesRef("" + term); IndexReaderContext topReaderContext = reader.getTopReaderContext(); - AtomicReaderContext[] leaves = topReaderContext.leaves(); - for (AtomicReaderContext atomicReaderContext : leaves) { + for (AtomicReaderContext atomicReaderContext : topReaderContext.leaves()) { DocsAndPositionsEnum docsAndPosEnum = getDocsAndPositions( atomicReaderContext.reader(), bytes, null); assertNotNull(docsAndPosEnum); @@ -193,7 +191,7 @@ public class TestDocsAndPositions extends LuceneTestCase { int max = 15678; int term = random().nextInt(max); int[] freqInDoc = new int[numDocs]; - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setOmitNorms(true); for (int i = 0; i < numDocs; i++) { Document doc = new Document(); @@ -216,8 +214,7 @@ public class TestDocsAndPositions extends LuceneTestCase { for (int i = 0; i < num; i++) { BytesRef bytes = new BytesRef("" + term); IndexReaderContext topReaderContext = reader.getTopReaderContext(); - AtomicReaderContext[] leaves = topReaderContext.leaves(); - for (AtomicReaderContext context : leaves) { + for (AtomicReaderContext context : topReaderContext.leaves()) { int maxDoc = context.reader().maxDoc(); DocsEnum docsEnum = _TestUtil.docs(random(), context.reader(), fieldName, bytes, null, null, true); if (findNext(freqInDoc, context.docBase, context.docBase + maxDoc) == Integer.MAX_VALUE) { @@ -270,7 +267,7 @@ public class TestDocsAndPositions extends LuceneTestCase { RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); int howMany = 1000; - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setOmitNorms(true); for (int i = 0; i < 39; i++) { Document doc = new Document(); @@ -295,8 +292,7 @@ public class TestDocsAndPositions extends LuceneTestCase { BytesRef bytes = new BytesRef("even"); IndexReaderContext topReaderContext = reader.getTopReaderContext(); - AtomicReaderContext[] leaves = topReaderContext.leaves(); - for (AtomicReaderContext atomicReaderContext : leaves) { + for (AtomicReaderContext atomicReaderContext : topReaderContext.leaves()) { DocsAndPositionsEnum docsAndPosEnum = getDocsAndPositions( atomicReaderContext.reader(), bytes, null); assertNotNull(docsAndPosEnum); @@ -326,7 +322,7 @@ public class TestDocsAndPositions extends LuceneTestCase { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); - doc.add(newField("foo", "bar", StringField.TYPE_UNSTORED)); + doc.add(newStringField("foo", "bar", Field.Store.NO)); writer.addDocument(doc); DirectoryReader reader = writer.getReader(); AtomicReader r = getOnlySegmentReader(reader); @@ -351,7 +347,7 @@ public class TestDocsAndPositions extends LuceneTestCase { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); - doc.add(newField("foo", "bar", TextField.TYPE_UNSTORED)); + doc.add(newTextField("foo", "bar", Field.Store.NO)); writer.addDocument(doc); DirectoryReader reader = writer.getReader(); AtomicReader r = getOnlySegmentReader(reader); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java index eab5a63..11445dd 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -119,8 +119,8 @@ public class TestDocumentWriter extends LuceneTestCase { IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer)); Document doc = new Document(); - doc.add(newField("repeated", "repeated one", TextField.TYPE_STORED)); - doc.add(newField("repeated", "repeated two", TextField.TYPE_STORED)); + doc.add(newTextField("repeated", "repeated one", Field.Store.YES)); + doc.add(newTextField("repeated", "repeated two", Field.Store.YES)); writer.addDocument(doc); writer.commit(); @@ -165,7 +165,7 @@ public class TestDocumentWriter extends LuceneTestCase { } if (first) { // set payload on first position only - payloadAtt.setPayload(new Payload(new byte[]{100})); + payloadAtt.setPayload(new BytesRef(new byte[]{100})); first = false; } @@ -192,7 +192,7 @@ public class TestDocumentWriter extends LuceneTestCase { IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer)); Document doc = new Document(); - doc.add(newField("f1", "a 5 a a", TextField.TYPE_STORED)); + doc.add(newTextField("f1", "a 5 a a", Field.Store.YES)); writer.addDocument(doc); writer.commit(); @@ -226,7 +226,7 @@ public class TestDocumentWriter extends LuceneTestCase { private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); @Override - public boolean incrementToken() throws IOException { + public boolean incrementToken() { if (index == tokens.length) { return false; } else { @@ -235,8 +235,7 @@ public class TestDocumentWriter extends LuceneTestCase { return true; } } - - })); + })); writer.addDocument(doc); writer.commit(); @@ -269,7 +268,7 @@ public class TestDocumentWriter extends LuceneTestCase { public void testMixedTermVectorSettingsSameField() throws Exception { Document doc = new Document(); // f1 first without tv then with tv - doc.add(newField("f1", "v1", StringField.TYPE_STORED)); + doc.add(newStringField("f1", "v1", Field.Store.YES)); FieldType customType2 = new FieldType(StringField.TYPE_STORED); customType2.setStoreTermVectors(true); customType2.setStoreTermVectorOffsets(true); @@ -277,7 +276,7 @@ public class TestDocumentWriter extends LuceneTestCase { doc.add(newField("f1", "v2", customType2)); // f2 first with tv then without tv doc.add(newField("f2", "v1", customType2)); - doc.add(newField("f2", "v2", StringField.TYPE_STORED)); + doc.add(newStringField("f2", "v2", Field.Store.YES)); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random()))); @@ -286,7 +285,7 @@ public class TestDocumentWriter extends LuceneTestCase { _TestUtil.checkIndex(dir); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); // f1 Terms tfv1 = reader.getTermVectors(0).terms("f1"); assertNotNull(tfv1); @@ -306,14 +305,14 @@ public class TestDocumentWriter extends LuceneTestCase { public void testLUCENE_1590() throws Exception { Document doc = new Document(); // f1 has no norms - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setOmitNorms(true); FieldType customType2 = new FieldType(); customType2.setStored(true); doc.add(newField("f1", "v1", customType)); doc.add(newField("f1", "v2", customType2)); // f2 has no TF - FieldType customType3 = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType3 = new FieldType(TextField.TYPE_NOT_STORED); customType3.setIndexOptions(IndexOptions.DOCS_ONLY); Field f = newField("f2", "v1", customType3); doc.add(f); @@ -327,7 +326,7 @@ public class TestDocumentWriter extends LuceneTestCase { _TestUtil.checkIndex(dir); - SegmentReader reader = getOnlySegmentReader(IndexReader.open(dir)); + SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(dir)); FieldInfos fi = reader.getFieldInfos(); // f1 assertFalse("f1 should have no norms", fi.fieldInfo("f1").hasNorms()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterDeleteQueue.java b/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterDeleteQueue.java index a2903cf..2985b0f 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterDeleteQueue.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterDeleteQueue.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterStallControl.java b/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterStallControl.java index 3a9fe2b..e24ba4b 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterStallControl.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterStallControl.java @@ -24,7 +24,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.lucene.index.DocumentsWriterStallControl.MemoryController; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.ThreadInterruptedException; @@ -38,11 +37,8 @@ public class TestDocumentsWriterStallControl extends LuceneTestCase { public void testSimpleStall() throws InterruptedException { DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl(); - SimpleMemCtrl memCtrl = new SimpleMemCtrl(); - memCtrl.limit = 1000; - memCtrl.netBytes = 1000; - memCtrl.flushBytes = 20; - ctrl.updateStalled(memCtrl); + + ctrl.updateStalled(false); Thread[] waitThreads = waitThreads(atLeast(1), ctrl); start(waitThreads); assertFalse(ctrl.hasBlocked()); @@ -50,43 +46,30 @@ public class TestDocumentsWriterStallControl extends LuceneTestCase { join(waitThreads, 10); // now stall threads and wake them up again - memCtrl.netBytes = 1001; - memCtrl.flushBytes = 100; - ctrl.updateStalled(memCtrl); + ctrl.updateStalled(true); waitThreads = waitThreads(atLeast(1), ctrl); start(waitThreads); awaitState(100, Thread.State.WAITING, waitThreads); assertTrue(ctrl.hasBlocked()); assertTrue(ctrl.anyStalledThreads()); - memCtrl.netBytes = 50; - memCtrl.flushBytes = 0; - ctrl.updateStalled(memCtrl); + ctrl.updateStalled(false); assertFalse(ctrl.anyStalledThreads()); join(waitThreads, 500); } public void testRandom() throws InterruptedException { final DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl(); - SimpleMemCtrl memCtrl = new SimpleMemCtrl(); - memCtrl.limit = 1000; - memCtrl.netBytes = 1; - ctrl.updateStalled(memCtrl); + ctrl.updateStalled(false); + Thread[] stallThreads = new Thread[atLeast(3)]; for (int i = 0; i < stallThreads.length; i++) { - final int threadId = i; + final int stallProbability = 1 +random().nextInt(10); stallThreads[i] = new Thread() { public void run() { - int baseBytes = threadId % 2 == 0 ? 500 : 700; - SimpleMemCtrl memCtrl = new SimpleMemCtrl(); - memCtrl.limit = 1000; - memCtrl.netBytes = 1; - memCtrl.flushBytes = 0; int iters = atLeast(1000); for (int j = 0; j < iters; j++) { - memCtrl.netBytes = baseBytes + random().nextInt(1000); - memCtrl.flushBytes = random().nextInt((int)memCtrl.netBytes); - ctrl.updateStalled(memCtrl); + ctrl.updateStalled(random().nextInt(stallProbability) == 0); if (random().nextInt(5) == 0) { // thread 0 only updates ctrl.waitIfStalled(); } @@ -102,7 +85,7 @@ public class TestDocumentsWriterStallControl extends LuceneTestCase { */ while ((System.currentTimeMillis() - time) < 100 * 1000 && !terminated(stallThreads)) { - ctrl.updateStalled(memCtrl); + ctrl.updateStalled(false); if (random().nextBoolean()) { Thread.yield(); } else { @@ -116,42 +99,36 @@ public class TestDocumentsWriterStallControl extends LuceneTestCase { public void testAccquireReleaseRace() throws InterruptedException { final DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl(); - SimpleMemCtrl memCtrl = new SimpleMemCtrl(); - memCtrl.limit = 1000; - memCtrl.netBytes = 1; - memCtrl.flushBytes = 0; - ctrl.updateStalled(memCtrl); + ctrl.updateStalled(false); final AtomicBoolean stop = new AtomicBoolean(false); final AtomicBoolean checkPoint = new AtomicBoolean(true); int numStallers = atLeast(1); int numReleasers = atLeast(1); int numWaiters = atLeast(1); - - final CountDownLatch[] latches = new CountDownLatch[] { - new CountDownLatch(numStallers + numReleasers), new CountDownLatch(1), - new CountDownLatch(numWaiters)}; + final Synchronizer sync = new Synchronizer(numStallers + numReleasers, numStallers + numReleasers+numWaiters); Thread[] threads = new Thread[numReleasers + numStallers + numWaiters]; List exceptions = Collections.synchronizedList(new ArrayList()); for (int i = 0; i < numReleasers; i++) { - threads[i] = new Updater(stop, checkPoint, ctrl, latches, true, exceptions); + threads[i] = new Updater(stop, checkPoint, ctrl, sync, true, exceptions); } for (int i = numReleasers; i < numReleasers + numStallers; i++) { - threads[i] = new Updater(stop, checkPoint, ctrl, latches, false, exceptions); + threads[i] = new Updater(stop, checkPoint, ctrl, sync, false, exceptions); } for (int i = numReleasers + numStallers; i < numReleasers + numStallers + numWaiters; i++) { - threads[i] = new Waiter(stop, checkPoint, ctrl, latches, exceptions); + threads[i] = new Waiter(stop, checkPoint, ctrl, sync, exceptions); } start(threads); - int iters = atLeast(20000); + int iters = atLeast(10000); + final float checkPointProbability = TEST_NIGHTLY ? 0.5f : 0.1f; for (int i = 0; i < iters; i++) { if (checkPoint.get()) { - assertTrue("timed out waiting for update threads - deadlock?", latches[0].await(10, TimeUnit.SECONDS)); + assertTrue("timed out waiting for update threads - deadlock?", sync.updateJoin.await(10, TimeUnit.SECONDS)); if (!exceptions.isEmpty()) { for (Throwable throwable : exceptions) { throwable.printStackTrace(); @@ -159,33 +136,41 @@ public class TestDocumentsWriterStallControl extends LuceneTestCase { fail("got exceptions in threads"); } - if (!ctrl.anyStalledThreads()) { - assertTrue( - "control claims no stalled threads but waiter seems to be blocked", - latches[2].await(10, TimeUnit.SECONDS)); - } - checkPoint.set(false); + if (ctrl.hasBlocked() && ctrl.isHealthy()) { + assertState(numReleasers, numStallers, numWaiters, threads, ctrl); + + + } - latches[1].countDown(); + checkPoint.set(false); + sync.waiter.countDown(); + sync.leftCheckpoint.await(); } assertFalse(checkPoint.get()); - if (random().nextInt(2) == 0) { - latches[0] = new CountDownLatch(numStallers + numReleasers); - latches[1] = new CountDownLatch(1); - latches[2] = new CountDownLatch(numWaiters); + assertEquals(0, sync.waiter.getCount()); + if (checkPointProbability >= random().nextFloat()) { + sync.reset(numStallers + numReleasers, numStallers + numReleasers + + numWaiters); checkPoint.set(true); } } + if (!checkPoint.get()) { + sync.reset(numStallers + numReleasers, numStallers + numReleasers + + numWaiters); + checkPoint.set(true); + } + assertTrue(sync.updateJoin.await(10, TimeUnit.SECONDS)); + assertState(numReleasers, numStallers, numWaiters, threads, ctrl); + checkPoint.set(false); stop.set(true); - latches[1].countDown(); + sync.waiter.countDown(); + sync.leftCheckpoint.await(); + for (int i = 0; i < threads.length; i++) { - memCtrl.limit = 1000; - memCtrl.netBytes = 1; - memCtrl.flushBytes = 0; - ctrl.updateStalled(memCtrl); + ctrl.updateStalled(false); threads[i].join(2000); if (threads[i].isAlive() && threads[i] instanceof Waiter) { if (threads[i].getState() == Thread.State.WAITING) { @@ -196,20 +181,45 @@ public class TestDocumentsWriterStallControl extends LuceneTestCase { } } + private void assertState(int numReleasers, int numStallers, int numWaiters, Thread[] threads, DocumentsWriterStallControl ctrl) throws InterruptedException { + int millisToSleep = 100; + while (true) { + if (ctrl.hasBlocked() && ctrl.isHealthy()) { + for (int n = numReleasers + numStallers; n < numReleasers + + numStallers + numWaiters; n++) { + if (ctrl.isThreadQueued(threads[n])) { + if (millisToSleep < 60000) { + Thread.sleep(millisToSleep); + millisToSleep *=2; + break; + } else { + fail("control claims no stalled threads but waiter seems to be blocked "); + } + } + } + break; + } else { + break; + } + } + + } + public static class Waiter extends Thread { - private CountDownLatch[] latches; + private Synchronizer sync; private DocumentsWriterStallControl ctrl; private AtomicBoolean checkPoint; private AtomicBoolean stop; private List exceptions; public Waiter(AtomicBoolean stop, AtomicBoolean checkPoint, - DocumentsWriterStallControl ctrl, CountDownLatch[] latches, + DocumentsWriterStallControl ctrl, Synchronizer sync, List exceptions) { + super("waiter"); this.stop = stop; this.checkPoint = checkPoint; this.ctrl = ctrl; - this.latches = latches; + this.sync = sync; this.exceptions = exceptions; } @@ -218,13 +228,10 @@ public class TestDocumentsWriterStallControl extends LuceneTestCase { while (!stop.get()) { ctrl.waitIfStalled(); if (checkPoint.get()) { - CountDownLatch join = latches[2]; - CountDownLatch wait = latches[1]; - join.countDown(); try { - assertTrue(wait.await(10, TimeUnit.SECONDS)); + assertTrue(sync.await()); } catch (InterruptedException e) { - System.out.println("[Waiter] got interrupted - wait count: " + wait.getCount()); + System.out.println("[Waiter] got interrupted - wait count: " + sync.waiter.getCount()); throw new ThreadInterruptedException(e); } } @@ -238,7 +245,7 @@ public class TestDocumentsWriterStallControl extends LuceneTestCase { public static class Updater extends Thread { - private CountDownLatch[] latches; + private Synchronizer sync; private DocumentsWriterStallControl ctrl; private AtomicBoolean checkPoint; private AtomicBoolean stop; @@ -246,44 +253,44 @@ public class TestDocumentsWriterStallControl extends LuceneTestCase { private List exceptions; public Updater(AtomicBoolean stop, AtomicBoolean checkPoint, - DocumentsWriterStallControl ctrl, CountDownLatch[] latches, + DocumentsWriterStallControl ctrl, Synchronizer sync, boolean release, List exceptions) { + super("updater"); this.stop = stop; this.checkPoint = checkPoint; this.ctrl = ctrl; - this.latches = latches; + this.sync = sync; this.release = release; this.exceptions = exceptions; } public void run() { try { - SimpleMemCtrl memCtrl = new SimpleMemCtrl(); - memCtrl.limit = 1000; - memCtrl.netBytes = release ? 1 : 2000; - memCtrl.flushBytes = random().nextInt((int)memCtrl.netBytes); + while (!stop.get()) { int internalIters = release && random().nextBoolean() ? atLeast(5) : 1; for (int i = 0; i < internalIters; i++) { - ctrl.updateStalled(memCtrl); + ctrl.updateStalled(random().nextBoolean()); } if (checkPoint.get()) { - CountDownLatch join = latches[0]; - CountDownLatch wait = latches[1]; - join.countDown(); + sync.updateJoin.countDown(); try { - assertTrue(wait.await(10, TimeUnit.SECONDS)); + assertTrue(sync.await()); } catch (InterruptedException e) { - System.out.println("[Updater] got interrupted - wait count: " + wait.getCount()); + System.out.println("[Updater] got interrupted - wait count: " + sync.waiter.getCount()); throw new ThreadInterruptedException(e); } + sync.leftCheckpoint.countDown(); + } + if (random().nextBoolean()) { + Thread.yield(); } - Thread.yield(); } } catch (Throwable e) { e.printStackTrace(); exceptions.add(e); } + sync.updateJoin.countDown(); } } @@ -345,24 +352,23 @@ public class TestDocumentsWriterStallControl extends LuceneTestCase { + " ms"); } - private static class SimpleMemCtrl implements MemoryController { - long netBytes; - long limit; - long flushBytes; + private static final class Synchronizer { + volatile CountDownLatch waiter; + volatile CountDownLatch updateJoin; + volatile CountDownLatch leftCheckpoint; - @Override - public long netBytes() { - return netBytes; + public Synchronizer(int numUpdater, int numThreads) { + reset(numUpdater, numThreads); } - @Override - public long stallLimitBytes() { - return limit; + public void reset(int numUpdaters, int numThreads) { + this.waiter = new CountDownLatch(1); + this.updateJoin = new CountDownLatch(numUpdaters); + this.leftCheckpoint = new CountDownLatch(numUpdaters); } - - @Override - public long flushBytes() { - return flushBytes; + + public boolean await() throws InterruptedException { + return waiter.await(10, TimeUnit.SECONDS); } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java b/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java index e89358d..80856d8 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java index 6ec77c6..9850a3d 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java index c035c81..5bbc43a 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -78,7 +78,7 @@ public class TestFieldsReader extends LuceneTestCase { public void test() throws IOException { assertTrue(dir != null); assertTrue(fieldInfos != null); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); Document doc = reader.document(0); assertTrue(doc != null); assertTrue(doc.getField(DocHelper.TEXT_FIELD_1_KEY) != null); @@ -115,7 +115,7 @@ public class TestFieldsReader extends LuceneTestCase { Directory fsDir; - public FaultyFSDirectory(File dir) throws IOException { + public FaultyFSDirectory(File dir) { fsDir = newFSDirectory(dir); lockFactory = fsDir.getLockFactory(); } @@ -203,7 +203,7 @@ public class TestFieldsReader extends LuceneTestCase { writer.forceMerge(1); writer.close(); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); FaultyIndexInput.doFail = true; @@ -249,13 +249,13 @@ public class TestFieldsReader extends LuceneTestCase { if (random().nextBoolean()) { final float f = random().nextFloat(); answer = Float.valueOf(f); - nf = new FloatField("nf", f); + nf = new FloatField("nf", f, Field.Store.NO); sf = new StoredField("nf", f); typeAnswer = NumericType.FLOAT; } else { final double d = random().nextDouble(); answer = Double.valueOf(d); - nf = new DoubleField("nf", d); + nf = new DoubleField("nf", d, Field.Store.NO); sf = new StoredField("nf", d); typeAnswer = NumericType.DOUBLE; } @@ -264,13 +264,13 @@ public class TestFieldsReader extends LuceneTestCase { if (random().nextBoolean()) { final int i = random().nextInt(); answer = Integer.valueOf(i); - nf = new IntField("nf", i); + nf = new IntField("nf", i, Field.Store.NO); sf = new StoredField("nf", i); typeAnswer = NumericType.INT; } else { final long l = random().nextLong(); answer = Long.valueOf(l); - nf = new LongField("nf", l); + nf = new LongField("nf", l, Field.Store.NO); sf = new StoredField("nf", l); typeAnswer = NumericType.LONG; } @@ -279,7 +279,7 @@ public class TestFieldsReader extends LuceneTestCase { doc.add(sf); answers[id] = answer; typeAnswers[id] = typeAnswer; - FieldType ft = new FieldType(IntField.TYPE); + FieldType ft = new FieldType(IntField.TYPE_NOT_STORED); ft.setNumericPrecisionStep(Integer.MAX_VALUE); doc.add(new IntField("id", id, ft)); w.addDocument(doc); @@ -309,7 +309,7 @@ public class TestFieldsReader extends LuceneTestCase { FieldType onlyStored = new FieldType(); onlyStored.setStored(true); doc.add(new Field("field", "value", onlyStored)); - doc.add(new Field("field2", "value", StringField.TYPE_STORED)); + doc.add(new StringField("field2", "value", Field.Store.YES)); w.addDocument(doc); IndexReader r = w.getReader(); w.close(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java b/lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java index 55a3880..376a7a4 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,7 +23,7 @@ import java.lang.reflect.Method; import java.lang.reflect.Modifier; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.Directory; import org.apache.lucene.store.MockDirectoryWrapper; @@ -132,15 +132,15 @@ public class TestFilterAtomicReader extends LuceneTestCase { IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); Document d1 = new Document(); - d1.add(newField("default","one two", TextField.TYPE_STORED)); + d1.add(newTextField("default", "one two", Field.Store.YES)); writer.addDocument(d1); Document d2 = new Document(); - d2.add(newField("default","one three", TextField.TYPE_STORED)); + d2.add(newTextField("default", "one three", Field.Store.YES)); writer.addDocument(d2); Document d3 = new Document(); - d3.add(newField("default","two four", TextField.TYPE_STORED)); + d3.add(newTextField("default", "two four", Field.Store.YES)); writer.addDocument(d3); writer.close(); @@ -151,11 +151,11 @@ public class TestFilterAtomicReader extends LuceneTestCase { ((MockDirectoryWrapper) target).setCrossCheckTermVectorsOnClose(false); writer = new IndexWriter(target, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); - IndexReader reader = new TestReader(IndexReader.open(directory)); + IndexReader reader = new TestReader(DirectoryReader.open(directory)); writer.addIndexes(reader); writer.close(); reader.close(); - reader = IndexReader.open(target); + reader = DirectoryReader.open(target); TermsEnum terms = MultiFields.getTerms(reader, "default").iterator(null); while (terms.next() != null) { @@ -175,7 +175,7 @@ public class TestFilterAtomicReader extends LuceneTestCase { target.close(); } - private void checkOverrideMethods(Class clazz) throws Exception { + private void checkOverrideMethods(Class clazz) { boolean fail = false; for (Method m : clazz.getMethods()) { int mods = m.getModifiers(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFlex.java b/lucene/core/src/test/org/apache/lucene/index/TestFlex.java index 0c7be51..76a1ee5 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestFlex.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestFlex.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -33,17 +33,17 @@ public class TestFlex extends LuceneTestCase { IndexWriter w = new IndexWriter( d, - new IndexWriterConfig(Version.LUCENE_31, new MockAnalyzer(random())). - setMaxBufferedDocs(7) + new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())). + setMaxBufferedDocs(7).setMergePolicy(newLogMergePolicy()) ); for(int iter=0;iter<2;iter++) { if (iter == 0) { Document doc = new Document(); - doc.add(newField("field1", "this is field1", TextField.TYPE_UNSTORED)); - doc.add(newField("field2", "this is field2", TextField.TYPE_UNSTORED)); - doc.add(newField("field3", "aaa", TextField.TYPE_UNSTORED)); - doc.add(newField("field4", "bbb", TextField.TYPE_UNSTORED)); + doc.add(newTextField("field1", "this is field1", Field.Store.NO)); + doc.add(newTextField("field2", "this is field2", Field.Store.NO)); + doc.add(newTextField("field3", "aaa", Field.Store.NO)); + doc.add(newTextField("field4", "bbb", Field.Store.NO)); for(int i=0;i (2.d * iwc.getRAMBufferSizeMB() * 1024.d * 1024.d)) { - assertTrue(docsWriter.flushControl.stallControl.wasStalled); + assertTrue(docsWriter.flushControl.stallControl.wasStalled()); } assertActiveBytesAfter(flushControl); writer.close(true); @@ -291,7 +288,7 @@ public class TestFlushByRamOrCountsPolicy extends LuceneTestCase { public class IndexThread extends Thread { IndexWriter writer; - IndexWriterConfig iwc; + LiveIndexWriterConfig iwc; LineFileDocs docs; private AtomicInteger pendingDocs; private final boolean doRandomCommit; diff --git a/lucene/core/src/test/org/apache/lucene/index/TestForTooMuchCloning.java b/lucene/core/src/test/org/apache/lucene/index/TestForTooMuchCloning.java index 182e526..a640731 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestForTooMuchCloning.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestForTooMuchCloning.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,6 +19,7 @@ package org.apache.lucene.index; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermRangeQuery; @@ -49,7 +50,7 @@ public class TestForTooMuchCloning extends LuceneTestCase { sb.append(' '); } final Document doc = new Document(); - doc.add(new TextField("field", sb.toString())); + doc.add(new TextField("field", sb.toString(), Field.Store.NO)); w.addDocument(doc); } final IndexReader r = w.getReader(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestForceMergeForever.java b/lucene/core/src/test/org/apache/lucene/index/TestForceMergeForever.java index 80456ea..80ead16 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestForceMergeForever.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestForceMergeForever.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -40,7 +40,7 @@ public class TestForceMergeForever extends LuceneTestCase { } @Override - public void merge(MergePolicy.OneMerge merge) throws CorruptIndexException, IOException { + public void merge(MergePolicy.OneMerge merge) throws IOException { if (merge.maxNumSegments != -1 && (first || merge.segments.size() == 1)) { first = false; if (VERBOSE) { @@ -59,7 +59,7 @@ public class TestForceMergeForever extends LuceneTestCase { // Try to make an index that requires merging: w.getConfig().setMaxBufferedDocs(_TestUtil.nextInt(random(), 2, 11)); final int numStartDocs = atLeast(20); - final LineFileDocs docs = new LineFileDocs(random(), defaultCodecSupportsDocValues()); + final LineFileDocs docs = new LineFileDocs(random(), true); for(int docIDX=0;docIDX getFileNames() throws IOException { return null; } + @Override public Collection getFileNames() { return null; } @Override public void delete() {} @Override public long getGeneration() { return 0; } - @Override public Map getUserData() throws IOException { return null; } + @Override public Map getUserData() { return null; } @Override public boolean isDeleted() { return false; } @Override public int getSegmentCount() { return 2; } }; @@ -46,10 +45,10 @@ public class TestIndexCommit extends LuceneTestCase { IndexCommit ic2 = new IndexCommit() { @Override public String getSegmentsFileName() { return "b"; } @Override public Directory getDirectory() { return dir; } - @Override public Collection getFileNames() throws IOException { return null; } + @Override public Collection getFileNames() { return null; } @Override public void delete() {} @Override public long getGeneration() { return 0; } - @Override public Map getUserData() throws IOException { return null; } + @Override public Map getUserData() { return null; } @Override public boolean isDeleted() { return false; } @Override public int getSegmentCount() { return 2; } }; diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java index 0dacc77..c4fc1e6 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,15 +22,10 @@ import java.util.*; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.FieldInfosReader; import org.apache.lucene.document.Document; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.search.similarities.DefaultSimilarity; -import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.MockDirectoryWrapper; @@ -61,7 +56,7 @@ public class TestIndexFileDeleter extends LuceneTestCase { for(i=0;i<35;i++) { addDoc(writer, i); } - mergePolicy.setUseCompoundFile(false); + ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); for(;i<45;i++) { addDoc(writer, i); } @@ -196,8 +191,8 @@ public class TestIndexFileDeleter extends LuceneTestCase { private void addDoc(IndexWriter writer, int id) throws IOException { Document doc = new Document(); - doc.add(newField("content", "aaa", TextField.TYPE_UNSTORED)); - doc.add(newField("id", Integer.toString(id), StringField.TYPE_UNSTORED)); + doc.add(newTextField("content", "aaa", Field.Store.NO)); + doc.add(newStringField("id", Integer.toString(id), Field.Store.NO)); writer.addDocument(doc); } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexInput.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexInput.java index 6f02364..23a129b 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexInput.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexInput.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,6 +18,7 @@ package org.apache.lucene.index; */ import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.ByteArrayDataOutput; import org.apache.lucene.store.DataInput; @@ -99,9 +100,9 @@ public class TestIndexInput extends LuceneTestCase { final long l1; if (rarely()) { // a long with lots of zeroes at the end - l1 = LONGS[i] = ((long) Math.abs(random.nextInt())) << 32; + l1 = LONGS[i] = _TestUtil.nextLong(random, 0, Integer.MAX_VALUE) << 32; } else { - l1 = LONGS[i] = Math.abs(random.nextLong()); + l1 = LONGS[i] = _TestUtil.nextLong(random, 0, Long.MAX_VALUE); } bdo.writeVLong(l1); bdo.writeLong(l1); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java index e12236f..195c455 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,10 +23,8 @@ import java.io.Reader; import java.io.StringReader; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Set; import org.apache.lucene.analysis.*; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; @@ -37,21 +35,16 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StoredField; -import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.PhraseQuery; -import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockFactory; import org.apache.lucene.store.LockObtainFailedException; @@ -68,7 +61,7 @@ import org.apache.lucene.util.packed.PackedInts; public class TestIndexWriter extends LuceneTestCase { - private static final FieldType storedTextType = new FieldType(TextField.TYPE_UNSTORED); + private static final FieldType storedTextType = new FieldType(TextField.TYPE_NOT_STORED); public void testDocCount() throws IOException { Directory dir = newDirectory(); @@ -99,7 +92,7 @@ public class TestIndexWriter extends LuceneTestCase { } writer.close(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); assertEquals(60, reader.numDocs()); reader.close(); @@ -112,7 +105,7 @@ public class TestIndexWriter extends LuceneTestCase { writer.close(); // check that the index reader gives the same numbers. - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); assertEquals(60, reader.maxDoc()); assertEquals(60, reader.numDocs()); reader.close(); @@ -129,7 +122,7 @@ public class TestIndexWriter extends LuceneTestCase { static void addDoc(IndexWriter writer) throws IOException { Document doc = new Document(); - doc.add(newField("content", "aaa", TextField.TYPE_UNSTORED)); + doc.add(newTextField("content", "aaa", Field.Store.NO)); writer.addDocument(doc); } @@ -179,7 +172,7 @@ public class TestIndexWriter extends LuceneTestCase { writer.close(); // now open reader: - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); assertEquals("should be one document", reader.numDocs(), 1); // now open index for create: @@ -189,7 +182,7 @@ public class TestIndexWriter extends LuceneTestCase { writer.close(); assertEquals("should be one document", reader.numDocs(), 1); - IndexReader reader2 = IndexReader.open(dir); + IndexReader reader2 = DirectoryReader.open(dir); assertEquals("should be one document", reader2.numDocs(), 1); reader.close(); reader2.close(); @@ -224,7 +217,7 @@ public class TestIndexWriter extends LuceneTestCase { writer.commit(); writer.close(); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); assertEquals(0, reader.maxDoc()); assertEquals(0, reader.numDocs()); reader.close(); @@ -233,7 +226,7 @@ public class TestIndexWriter extends LuceneTestCase { writer.commit(); writer.close(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); assertEquals(0, reader.maxDoc()); assertEquals(0, reader.numDocs()); reader.close(); @@ -255,7 +248,7 @@ public class TestIndexWriter extends LuceneTestCase { } writer.close(); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); assertEquals(100, reader.maxDoc()); assertEquals(100, reader.numDocs()); for(int j=0;j<100;j++) { @@ -450,7 +443,7 @@ public class TestIndexWriter extends LuceneTestCase { } writer.close(); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); int totalHits = searcher.search(new TermQuery(new Term("field", "aaa")), null, 1).totalHits; assertEquals(n*100, totalHits); @@ -481,7 +474,7 @@ public class TestIndexWriter extends LuceneTestCase { Term searchTerm = new Term("field", "aaa"); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals(10, hits.length); @@ -503,13 +496,13 @@ public class TestIndexWriter extends LuceneTestCase { writer.addDocument(doc); } writer.close(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); searcher = new IndexSearcher(reader); hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals(27, hits.length); reader.close(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); reader.close(); dir.close(); @@ -536,7 +529,7 @@ public class TestIndexWriter extends LuceneTestCase { writer.addDocument(doc); writer.close(); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); assertEquals(1, reader.maxDoc()); assertEquals(1, reader.numDocs()); Term t = new Term("field", "a"); @@ -581,7 +574,7 @@ public class TestIndexWriter extends LuceneTestCase { } writer.close(); Term searchTerm = new Term("content", "aaa"); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals("did not get right number of hits", 100, hits.length); @@ -637,7 +630,7 @@ public class TestIndexWriter extends LuceneTestCase { } writer.addDocument(new Document()); writer.close(); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); assertEquals(2, reader.numDocs()); reader.close(); dir.close(); @@ -658,7 +651,7 @@ public class TestIndexWriter extends LuceneTestCase { TEST_VERSION_CURRENT, new MockAnalyzer(random()))); Document document = new Document(); - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setStoreTermVectors(true); document.add(newField("tvtest", "", customType)); iw.addDocument(document); @@ -677,7 +670,7 @@ public class TestIndexWriter extends LuceneTestCase { ((LogMergePolicy) conf.getMergePolicy()).setMergeFactor(2); IndexWriter iw = new IndexWriter(dir, conf); Document document = new Document(); - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setStoreTermVectors(true); document.add(newField("tvtest", "a b c", customType)); Thread.currentThread().setPriority(Thread.MAX_PRIORITY); @@ -707,14 +700,14 @@ public class TestIndexWriter extends LuceneTestCase { FieldType type = null; if (i == 7) { // Add empty docs here - doc.add(newField("content3", "", TextField.TYPE_UNSTORED)); + doc.add(newTextField("content3", "", Field.Store.NO)); } else { if (i%2 == 0) { doc.add(newField("content4", contents, customType)); type = customType; } else - type = TextField.TYPE_UNSTORED; - doc.add(newField("content1", contents, TextField.TYPE_UNSTORED)); + type = TextField.TYPE_NOT_STORED; + doc.add(newTextField("content1", contents, Field.Store.NO)); doc.add(newField("content3", "", customType)); doc.add(newField("content5", "", type)); } @@ -746,11 +739,11 @@ public class TestIndexWriter extends LuceneTestCase { for(int i=0;i<10000;i++) b.append(" a"); b.append(" x"); - doc.add(newField("field", b.toString(), TextField.TYPE_UNSTORED)); + doc.add(newTextField("field", b.toString(), Field.Store.NO)); writer.addDocument(doc); writer.close(); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); Term t = new Term("field", "x"); assertEquals(1, reader.docFreq(t)); reader.close(); @@ -764,7 +757,7 @@ public class TestIndexWriter extends LuceneTestCase { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random()))); Document doc = new Document(); - doc.add(newField("", "a b c", TextField.TYPE_UNSTORED)); + doc.add(newTextField("", "a b c", Field.Store.NO)); writer.addDocument(doc); writer.close(); dir.close(); @@ -774,10 +767,10 @@ public class TestIndexWriter extends LuceneTestCase { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random()))); Document doc = new Document(); - doc.add(newField("", "a b c", TextField.TYPE_UNSTORED)); + doc.add(newTextField("", "a b c", Field.Store.NO)); writer.addDocument(doc); writer.close(); - DirectoryReader reader = IndexReader.open(dir); + DirectoryReader reader = DirectoryReader.open(dir); AtomicReader subreader = getOnlySegmentReader(reader); TermsEnum te = subreader.fields().terms("").iterator(null); assertEquals(new BytesRef("a"), te.next()); @@ -792,13 +785,13 @@ public class TestIndexWriter extends LuceneTestCase { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random()))); Document doc = new Document(); - doc.add(newField("", "", StringField.TYPE_UNSTORED)); - doc.add(newField("", "a", StringField.TYPE_UNSTORED)); - doc.add(newField("", "b", StringField.TYPE_UNSTORED)); - doc.add(newField("", "c", StringField.TYPE_UNSTORED)); + doc.add(newStringField("", "", Field.Store.NO)); + doc.add(newStringField("", "a", Field.Store.NO)); + doc.add(newStringField("", "b", Field.Store.NO)); + doc.add(newStringField("", "c", Field.Store.NO)); writer.addDocument(doc); writer.close(); - DirectoryReader reader = IndexReader.open(dir); + DirectoryReader reader = DirectoryReader.open(dir); AtomicReader subreader = getOnlySegmentReader(reader); TermsEnum te = subreader.fields().terms("").iterator(null); assertEquals(new BytesRef(""), te.next()); @@ -827,7 +820,7 @@ public class TestIndexWriter extends LuceneTestCase { } @Override - protected void doBeforeFlush() throws IOException { + protected void doBeforeFlush() { beforeWasCalled = true; } } @@ -852,7 +845,7 @@ public class TestIndexWriter extends LuceneTestCase { assertTrue(w.afterWasCalled); w.close(); - IndexReader ir = IndexReader.open(dir); + IndexReader ir = DirectoryReader.open(dir); assertEquals(0, ir.numDocs()); ir.close(); @@ -883,39 +876,16 @@ public class TestIndexWriter extends LuceneTestCase { IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random()))); Document doc = new Document(); doc.add(new TextField("field", tokens)); - w.addDocument(doc); - w.commit(); - - IndexReader r = IndexReader.open(dir); - IndexSearcher s = new IndexSearcher(r); - PhraseQuery pq = new PhraseQuery(); - pq.add(new Term("field", "a")); - pq.add(new Term("field", "b")); - pq.add(new Term("field", "c")); - ScoreDoc[] hits = s.search(pq, null, 1000).scoreDocs; - assertEquals(1, hits.length); - - Query q = new SpanTermQuery(new Term("field", "a")); - hits = s.search(q, null, 1000).scoreDocs; - assertEquals(1, hits.length); - - DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(s.getIndexReader(), - MultiFields.getLiveDocs(s.getIndexReader()), - "field", - new BytesRef("a"), - false); - - assertTrue(tps.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); - assertEquals(1, tps.freq()); - assertEquals(0, tps.nextPosition()); + try { + w.addDocument(doc); + fail("did not hit expected exception"); + } catch (IllegalArgumentException iea) { + // expected + } w.close(); - - r.close(); dir.close(); } - - // LUCENE-1219 public void testBinaryFieldOffsetLength() throws IOException { Directory dir = newDirectory(); @@ -935,7 +905,7 @@ public class TestIndexWriter extends LuceneTestCase { w.addDocument(doc); w.close(); - IndexReader ir = IndexReader.open(dir); + IndexReader ir = DirectoryReader.open(dir); Document doc2 = ir.document(0); IndexableField f2 = doc2.getField("binary"); b = f2.binaryValue().bytes; @@ -954,7 +924,7 @@ public class TestIndexWriter extends LuceneTestCase { IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)); Document doc = new Document(); - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setStoreTermVectors(true); customType.setStoreTermVectorPositions(true); Field f = newField("field", "", customType); @@ -964,7 +934,7 @@ public class TestIndexWriter extends LuceneTestCase { w.addDocument(doc); w.close(); - IndexReader r = IndexReader.open(dir); + IndexReader r = DirectoryReader.open(dir); Terms tpv = r.getTermVectors(0).terms("field"); TermsEnum termsEnum = tpv.iterator(null); assertNotNull(termsEnum.next()); @@ -1008,11 +978,11 @@ public class TestIndexWriter extends LuceneTestCase { writer2.addDocument(doc); writer2.close(); - IndexReader r1 = IndexReader.open(dir2); + IndexReader r1 = DirectoryReader.open(dir2); writer.addIndexes(r1, r1); writer.close(); - IndexReader r3 = IndexReader.open(dir); + IndexReader r3 = DirectoryReader.open(dir); assertEquals(5, r3.numDocs()); r3.close(); @@ -1056,7 +1026,7 @@ public class TestIndexWriter extends LuceneTestCase { w.close(); w = null; _TestUtil.checkIndex(dir); - IndexReader.open(dir).close(); + DirectoryReader.open(dir).close(); // Strangely, if we interrupt a thread before // all classes are loaded, the class loader @@ -1106,12 +1076,12 @@ public class TestIndexWriter extends LuceneTestCase { e.printStackTrace(System.out); } try { - IndexReader r = IndexReader.open(dir); + IndexReader r = DirectoryReader.open(dir); //System.out.println("doc count=" + r.numDocs()); r.close(); } catch (Exception e) { failed = true; - System.out.println("IndexReader.open FAILED: unexpected exception"); + System.out.println("DirectoryReader.open FAILED: unexpected exception"); e.printStackTrace(System.out); } } @@ -1192,7 +1162,7 @@ public class TestIndexWriter extends LuceneTestCase { w.forceMerge(1); // force segment merge. w.close(); - IndexReader ir = IndexReader.open(dir); + IndexReader ir = DirectoryReader.open(dir); Document doc2 = ir.document(0); IndexableField f3 = doc2.getField("binary"); b = f3.binaryValue().bytes; @@ -1279,7 +1249,7 @@ public class TestIndexWriter extends LuceneTestCase { s.append(' ').append(i); } Document d = new Document(); - Field f = newField("field", s.toString(), TextField.TYPE_UNSTORED); + Field f = newTextField("field", s.toString(), Field.Store.NO); d.add(f); w.addDocument(d); @@ -1311,7 +1281,7 @@ public class TestIndexWriter extends LuceneTestCase { setMergePolicy(mergePolicy) ); Document doc = new Document(); - doc.add(newField("field", "go", TextField.TYPE_UNSTORED)); + doc.add(newTextField("field", "go", Field.Store.NO)); w.addDocument(doc); DirectoryReader r; if (iter == 0) { @@ -1320,7 +1290,7 @@ public class TestIndexWriter extends LuceneTestCase { } else { // don't use NRT w.commit(); - r = IndexReader.open(dir); + r = DirectoryReader.open(dir); } List files = Arrays.asList(dir.listAll()); @@ -1525,7 +1495,7 @@ public class TestIndexWriter extends LuceneTestCase { _TestUtil.checkIndex(dir); assertNoUnreferencedFiles(dir, "no tv files"); - DirectoryReader r0 = IndexReader.open(dir); + DirectoryReader r0 = DirectoryReader.open(dir); for (IndexReader r : r0.getSequentialSubReaders()) { SegmentInfoPerCommit s = ((SegmentReader) r).getSegmentInfo(); assertFalse(((SegmentReader) r).getFieldInfos().hasVectors()); @@ -1557,7 +1527,7 @@ public class TestIndexWriter extends LuceneTestCase { } @Override - public final boolean incrementToken() throws IOException { + public final boolean incrementToken() { clearAttributes(); if (upto < tokens.length) { termAtt.setEmpty(); @@ -1597,12 +1567,12 @@ public class TestIndexWriter extends LuceneTestCase { // This contents produces a too-long term: String contents = "abc xyz x" + bigTerm + " another term"; - doc.add(new TextField("content", contents)); + doc.add(new TextField("content", contents, Field.Store.NO)); w.addDocument(doc); // Make sure we can add another normal document doc = new Document(); - doc.add(new TextField("content", "abc bbb ccc")); + doc.add(new TextField("content", "abc bbb ccc", Field.Store.NO)); w.addDocument(doc); IndexReader reader = w.getReader(); @@ -1632,7 +1602,7 @@ public class TestIndexWriter extends LuceneTestCase { // Make sure we can add a document with exactly the // maximum length term, and search on that term: doc = new Document(); - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setTokenized(false); Field contentField = new Field("content", "", customType); doc.add(contentField); @@ -1671,7 +1641,7 @@ public class TestIndexWriter extends LuceneTestCase { iwc.setReaderTermsIndexDivisor(1); IndexWriter writer = new IndexWriter(dir, iwc); Document doc = new Document(); - doc.add(newField("", "a b c", TextField.TYPE_UNSTORED)); + doc.add(newTextField("", "a b c", Field.Store.NO)); writer.addDocument(doc); writer.close(); dir.close(); @@ -1687,7 +1657,7 @@ public class TestIndexWriter extends LuceneTestCase { w.addDocument(doc); } w.commit(); - IndexReader.open(w, true).close(); + DirectoryReader.open(w, true).close(); w.deleteAll(); w.commit(); @@ -1705,7 +1675,7 @@ public class TestIndexWriter extends LuceneTestCase { Directory d = new MockDirectoryWrapper(random(), new RAMDirectory()); IndexWriter w = new IndexWriter(d, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); Document doc = new Document(); - doc.add(newField("id", "0", StringField.TYPE_STORED)); + doc.add(newStringField("id", "0", Field.Store.YES)); w.addDocument(doc); DirectoryReader r = w.getReader(); long version = r.getVersion(); @@ -1749,10 +1719,10 @@ public class TestIndexWriter extends LuceneTestCase { IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); - FieldType docsAndFreqs = new FieldType(TextField.TYPE_UNSTORED); + FieldType docsAndFreqs = new FieldType(TextField.TYPE_NOT_STORED); docsAndFreqs.setIndexOptions(IndexOptions.DOCS_AND_FREQS); - FieldType docsOnly = new FieldType(TextField.TYPE_UNSTORED); + FieldType docsOnly = new FieldType(TextField.TYPE_NOT_STORED); docsOnly.setIndexOptions(IndexOptions.DOCS_ONLY); Document doc = new Document(); @@ -1795,7 +1765,7 @@ public class TestIndexWriter extends LuceneTestCase { } w.commit(); w.close(); - IndexReader r = IndexReader.open(dir); + IndexReader r = DirectoryReader.open(dir); assertEquals(0, r.maxDoc()); r.close(); dir.close(); @@ -1824,7 +1794,7 @@ public class TestIndexWriter extends LuceneTestCase { w.prepareCommit(); w.rollback(); assertTrue(DirectoryReader.indexExists(dir)); - IndexReader r = IndexReader.open(dir); + IndexReader r = DirectoryReader.open(dir); assertEquals(0, r.maxDoc()); r.close(); dir.close(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java index dedfa9d..376f747 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -26,7 +26,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.lucene.analysis.*; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; @@ -51,20 +50,20 @@ public class TestIndexWriterCommit extends LuceneTestCase { writer.close(); Term searchTerm = new Term("content", "aaa"); - DirectoryReader reader = IndexReader.open(dir); + DirectoryReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals("first number of hits", 14, hits.length); reader.close(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random()))); for(int i=0;i<3;i++) { for(int j=0;j<11;j++) { TestIndexWriter.addDoc(writer); } - IndexReader r = IndexReader.open(dir); + IndexReader r = DirectoryReader.open(dir); searcher = new IndexSearcher(r); hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals("reader incorrectly sees changes from writer", 14, hits.length); @@ -76,7 +75,7 @@ public class TestIndexWriterCommit extends LuceneTestCase { writer.close(); assertFalse("reader should not be current now", reader.isCurrent()); - IndexReader r = IndexReader.open(dir); + IndexReader r = DirectoryReader.open(dir); searcher = new IndexSearcher(r); hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals("reader did not see changes after writer was closed", 47, hits.length); @@ -102,7 +101,7 @@ public class TestIndexWriterCommit extends LuceneTestCase { writer.close(); Term searchTerm = new Term("content", "aaa"); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals("first number of hits", 14, hits.length); @@ -116,7 +115,7 @@ public class TestIndexWriterCommit extends LuceneTestCase { // Delete all docs: writer.deleteDocuments(searchTerm); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); searcher = new IndexSearcher(reader); hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals("reader incorrectly sees changes from writer", 14, hits.length); @@ -127,7 +126,7 @@ public class TestIndexWriterCommit extends LuceneTestCase { TestIndexWriter.assertNoUnreferencedFiles(dir, "unreferenced files remain after rollback()"); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); searcher = new IndexSearcher(reader); hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals("saw changes after writer.abort", 14, hits.length); @@ -146,7 +145,7 @@ public class TestIndexWriterCommit extends LuceneTestCase { for(int j=0;j<17;j++) { TestIndexWriter.addDoc(writer); } - IndexReader r = IndexReader.open(dir); + IndexReader r = DirectoryReader.open(dir); searcher = new IndexSearcher(r); hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals("reader incorrectly sees changes from writer", 14, hits.length); @@ -154,7 +153,7 @@ public class TestIndexWriterCommit extends LuceneTestCase { } writer.close(); - IndexReader r = IndexReader.open(dir); + IndexReader r = DirectoryReader.open(dir); searcher = new IndexSearcher(r); hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals("didn't see changes after close", 218, hits.length); @@ -235,7 +234,7 @@ public class TestIndexWriterCommit extends LuceneTestCase { writer.forceMerge(1); writer.close(); - IndexReader.open(dir).close(); + DirectoryReader.open(dir).close(); long endDiskUsage = dir.getMaxUsedSizeInBytes(); @@ -279,11 +278,11 @@ public class TestIndexWriterCommit extends LuceneTestCase { writer.forceMerge(1); // Open a reader before closing (commiting) the writer: - DirectoryReader reader = IndexReader.open(dir); + DirectoryReader reader = DirectoryReader.open(dir); // Reader should see index as multi-seg at this // point: - assertTrue("Reader incorrectly sees one segment", reader.getSequentialSubReaders().length > 1); + assertTrue("Reader incorrectly sees one segment", reader.getSequentialSubReaders().size() > 1); reader.close(); // Abort the writer: @@ -291,10 +290,10 @@ public class TestIndexWriterCommit extends LuceneTestCase { TestIndexWriter.assertNoUnreferencedFiles(dir, "aborted writer after forceMerge"); // Open a reader after aborting writer: - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); // Reader should still see index as multi-segment - assertTrue("Reader incorrectly sees one segment", reader.getSequentialSubReaders().length > 1); + assertTrue("Reader incorrectly sees one segment", reader.getSequentialSubReaders().size() > 1); reader.close(); if (VERBOSE) { @@ -310,10 +309,10 @@ public class TestIndexWriterCommit extends LuceneTestCase { TestIndexWriter.assertNoUnreferencedFiles(dir, "aborted writer after forceMerge"); // Open a reader after aborting writer: - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); // Reader should see index as one segment - assertEquals("Reader incorrectly sees more than one segment", 1, reader.getSequentialSubReaders().length); + assertEquals("Reader incorrectly sees more than one segment", 1, reader.getSequentialSubReaders().size()); reader.close(); dir.close(); } @@ -339,8 +338,8 @@ public class TestIndexWriterCommit extends LuceneTestCase { public void run() { try { final Document doc = new Document(); - DirectoryReader r = IndexReader.open(dir); - Field f = newField("f", "", StringField.TYPE_UNSTORED); + DirectoryReader r = DirectoryReader.open(dir); + Field f = newStringField("f", "", Field.Store.NO); doc.add(f); int count = 0; do { @@ -390,7 +389,7 @@ public class TestIndexWriterCommit extends LuceneTestCase { for (int i = 0; i < 23; i++) TestIndexWriter.addDoc(writer); - DirectoryReader reader = IndexReader.open(dir); + DirectoryReader reader = DirectoryReader.open(dir); assertEquals(0, reader.numDocs()); writer.commit(); DirectoryReader reader2 = DirectoryReader.openIfChanged(reader); @@ -403,12 +402,12 @@ public class TestIndexWriterCommit extends LuceneTestCase { TestIndexWriter.addDoc(writer); assertEquals(23, reader2.numDocs()); reader2.close(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); assertEquals(23, reader.numDocs()); reader.close(); writer.commit(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); assertEquals(40, reader.numDocs()); reader.close(); writer.close(); @@ -501,12 +500,12 @@ public class TestIndexWriterCommit extends LuceneTestCase { for (int i = 0; i < 23; i++) TestIndexWriter.addDoc(writer); - DirectoryReader reader = IndexReader.open(dir); + DirectoryReader reader = DirectoryReader.open(dir); assertEquals(0, reader.numDocs()); writer.prepareCommit(); - IndexReader reader2 = IndexReader.open(dir); + IndexReader reader2 = DirectoryReader.open(dir); assertEquals(0, reader2.numDocs()); writer.commit(); @@ -524,18 +523,18 @@ public class TestIndexWriterCommit extends LuceneTestCase { assertEquals(23, reader3.numDocs()); reader3.close(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); assertEquals(23, reader.numDocs()); reader.close(); writer.prepareCommit(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); assertEquals(23, reader.numDocs()); reader.close(); writer.commit(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); assertEquals(40, reader.numDocs()); reader.close(); writer.close(); @@ -558,12 +557,12 @@ public class TestIndexWriterCommit extends LuceneTestCase { for (int i = 0; i < 23; i++) TestIndexWriter.addDoc(writer); - DirectoryReader reader = IndexReader.open(dir); + DirectoryReader reader = DirectoryReader.open(dir); assertEquals(0, reader.numDocs()); writer.prepareCommit(); - IndexReader reader2 = IndexReader.open(dir); + IndexReader reader2 = DirectoryReader.open(dir); assertEquals(0, reader2.numDocs()); writer.rollback(); @@ -579,18 +578,18 @@ public class TestIndexWriterCommit extends LuceneTestCase { for (int i = 0; i < 17; i++) TestIndexWriter.addDoc(writer); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); assertEquals(0, reader.numDocs()); reader.close(); writer.prepareCommit(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); assertEquals(0, reader.numDocs()); reader.close(); writer.commit(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); assertEquals(17, reader.numDocs()); reader.close(); writer.close(); @@ -606,7 +605,7 @@ public class TestIndexWriterCommit extends LuceneTestCase { writer.commit(); writer.close(); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); assertEquals(0, reader.numDocs()); reader.close(); dir.close(); @@ -620,7 +619,7 @@ public class TestIndexWriterCommit extends LuceneTestCase { TestIndexWriter.addDoc(w); w.close(); - DirectoryReader r = IndexReader.open(dir); + DirectoryReader r = DirectoryReader.open(dir); // commit(Map) never called for this index assertEquals(0, r.getIndexCommit().getUserData().size()); r.close(); @@ -633,7 +632,7 @@ public class TestIndexWriterCommit extends LuceneTestCase { w.commit(data); w.close(); - r = IndexReader.open(dir); + r = DirectoryReader.open(dir); assertEquals("test1", r.getIndexCommit().getUserData().get("label")); r.close(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java index 20e9dae..10c66af 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -25,10 +25,13 @@ import java.util.Set; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field.Store; import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.store.Directory; import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.LuceneTestCase; import org.junit.Test; @@ -111,19 +114,71 @@ public class TestIndexWriterConfig extends LuceneTestCase { @Test public void testSettersChaining() throws Exception { - // Ensures that every setter returns IndexWriterConfig to enable easy - // chaining. + // Ensures that every setter returns IndexWriterConfig to allow chaining. + HashSet liveSetters = new HashSet(); + HashSet allSetters = new HashSet(); for (Method m : IndexWriterConfig.class.getDeclaredMethods()) { - if (m.getDeclaringClass() == IndexWriterConfig.class - && m.getName().startsWith("set") - && !Modifier.isStatic(m.getModifiers())) { - assertEquals("method " + m.getName() + " does not return IndexWriterConfig", - IndexWriterConfig.class, m.getReturnType()); + if (m.getName().startsWith("set") && !Modifier.isStatic(m.getModifiers())) { + allSetters.add(m.getName()); + // setters overridden from LiveIndexWriterConfig are returned twice, once with + // IndexWriterConfig return type and second with LiveIndexWriterConfig. The ones + // from LiveIndexWriterConfig are marked 'synthetic', so just collect them and + // assert in the end that we also received them from IWC. + if (m.isSynthetic()) { + liveSetters.add(m.getName()); + } else { + assertEquals("method " + m.getName() + " does not return IndexWriterConfig", + IndexWriterConfig.class, m.getReturnType()); + } } } + for (String setter : liveSetters) { + assertTrue("setter method not overridden by IndexWriterConfig: " + setter, allSetters.contains(setter)); + } } @Test + public void testReuse() throws Exception { + Directory dir = newDirectory(); + // test that if the same IWC is reused across two IWs, it is cloned by each. + IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir, conf); + LiveIndexWriterConfig liveConf1 = iw.w.getConfig(); + iw.close(); + + iw = new RandomIndexWriter(random(), dir, conf); + LiveIndexWriterConfig liveConf2 = iw.w.getConfig(); + iw.close(); + + // LiveIndexWriterConfig's "copy" constructor doesn't clone objects. + assertNotSame("IndexWriterConfig should have been cloned", liveConf1.getMergePolicy(), liveConf2.getMergePolicy()); + + dir.close(); + } + + @Test + public void testOverrideGetters() throws Exception { + // Test that IndexWriterConfig overrides all getters, so that javadocs + // contain all methods for the users. Also, ensures that IndexWriterConfig + // doesn't declare getters that are not declared on LiveIWC. + HashSet liveGetters = new HashSet(); + for (Method m : LiveIndexWriterConfig.class.getDeclaredMethods()) { + if (m.getName().startsWith("get") && !Modifier.isStatic(m.getModifiers())) { + liveGetters.add(m.getName()); + } + } + + for (Method m : IndexWriterConfig.class.getDeclaredMethods()) { + if (m.getName().startsWith("get") && !Modifier.isStatic(m.getModifiers())) { + assertEquals("method " + m.getName() + " not overrided by IndexWriterConfig", + IndexWriterConfig.class, m.getDeclaringClass()); + assertTrue("method " + m.getName() + " not declared on LiveIndexWriterConfig", + liveGetters.contains(m.getName())); + } + } + } + + @Test public void testConstants() throws Exception { // Tests that the values of the constants does not change assertEquals(1000, IndexWriterConfig.WRITE_LOCK_TIMEOUT); @@ -149,6 +204,9 @@ public class TestIndexWriterConfig extends LuceneTestCase { // toString. continue; } + if (f.getName().equals("inUseByIndexWriter")) { + continue; + } assertTrue(f.getName() + " not found in toString", str.indexOf(f.getName()) != -1); } } @@ -269,4 +327,31 @@ public class TestIndexWriterConfig extends LuceneTestCase { conf.setMergePolicy(null); assertEquals(LogByteSizeMergePolicy.class, conf.getMergePolicy().getClass()); } + + public void testLiveChangeToCFS() throws Exception { + Directory dir = newDirectory(); + IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); + iwc.setMergePolicy(newLogMergePolicy()); + + // Start false: + ((LogMergePolicy) iwc.getMergePolicy()).setUseCompoundFile(false); + IndexWriter w = new IndexWriter(dir, iwc); + + // Change to true: + ((LogMergePolicy) w.getConfig().getMergePolicy()).setNoCFSRatio(1.0); + ((LogMergePolicy) w.getConfig().getMergePolicy()).setUseCompoundFile(true); + + Document doc = new Document(); + doc.add(newStringField("field", "foo", Store.NO)); + w.addDocument(doc); + w.commit(); + + for(String file : dir.listAll()) { + // frq file should be stuck into CFS + assertFalse(file.endsWith(".frq")); + } + w.close(); + dir.close(); + } + } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java index d7db99a..5ab6f78 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -28,9 +28,9 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.analysis.*; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; @@ -58,10 +58,10 @@ public class TestIndexWriterDelete extends LuceneTestCase { custom1.setStored(true); for (int i = 0; i < keywords.length; i++) { Document doc = new Document(); - doc.add(newField("id", keywords[i], StringField.TYPE_STORED)); + doc.add(newStringField("id", keywords[i], Field.Store.YES)); doc.add(newField("country", unindexed[i], custom1)); - doc.add(newField("contents", unstored[i], TextField.TYPE_UNSTORED)); - doc.add(newField("city", text[i], TextField.TYPE_STORED)); + doc.add(newTextField("contents", unstored[i], Field.Store.NO)); + doc.add(newTextField("city", text[i], Field.Store.YES)); modifier.addDocument(doc); } modifier.forceMerge(1); @@ -106,7 +106,7 @@ public class TestIndexWriterDelete extends LuceneTestCase { modifier.commit(); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); assertEquals(7, reader.numDocs()); reader.close(); @@ -114,7 +114,7 @@ public class TestIndexWriterDelete extends LuceneTestCase { modifier.commit(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); assertEquals(0, reader.numDocs()); reader.close(); modifier.close(); @@ -166,7 +166,7 @@ public class TestIndexWriterDelete extends LuceneTestCase { assertEquals(0, modifier.getSegmentCount()); modifier.commit(); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); assertEquals(1, reader.numDocs()); int hitCount = getHitCount(dir, new Term("id", String.valueOf(id))); @@ -204,7 +204,7 @@ public class TestIndexWriterDelete extends LuceneTestCase { modifier.commit(); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); assertEquals(5, reader.numDocs()); modifier.close(); reader.close(); @@ -226,7 +226,7 @@ public class TestIndexWriterDelete extends LuceneTestCase { } modifier.commit(); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); assertEquals(7, reader.numDocs()); reader.close(); @@ -236,7 +236,7 @@ public class TestIndexWriterDelete extends LuceneTestCase { modifier.commit(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); assertEquals(5, reader.numDocs()); reader.close(); @@ -246,7 +246,7 @@ public class TestIndexWriterDelete extends LuceneTestCase { } modifier.deleteDocuments(terms); modifier.commit(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); assertEquals(2, reader.numDocs()); reader.close(); @@ -269,7 +269,7 @@ public class TestIndexWriterDelete extends LuceneTestCase { } modifier.commit(); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); assertEquals(7, reader.numDocs()); reader.close(); @@ -280,7 +280,7 @@ public class TestIndexWriterDelete extends LuceneTestCase { modifier.deleteAll(); // Delete all shouldn't be on disk yet - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); assertEquals(7, reader.numDocs()); reader.close(); @@ -292,7 +292,7 @@ public class TestIndexWriterDelete extends LuceneTestCase { modifier.commit(); // Validate there are no docs left - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); assertEquals(2, reader.numDocs()); reader.close(); @@ -317,7 +317,7 @@ public class TestIndexWriterDelete extends LuceneTestCase { addDoc(modifier, ++id, value); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); assertEquals(7, reader.numDocs()); reader.close(); @@ -329,7 +329,7 @@ public class TestIndexWriterDelete extends LuceneTestCase { modifier.close(); // Validate that the docs are still there - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); assertEquals(7, reader.numDocs()); reader.close(); @@ -372,7 +372,7 @@ public class TestIndexWriterDelete extends LuceneTestCase { modifier.close(); // Validate that the docs are still there - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); assertEquals(7, reader.numDocs()); reader.close(); @@ -383,9 +383,9 @@ public class TestIndexWriterDelete extends LuceneTestCase { private void updateDoc(IndexWriter modifier, int id, int value) throws IOException { Document doc = new Document(); - doc.add(newField("content", "aaa", TextField.TYPE_UNSTORED)); - doc.add(newField("id", String.valueOf(id), StringField.TYPE_STORED)); - doc.add(newField("value", String.valueOf(value), StringField.TYPE_UNSTORED)); + doc.add(newTextField("content", "aaa", Field.Store.NO)); + doc.add(newStringField("id", String.valueOf(id), Field.Store.YES)); + doc.add(newStringField("value", String.valueOf(value), Field.Store.NO)); modifier.updateDocument(new Term("id", String.valueOf(id)), doc); } @@ -393,14 +393,14 @@ public class TestIndexWriterDelete extends LuceneTestCase { private void addDoc(IndexWriter modifier, int id, int value) throws IOException { Document doc = new Document(); - doc.add(newField("content", "aaa", TextField.TYPE_UNSTORED)); - doc.add(newField("id", String.valueOf(id), StringField.TYPE_STORED)); - doc.add(newField("value", String.valueOf(value), StringField.TYPE_UNSTORED)); + doc.add(newTextField("content", "aaa", Field.Store.NO)); + doc.add(newStringField("id", String.valueOf(id), Field.Store.YES)); + doc.add(newStringField("value", String.valueOf(value), Field.Store.NO)); modifier.addDocument(doc); } private int getHitCount(Directory dir, Term term) throws IOException { - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); int hitCount = searcher.search(new TermQuery(term), null, 1000).totalHits; reader.close(); @@ -432,8 +432,8 @@ public class TestIndexWriterDelete extends LuceneTestCase { IndexWriter writer = new IndexWriter(startDir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false))); for (int i = 0; i < 157; i++) { Document d = new Document(); - d.add(newField("id", Integer.toString(i), StringField.TYPE_STORED)); - d.add(newField("content", "aaa " + i, TextField.TYPE_UNSTORED)); + d.add(newStringField("id", Integer.toString(i), Field.Store.YES)); + d.add(newTextField("content", "aaa " + i, Field.Store.NO)); writer.addDocument(d); } writer.close(); @@ -510,8 +510,8 @@ public class TestIndexWriterDelete extends LuceneTestCase { for (int i = 0; i < 13; i++) { if (updates) { Document d = new Document(); - d.add(newField("id", Integer.toString(i), StringField.TYPE_STORED)); - d.add(newField("content", "bbb " + i, TextField.TYPE_UNSTORED)); + d.add(newStringField("id", Integer.toString(i), Field.Store.YES)); + d.add(newTextField("content", "bbb " + i, Field.Store.NO)); modifier.updateDocument(new Term("id", Integer.toString(docId)), d); } else { // deletes modifier.deleteDocuments(new Term("id", Integer.toString(docId))); @@ -566,7 +566,7 @@ public class TestIndexWriterDelete extends LuceneTestCase { // changed (transactional semantics): IndexReader newReader = null; try { - newReader = IndexReader.open(dir); + newReader = DirectoryReader.open(dir); } catch (IOException e) { e.printStackTrace(); @@ -702,10 +702,10 @@ public class TestIndexWriterDelete extends LuceneTestCase { custom1.setStored(true); for (int i = 0; i < keywords.length; i++) { Document doc = new Document(); - doc.add(newField("id", keywords[i], StringField.TYPE_STORED)); + doc.add(newStringField("id", keywords[i], Field.Store.YES)); doc.add(newField("country", unindexed[i], custom1)); - doc.add(newField("contents", unstored[i], TextField.TYPE_UNSTORED)); - doc.add(newField("city", text[i], TextField.TYPE_STORED)); + doc.add(newTextField("contents", unstored[i], Field.Store.NO)); + doc.add(newTextField("city", text[i], Field.Store.YES)); modifier.addDocument(doc); } // flush (and commit if ac) @@ -823,10 +823,10 @@ public class TestIndexWriterDelete extends LuceneTestCase { custom1.setStored(true); for (int i = 0; i < keywords.length; i++) { Document doc = new Document(); - doc.add(newField("id", keywords[i], StringField.TYPE_STORED)); + doc.add(newStringField("id", keywords[i], Field.Store.YES)); doc.add(newField("country", unindexed[i], custom1)); - doc.add(newField("contents", unstored[i], TextField.TYPE_UNSTORED)); - doc.add(newField("city", text[i], TextField.TYPE_STORED)); + doc.add(newTextField("contents", unstored[i], Field.Store.NO)); + doc.add(newTextField("city", text[i], Field.Store.YES)); try { modifier.addDocument(doc); } catch (IOException io) { @@ -869,7 +869,7 @@ public class TestIndexWriterDelete extends LuceneTestCase { Collections.shuffle(ids, random()); for(int id : ids) { Document doc = new Document(); - doc.add(newField("id", ""+id, StringField.TYPE_UNSTORED)); + doc.add(newStringField("id", ""+id, Field.Store.NO)); w.addDocument(doc); } Collections.shuffle(ids, random()); @@ -905,7 +905,7 @@ public class TestIndexWriterDelete extends LuceneTestCase { }; IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setRAMBufferSizeMB(1.0).setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH)); Document doc = new Document(); - doc.add(newField("field", "go 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20", TextField.TYPE_UNSTORED)); + doc.add(newTextField("field", "go 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20", Field.Store.NO)); int num = atLeast(3); for (int iter = 0; iter < num; iter++) { int count = 0; @@ -949,7 +949,7 @@ public class TestIndexWriterDelete extends LuceneTestCase { int count = 0; while(true) { Document doc = new Document(); - doc.add(new StringField("id", count+"")); + doc.add(new StringField("id", count+"", Field.Store.NO)); final Term delTerm; if (count == 1010) { // This is the only delete that applies @@ -995,7 +995,7 @@ public class TestIndexWriterDelete extends LuceneTestCase { int count = 0; while(true) { Document doc = new Document(); - doc.add(new StringField("id", count+"")); + doc.add(new StringField("id", count+"", Field.Store.NO)); final Term delTerm; if (count == 1010) { // This is the only delete that applies @@ -1050,8 +1050,8 @@ public class TestIndexWriterDelete extends LuceneTestCase { w.deleteDocuments(new Term("id", "0")); } Document doc = new Document(); - doc.add(newField("id", ""+id, StringField.TYPE_UNSTORED)); - doc.add(newField("body", sb.toString(), TextField.TYPE_UNSTORED)); + doc.add(newStringField("id", ""+id, Field.Store.NO)); + doc.add(newTextField("body", sb.toString(), Field.Store.NO)); w.updateDocument(new Term("id", ""+id), doc); docsInSegment.incrementAndGet(); // TODO: fix this test diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java index 1a64144..50a0480 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -57,10 +57,10 @@ public class TestIndexWriterExceptions extends LuceneTestCase { /* private field types */ /* private field types */ - private static final FieldType custom1 = new FieldType(TextField.TYPE_UNSTORED); + private static final FieldType custom1 = new FieldType(TextField.TYPE_NOT_STORED); private static final FieldType custom2 = new FieldType(); private static final FieldType custom3 = new FieldType(); - private static final FieldType custom4 = new FieldType(StringField.TYPE_UNSTORED); + private static final FieldType custom4 = new FieldType(StringField.TYPE_NOT_STORED); private static final FieldType custom5 = new FieldType(TextField.TYPE_STORED); static { @@ -129,13 +129,13 @@ public class TestIndexWriterExceptions extends LuceneTestCase { final Document doc = new Document(); - doc.add(newField(r, "content1", "aaa bbb ccc ddd", TextField.TYPE_STORED)); + doc.add(newTextField(r, "content1", "aaa bbb ccc ddd", Field.Store.YES)); doc.add(newField(r, "content6", "aaa bbb ccc ddd", DocCopyIterator.custom1)); doc.add(newField(r, "content2", "aaa bbb ccc ddd", DocCopyIterator.custom2)); doc.add(newField(r, "content3", "aaa bbb ccc ddd", DocCopyIterator.custom3)); - doc.add(newField(r, "content4", "aaa bbb ccc ddd", TextField.TYPE_UNSTORED)); - doc.add(newField(r, "content5", "aaa bbb ccc ddd", StringField.TYPE_UNSTORED)); + doc.add(newTextField(r, "content4", "aaa bbb ccc ddd", Field.Store.NO)); + doc.add(newStringField(r, "content5", "aaa bbb ccc ddd", Field.Store.NO)); doc.add(newField(r, "content7", "aaa bbb ccc ddd", DocCopyIterator.custom4)); @@ -255,7 +255,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { } // Confirm that when doc hits exception partway through tokenization, it's deleted: - IndexReader r2 = IndexReader.open(dir); + IndexReader r2 = DirectoryReader.open(dir); final int count = r2.docFreq(new Term("content4", "aaa")); final int count2 = r2.docFreq(new Term("content4", "ddd")); assertEquals(count, count2); @@ -301,7 +301,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { } // Confirm that when doc hits exception partway through tokenization, it's deleted: - IndexReader r2 = IndexReader.open(dir); + IndexReader r2 = DirectoryReader.open(dir); final int count = r2.docFreq(new Term("content4", "aaa")); final int count2 = r2.docFreq(new Term("content4", "ddd")); assertEquals(count, count2); @@ -356,7 +356,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { Directory dir = newDirectory(); MockIndexWriter2 w = new MockIndexWriter2(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random()))); Document doc = new Document(); - doc.add(newField("field", "a field", TextField.TYPE_STORED)); + doc.add(newTextField("field", "a field", Field.Store.YES)); w.addDocument(doc); w.doFail = true; try { @@ -374,7 +374,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { Directory dir = newDirectory(); MockIndexWriter w = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMaxBufferedDocs(2)); Document doc = new Document(); - doc.add(newField("field", "a field", TextField.TYPE_STORED)); + doc.add(newTextField("field", "a field", Field.Store.YES)); w.addDocument(doc); Analyzer analyzer = new Analyzer(new Analyzer.PerFieldReuseStrategy()) { @@ -387,7 +387,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { }; Document crashDoc = new Document(); - crashDoc.add(newField("crash", "do it on token 4", TextField.TYPE_STORED)); + crashDoc.add(newTextField("crash", "do it on token 4", Field.Store.YES)); try { w.addDocument(crashDoc, analyzer); fail("did not hit expected exception"); @@ -428,7 +428,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { MockIndexWriter3 w = new MockIndexWriter3(dir, conf); w.doFail = true; Document doc = new Document(); - doc.add(newField("field", "a field", TextField.TYPE_STORED)); + doc.add(newTextField("field", "a field", Field.Store.YES)); for(int i=0;i<10;i++) try { w.addDocument(doc); @@ -477,7 +477,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { Document doc = new Document(); String contents = "aa bb cc dd ee ff gg hh ii jj kk"; - doc.add(newField("content", contents, TextField.TYPE_UNSTORED)); + doc.add(newTextField("content", contents, Field.Store.NO)); try { writer.addDocument(doc); fail("did not hit expected exception"); @@ -486,16 +486,16 @@ public class TestIndexWriterExceptions extends LuceneTestCase { // Make sure we can add another normal document doc = new Document(); - doc.add(newField("content", "aa bb cc dd", TextField.TYPE_UNSTORED)); + doc.add(newTextField("content", "aa bb cc dd", Field.Store.NO)); writer.addDocument(doc); // Make sure we can add another normal document doc = new Document(); - doc.add(newField("content", "aa bb cc dd", TextField.TYPE_UNSTORED)); + doc.add(newTextField("content", "aa bb cc dd", Field.Store.NO)); writer.addDocument(doc); writer.close(); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); final Term t = new Term("content", "aa"); assertEquals(3, reader.docFreq(t)); @@ -564,7 +564,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMaxBufferedDocs(2)); Document doc = new Document(); String contents = "aa bb cc dd ee ff gg hh ii jj kk"; - doc.add(newField("content", contents, TextField.TYPE_UNSTORED)); + doc.add(newTextField("content", contents, Field.Store.NO)); boolean hitError = false; for(int i=0;i<200;i++) { try { @@ -577,7 +577,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { } assertTrue(hitError); writer.close(); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); assertEquals(198, reader.docFreq(new Term("content", "aa"))); reader.close(); dir.close(); @@ -632,7 +632,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { if (VERBOSE) { System.out.println("TEST: open reader"); } - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); if (i == 0) { int expected = 5; assertEquals(expected, reader.docFreq(new Term("contents", "here"))); @@ -661,7 +661,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { writer.forceMerge(1); writer.close(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); int expected = 19+(1-i)*2; assertEquals(expected, reader.docFreq(new Term("contents", "here"))); assertEquals(expected, reader.maxDoc()); @@ -747,7 +747,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { writer.close(); } - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); int expected = (3+(1-i)*2)*NUM_THREAD*NUM_ITER; assertEquals("i=" + i, expected, reader.docFreq(new Term("contents", "here"))); assertEquals(expected, reader.maxDoc()); @@ -775,7 +775,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { writer.forceMerge(1); writer.close(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); expected += 17-NUM_THREAD*NUM_ITER; assertEquals(expected, reader.docFreq(new Term("contents", "here"))); assertEquals(expected, reader.maxDoc()); @@ -816,7 +816,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { private void addDoc(IndexWriter writer) throws IOException { Document doc = new Document(); - doc.add(newField("content", "aaa", TextField.TYPE_UNSTORED)); + doc.add(newTextField("content", "aaa", Field.Store.NO)); writer.addDocument(doc); } @@ -850,7 +850,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { failure.clearDoFail(); writer.close(); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); assertEquals(23, reader.numDocs()); reader.close(); dir.close(); @@ -913,7 +913,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random()))); Document doc = new Document(); - doc.add(newField("field", "a field", TextField.TYPE_STORED)); + doc.add(newTextField("field", "a field", Field.Store.YES)); w.addDocument(doc); dir.failOn(failure); try { @@ -983,7 +983,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { } @Override - public void close() throws IOException {} + public void close() {} })); try { @@ -1063,7 +1063,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { IndexReader reader = null; try { - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); } catch (IOException e) { e.printStackTrace(System.out); fail("segmentInfos failed to retry fallback to correct segments_N file"); @@ -1110,7 +1110,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { IndexReader reader = null; try { - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); fail("reader did not hit IOException on opening a corrupt index"); } catch (Exception e) { } @@ -1159,7 +1159,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { IndexReader reader = null; try { - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); fail("reader did not hit IOException on opening a corrupt index"); } catch (Exception e) { } @@ -1210,7 +1210,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { IndexReader reader = null; try { - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); } catch (Exception e) { fail("reader failed to open on a crashed index"); } @@ -1247,7 +1247,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { int numDocs = 10 + random().nextInt(30); for (int i = 0; i < numDocs; i++) { Document doc = new Document(); - Field field = newField(random(), "field", "a field", TextField.TYPE_STORED); + Field field = newTextField(random(), "field", "a field", Field.Store.YES); doc.add(field); // random TV try { @@ -1263,12 +1263,12 @@ public class TestIndexWriterExceptions extends LuceneTestCase { } Document document = new Document(); - document.add(new Field("field", "a field", TextField.TYPE_STORED)); + document.add(new TextField("field", "a field", Field.Store.YES)); w.addDocument(document); for (int i = 0; i < numDocs; i++) { Document doc = new Document(); - Field field = newField(random(), "field", "a field", TextField.TYPE_STORED); + Field field = newTextField(random(), "field", "a field", Field.Store.YES); doc.add(field); // random TV try { @@ -1283,10 +1283,10 @@ public class TestIndexWriterExceptions extends LuceneTestCase { } } document = new Document(); - document.add(new Field("field", "a field", TextField.TYPE_STORED)); + document.add(new TextField("field", "a field", Field.Store.YES)); w.addDocument(document); w.close(); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); assertTrue(reader.numDocs() > 0); SegmentInfos sis = new SegmentInfos(); sis.read(dir); @@ -1333,7 +1333,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { final int numDocs1 = random().nextInt(25); for(int docCount=0;docCount 1); + DirectoryReader reader = DirectoryReader.open(dir); + assertTrue(reader.getSequentialSubReaders().size() > 1); reader.close(); SegmentInfos infos = new SegmentInfos(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterLockRelease.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterLockRelease.java index 2952f07..cb2a8e8 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterLockRelease.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterLockRelease.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java index 755494b..114f583 100755 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,7 +21,7 @@ import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.store.Directory; @@ -223,11 +223,11 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase { private void addDoc(IndexWriter writer) throws IOException { Document doc = new Document(); - doc.add(newField("content", "aaa", TextField.TYPE_UNSTORED)); + doc.add(newTextField("content", "aaa", Field.Store.NO)); writer.addDocument(doc); } - private void checkInvariants(IndexWriter writer) throws IOException { + private void checkInvariants(IndexWriter writer) { writer.waitForMerges(); int maxBufferedDocs = writer.getConfig().getMaxBufferedDocs(); int mergeFactor = ((LogMergePolicy) writer.getConfig().getMergePolicy()).getMergeFactor(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMerging.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMerging.java index 24e481f..d07a775 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMerging.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMerging.java @@ -21,7 +21,6 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.util.LuceneTestCase; @@ -80,7 +79,7 @@ public class TestIndexWriterMerging extends LuceneTestCase private boolean verifyIndex(Directory directory, int startAt) throws IOException { boolean fail = false; - IndexReader reader = IndexReader.open(directory); + IndexReader reader = DirectoryReader.open(directory); int max = reader.maxDoc(); for (int i = 0; i < max; i++) @@ -111,7 +110,7 @@ public class TestIndexWriterMerging extends LuceneTestCase for (int i = start; i < (start + numDocs); i++) { Document temp = new Document(); - temp.add(newField("count", (""+i), StringField.TYPE_STORED)); + temp.add(newStringField("count", (""+i), Field.Store.YES)); writer.addDocument(temp); } @@ -131,13 +130,13 @@ public class TestIndexWriterMerging extends LuceneTestCase FieldType customType = new FieldType(); customType.setStored(true); - FieldType customType1 = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType1 = new FieldType(TextField.TYPE_NOT_STORED); customType1.setTokenized(false); customType1.setStoreTermVectors(true); customType1.setStoreTermVectorPositions(true); customType1.setStoreTermVectorOffsets(true); - Field idField = newField("id", "", StringField.TYPE_UNSTORED); + Field idField = newStringField("id", "", Field.Store.NO); document.add(idField); Field storedField = newField("stored", "stored", customType); document.add(storedField); @@ -149,7 +148,7 @@ public class TestIndexWriterMerging extends LuceneTestCase } writer.close(); - IndexReader ir = IndexReader.open(dir); + IndexReader ir = DirectoryReader.open(dir); assertEquals(10, ir.maxDoc()); assertEquals(10, ir.numDocs()); ir.close(); @@ -161,7 +160,7 @@ public class TestIndexWriterMerging extends LuceneTestCase writer.deleteDocuments(new Term("id", "7")); writer.close(); - ir = IndexReader.open(dir); + ir = DirectoryReader.open(dir); assertEquals(8, ir.numDocs()); ir.close(); @@ -171,7 +170,7 @@ public class TestIndexWriterMerging extends LuceneTestCase writer.forceMergeDeletes(); assertEquals(8, writer.numDocs()); writer.close(); - ir = IndexReader.open(dir); + ir = DirectoryReader.open(dir); assertEquals(8, ir.maxDoc()); assertEquals(8, ir.numDocs()); ir.close(); @@ -194,7 +193,7 @@ public class TestIndexWriterMerging extends LuceneTestCase FieldType customType = new FieldType(); customType.setStored(true); - FieldType customType1 = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType1 = new FieldType(TextField.TYPE_NOT_STORED); customType1.setTokenized(false); customType1.setStoreTermVectors(true); customType1.setStoreTermVectorPositions(true); @@ -204,7 +203,7 @@ public class TestIndexWriterMerging extends LuceneTestCase document.add(storedField); Field termVectorField = newField("termVector", "termVector", customType1); document.add(termVectorField); - Field idField = newField("id", "", StringField.TYPE_UNSTORED); + Field idField = newStringField("id", "", Field.Store.NO); document.add(idField); for(int i=0;i<98;i++) { idField.setStringValue("" + i); @@ -212,7 +211,7 @@ public class TestIndexWriterMerging extends LuceneTestCase } writer.close(); - IndexReader ir = IndexReader.open(dir); + IndexReader ir = DirectoryReader.open(dir); assertEquals(98, ir.maxDoc()); assertEquals(98, ir.numDocs()); ir.close(); @@ -225,7 +224,7 @@ public class TestIndexWriterMerging extends LuceneTestCase } writer.close(); - ir = IndexReader.open(dir); + ir = DirectoryReader.open(dir); assertEquals(49, ir.numDocs()); ir.close(); @@ -237,7 +236,7 @@ public class TestIndexWriterMerging extends LuceneTestCase assertEquals(49, writer.numDocs()); writer.forceMergeDeletes(); writer.close(); - ir = IndexReader.open(dir); + ir = DirectoryReader.open(dir); assertEquals(49, ir.maxDoc()); assertEquals(49, ir.numDocs()); ir.close(); @@ -259,7 +258,7 @@ public class TestIndexWriterMerging extends LuceneTestCase FieldType customType = new FieldType(); customType.setStored(true); - FieldType customType1 = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType1 = new FieldType(TextField.TYPE_NOT_STORED); customType1.setTokenized(false); customType1.setStoreTermVectors(true); customType1.setStoreTermVectorPositions(true); @@ -270,7 +269,7 @@ public class TestIndexWriterMerging extends LuceneTestCase document.add(storedField); Field termVectorField = newField("termVector", "termVector", customType1); document.add(termVectorField); - Field idField = newField("id", "", StringField.TYPE_UNSTORED); + Field idField = newStringField("id", "", Field.Store.NO); document.add(idField); for(int i=0;i<98;i++) { idField.setStringValue("" + i); @@ -278,7 +277,7 @@ public class TestIndexWriterMerging extends LuceneTestCase } writer.close(); - IndexReader ir = IndexReader.open(dir); + IndexReader ir = DirectoryReader.open(dir); assertEquals(98, ir.maxDoc()); assertEquals(98, ir.numDocs()); ir.close(); @@ -290,7 +289,7 @@ public class TestIndexWriterMerging extends LuceneTestCase writer.deleteDocuments(new Term("id", "" + i)); } writer.close(); - ir = IndexReader.open(dir); + ir = DirectoryReader.open(dir); assertEquals(49, ir.numDocs()); ir.close(); @@ -301,7 +300,7 @@ public class TestIndexWriterMerging extends LuceneTestCase ); writer.forceMergeDeletes(false); writer.close(); - ir = IndexReader.open(dir); + ir = DirectoryReader.open(dir); assertEquals(49, ir.maxDoc()); assertEquals(49, ir.numDocs()); ir.close(); @@ -313,7 +312,7 @@ public class TestIndexWriterMerging extends LuceneTestCase private class MyMergeScheduler extends MergeScheduler { @Override synchronized public void merge(IndexWriter writer) - throws CorruptIndexException, IOException { + throws IOException { while(true) { MergePolicy.OneMerge merge = writer.getNextMerge(); @@ -343,7 +342,7 @@ public class TestIndexWriterMerging extends LuceneTestCase IndexWriter iw = new IndexWriter(dir, conf); Document document = new Document(); - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setStoreTermVectors(true); document.add(newField("tvtest", "a b c", customType)); @@ -438,7 +437,7 @@ public class TestIndexWriterMerging extends LuceneTestCase t1.join(); // Make sure reader can read - IndexReader reader = IndexReader.open(directory); + IndexReader reader = DirectoryReader.open(directory); reader.close(); // Reopen diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterNRTIsCurrent.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterNRTIsCurrent.java index 496d3e5..da6c43f 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterNRTIsCurrent.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterNRTIsCurrent.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF @@ -25,7 +25,6 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; @@ -36,8 +35,8 @@ public class TestIndexWriterNRTIsCurrent extends LuceneTestCase { volatile boolean stop = false; } - public void testIsCurrentWithThreads() throws CorruptIndexException, - LockObtainFailedException, IOException, InterruptedException { + public void testIsCurrentWithThreads() throws + IOException, InterruptedException { Directory dir = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); @@ -92,7 +91,7 @@ public class TestIndexWriterNRTIsCurrent extends LuceneTestCase { Random random = LuceneTestCase.random(); try { Document doc = new Document(); - doc.add(new Field("id", "1", TextField.TYPE_UNSTORED)); + doc.add(new TextField("id", "1", Field.Store.NO)); writer.addDocument(doc); holder.reader = currentReader = writer.getReader(true); Term term = new Term("id"); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java index 0129ef5..fd0a60e 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,6 +22,7 @@ import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.LiveDocsFormat; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; @@ -114,7 +115,7 @@ public class TestIndexWriterOnDiskFull extends LuceneTestCase { assertNoUnreferencedFiles(dir, "after disk full during addDocument"); // Make sure reader can open the index: - IndexReader.open(dir).close(); + DirectoryReader.open(dir).close(); } dir.close(); @@ -190,7 +191,7 @@ public class TestIndexWriterOnDiskFull extends LuceneTestCase { // Make sure starting index seems to be working properly: Term searchTerm = new Term("content", "aaa"); - IndexReader reader = IndexReader.open(startDir); + IndexReader reader = DirectoryReader.open(startDir); assertEquals("first docFreq", 57, reader.docFreq(searchTerm)); IndexSearcher searcher = newSearcher(reader); @@ -313,7 +314,7 @@ public class TestIndexWriterOnDiskFull extends LuceneTestCase { } else if (1 == method) { IndexReader readers[] = new IndexReader[dirs.length]; for(int i=0;i allTerms = new HashSet(); @@ -318,12 +315,9 @@ public class TestIndexWriterUnicode extends LuceneTestCase { IndexReader r = writer.getReader(); // Test each sub-segment - new ReaderUtil.Gather(r) { - @Override - protected void add(int base, AtomicReader r) throws IOException { - checkTermsOrder(r, allTerms, false); - } - }.run(); + for (AtomicReaderContext ctx : r.getTopReaderContext().leaves()) { + checkTermsOrder(ctx.reader(), allTerms, false); + } checkTermsOrder(r, allTerms, true); // Test multi segment diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java index 8e909fe..76b67fb 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,6 +19,10 @@ package org.apache.lucene.index; import java.io.IOException; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; @@ -31,13 +35,16 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.ThreadInterruptedException; import org.apache.lucene.util._TestUtil; +import org.apache.lucene.util.LuceneTestCase.Slow; /** * MultiThreaded IndexWriter tests */ +@Slow public class TestIndexWriterWithThreads extends LuceneTestCase { // Used by test cases below @@ -210,7 +217,7 @@ public class TestIndexWriterWithThreads extends LuceneTestCase { } // Quick test to make sure index is not corrupt: - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); DocsEnum tdocs = _TestUtil.docs(random(), reader, "field", new BytesRef("aaa"), @@ -277,7 +284,7 @@ public class TestIndexWriterWithThreads extends LuceneTestCase { } if (success) { - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); final Bits delDocs = MultiFields.getLiveDocs(reader); for(int j=0;j writerRef = new AtomicReference(); + writerRef.set(new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())))); + final LineFileDocs docs = new LineFileDocs(random()); + final Thread[] threads = new Thread[threadCount]; + final int iters = atLeast(1000); + final AtomicBoolean failed = new AtomicBoolean(); + final Lock rollbackLock = new ReentrantLock(); + final Lock commitLock = new ReentrantLock(); + for(int threadID=0;threadID allFiles = new HashSet(); - DirectoryReader r = IndexReader.open(d); + DirectoryReader r = DirectoryReader.open(d); while(System.currentTimeMillis() < stopTime) { final IndexCommit ic = r.getIndexCommit(); if (VERBOSE) { diff --git a/lucene/core/src/test/org/apache/lucene/index/TestNewestSegment.java b/lucene/core/src/test/org/apache/lucene/index/TestNewestSegment.java index 1ea965d..348ef43 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestNewestSegment.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestNewestSegment.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/index/TestNoDeletionPolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestNoDeletionPolicy.java index 2c0de06..3e032bb 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestNoDeletionPolicy.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestNoDeletionPolicy.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,7 +23,7 @@ import java.lang.reflect.Modifier; import java.util.Arrays; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import org.junit.Test; @@ -73,7 +73,7 @@ public class TestNoDeletionPolicy extends LuceneTestCase { .setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE)); for (int i = 0; i < 10; i++) { Document doc = new Document(); - doc.add(newField("c", "a" + i, TextField.TYPE_STORED)); + doc.add(newTextField("c", "a" + i, Field.Store.YES)); writer.addDocument(doc); writer.commit(); assertEquals("wrong number of commits !", i + 1, DirectoryReader.listCommits(dir).size()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestNoMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestNoMergePolicy.java index f5f4a4e..4014cf3 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestNoMergePolicy.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestNoMergePolicy.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -62,6 +62,9 @@ public class TestNoMergePolicy extends LuceneTestCase { // context, including ones from Object. So just filter out Object. If in // the future MergePolicy will extend a different class than Object, this // will need to change. + if (m.getName().equals("clone")) { + continue; + } if (m.getDeclaringClass() != Object.class) { assertTrue(m + " is not overridden !", m.getDeclaringClass() == NoMergePolicy.class); } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestNoMergeScheduler.java b/lucene/core/src/test/org/apache/lucene/index/TestNoMergeScheduler.java index 78fb4dc..a2f0c1b 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestNoMergeScheduler.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestNoMergeScheduler.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/index/TestNorms.java b/lucene/core/src/test/org/apache/lucene/index/TestNorms.java index 5219c65..f7447aa 100755 --- a/lucene/core/src/test/org/apache/lucene/index/TestNorms.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestNorms.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -31,6 +31,7 @@ import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LineFileDocs; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; @@ -40,6 +41,7 @@ import org.apache.lucene.util._TestUtil; * separate norms, addDocument, addIndexes, forceMerge. */ @SuppressCodecs({ "SimpleText", "Memory" }) +@Slow public class TestNorms extends LuceneTestCase { final String byteTestField = "normsTestByte"; @@ -67,8 +69,8 @@ public class TestNorms extends LuceneTestCase { config.setSimilarity(new CustomNormEncodingSimilarity()); RandomIndexWriter writer = new RandomIndexWriter(random(), dir, config); Document doc = new Document(); - Field foo = newField("foo", "", TextField.TYPE_UNSTORED); - Field bar = newField("bar", "", TextField.TYPE_UNSTORED); + Field foo = newTextField("foo", "", Field.Store.NO); + Field bar = newTextField("bar", "", Field.Store.NO); doc.add(foo); doc.add(bar); @@ -95,7 +97,7 @@ public class TestNorms extends LuceneTestCase { public void testMaxByteNorms() throws IOException { Directory dir = newFSDirectory(_TestUtil.getTempDir("TestNorms.testMaxByteNorms")); buildIndex(dir, true); - AtomicReader open = SlowCompositeReaderWrapper.wrap(IndexReader.open(dir)); + AtomicReader open = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir)); DocValues normValues = open.normValues(byteTestField); assertNotNull(normValues); Source source = normValues.getSource(); @@ -126,7 +128,7 @@ public class TestNorms extends LuceneTestCase { boolean secondWriteNorm = random().nextBoolean(); buildIndex(otherDir, secondWriteNorm); - AtomicReader reader = SlowCompositeReaderWrapper.wrap(IndexReader.open(otherDir)); + AtomicReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(otherDir)); FieldInfos fieldInfos = reader.getFieldInfos(); FieldInfo fieldInfo = fieldInfos.fieldInfo(byteTestField); assertFalse(fieldInfo.omitsNorms()); @@ -175,21 +177,19 @@ public class TestNorms extends LuceneTestCase { otherDir.close(); } - public void buildIndex(Directory dir, boolean writeNorms) throws IOException, - CorruptIndexException { + public void buildIndex(Directory dir, boolean writeNorms) throws IOException { Random random = random(); IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); Similarity provider = new MySimProvider(writeNorms); config.setSimilarity(provider); RandomIndexWriter writer = new RandomIndexWriter(random, dir, config); - final LineFileDocs docs = new LineFileDocs(random, defaultCodecSupportsDocValues()); + final LineFileDocs docs = new LineFileDocs(random, true); int num = atLeast(100); for (int i = 0; i < num; i++) { Document doc = docs.nextDoc(); int boost = writeNorms ? 1 + random().nextInt(255) : 0; - Field f = new Field(byteTestField, "" + boost, - TextField.TYPE_STORED); + Field f = new TextField(byteTestField, "" + boost, Field.Store.YES); f.setBoost(boost); doc.add(f); writer.addDocument(doc); @@ -247,6 +247,5 @@ public class TestNorms extends LuceneTestCase { norm.setByte((byte) (0xFF & boost)); } } - } - + } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOmitNorms.java b/lucene/core/src/test/org/apache/lucene/index/TestOmitNorms.java index 0a0fd40..d980984 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestOmitNorms.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestOmitNorms.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -39,11 +39,11 @@ public class TestOmitNorms extends LuceneTestCase { Document d = new Document(); // this field will have norms - Field f1 = newField("f1", "This field has norms", TextField.TYPE_UNSTORED); + Field f1 = newTextField("f1", "This field has norms", Field.Store.NO); d.add(f1); // this field will NOT have norms - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setOmitNorms(true); Field f2 = newField("f2", "This field has NO norms in all docs", customType); d.add(f2); @@ -57,7 +57,7 @@ public class TestOmitNorms extends LuceneTestCase { // Reverse d.add(newField("f1", "This field has norms", customType)); - d.add(newField("f2", "This field has NO norms in all docs", TextField.TYPE_UNSTORED)); + d.add(newTextField("f2", "This field has NO norms in all docs", Field.Store.NO)); writer.addDocument(d); @@ -66,7 +66,7 @@ public class TestOmitNorms extends LuceneTestCase { // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); + SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(ram)); FieldInfos fi = reader.getFieldInfos(); assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f1").omitsNorms()); assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f2").omitsNorms()); @@ -89,11 +89,11 @@ public class TestOmitNorms extends LuceneTestCase { Document d = new Document(); // this field will have norms - Field f1 = newField("f1", "This field has norms", TextField.TYPE_UNSTORED); + Field f1 = newTextField("f1", "This field has norms", Field.Store.NO); d.add(f1); // this field will NOT have norms - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setOmitNorms(true); Field f2 = newField("f2", "This field has NO norms in all docs", customType); d.add(f2); @@ -109,7 +109,7 @@ public class TestOmitNorms extends LuceneTestCase { // Reverese d.add(newField("f1", "This field has norms", customType)); - d.add(newField("f2", "This field has NO norms in all docs", TextField.TYPE_UNSTORED)); + d.add(newTextField("f2", "This field has NO norms in all docs", Field.Store.NO)); for (int i = 0; i < 30; i++) { writer.addDocument(d); @@ -120,7 +120,7 @@ public class TestOmitNorms extends LuceneTestCase { // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); + SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(ram)); FieldInfos fi = reader.getFieldInfos(); assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f1").omitsNorms()); assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f2").omitsNorms()); @@ -144,12 +144,12 @@ public class TestOmitNorms extends LuceneTestCase { Document d = new Document(); // this field will have norms - Field f1 = newField("f1", "This field has norms", TextField.TYPE_UNSTORED); + Field f1 = newTextField("f1", "This field has norms", Field.Store.NO); d.add(f1); // this field will NOT have norms - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setOmitNorms(true); Field f2 = newField("f2", "This field has NO norms in all docs", customType); d.add(f2); @@ -168,7 +168,7 @@ public class TestOmitNorms extends LuceneTestCase { // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); + SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(ram)); FieldInfos fi = reader.getFieldInfos(); assertTrue("OmitNorms field bit should not be set.", !fi.fieldInfo("f1").omitsNorms()); assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f2").omitsNorms()); @@ -196,7 +196,7 @@ public class TestOmitNorms extends LuceneTestCase { lmp.setUseCompoundFile(false); Document d = new Document(); - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setOmitNorms(true); Field f1 = newField("f1", "This field has no norms", customType); d.add(f1); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java b/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java index 6cc2e92..84a01ca 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -40,7 +40,7 @@ public class TestOmitPositions extends LuceneTestCase { Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir); Document doc = new Document(); - FieldType ft = new FieldType(TextField.TYPE_UNSTORED); + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS); Field f = newField("foo", "this is a test test", ft); doc.add(f); @@ -71,7 +71,7 @@ public class TestOmitPositions extends LuceneTestCase { Document d = new Document(); // f1,f2,f3: docs only - FieldType ft = new FieldType(TextField.TYPE_UNSTORED); + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); ft.setIndexOptions(IndexOptions.DOCS_ONLY); Field f1 = newField("f1", "This field has docs only", ft); @@ -83,7 +83,7 @@ public class TestOmitPositions extends LuceneTestCase { Field f3 = newField("f3", "This field has docs only", ft); d.add(f3); - FieldType ft2 = new FieldType(TextField.TYPE_UNSTORED); + FieldType ft2 = new FieldType(TextField.TYPE_NOT_STORED); ft2.setIndexOptions(IndexOptions.DOCS_AND_FREQS); // f4,f5,f6 docs and freqs @@ -96,7 +96,7 @@ public class TestOmitPositions extends LuceneTestCase { Field f6 = newField("f6", "This field has docs and freqs", ft2); d.add(f6); - FieldType ft3 = new FieldType(TextField.TYPE_UNSTORED); + FieldType ft3 = new FieldType(TextField.TYPE_NOT_STORED); ft3.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); // f7,f8,f9 docs/freqs/positions @@ -153,7 +153,7 @@ public class TestOmitPositions extends LuceneTestCase { // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); + SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(ram)); FieldInfos fi = reader.getFieldInfos(); // docs + docs = docs assertEquals(IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").getIndexOptions()); @@ -197,7 +197,7 @@ public class TestOmitPositions extends LuceneTestCase { lmp.setUseCompoundFile(false); Document d = new Document(); - FieldType ft = new FieldType(TextField.TYPE_UNSTORED); + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS); Field f1 = newField("f1", "This field has term freqs", ft); d.add(f1); @@ -211,7 +211,7 @@ public class TestOmitPositions extends LuceneTestCase { // now add some documents with positions, and check there is no prox after optimization d = new Document(); - f1 = newField("f1", "This field has positions", TextField.TYPE_UNSTORED); + f1 = newTextField("f1", "This field has positions", Field.Store.NO); d.add(f1); for(int i=0;i<30;i++) diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java b/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java index 0e71ce9..efceb1e 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -30,7 +30,6 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; import org.apache.lucene.search.*; import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.search.similarities.TFIDFSimilarity; import org.apache.lucene.store.Directory; @@ -50,8 +49,8 @@ public class TestOmitTf extends LuceneTestCase { @Override public float scorePayload(int doc, int start, int end, BytesRef payload) { return 1.0f; } } - private static final FieldType omitType = new FieldType(TextField.TYPE_UNSTORED); - private static final FieldType normalType = new FieldType(TextField.TYPE_UNSTORED); + private static final FieldType omitType = new FieldType(TextField.TYPE_NOT_STORED); + private static final FieldType normalType = new FieldType(TextField.TYPE_NOT_STORED); static { omitType.setIndexOptions(IndexOptions.DOCS_ONLY); @@ -93,7 +92,7 @@ public class TestOmitTf extends LuceneTestCase { // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); + SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(ram)); FieldInfos fi = reader.getFieldInfos(); assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").getIndexOptions()); assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").getIndexOptions()); @@ -145,7 +144,7 @@ public class TestOmitTf extends LuceneTestCase { // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); + SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(ram)); FieldInfos fi = reader.getFieldInfos(); assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").getIndexOptions()); assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").getIndexOptions()); @@ -188,7 +187,7 @@ public class TestOmitTf extends LuceneTestCase { // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); + SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(ram)); FieldInfos fi = reader.getFieldInfos(); assertEquals("OmitTermFreqAndPositions field bit should not be set.", IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, fi.fieldInfo("f1").getIndexOptions()); assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").getIndexOptions()); @@ -229,7 +228,7 @@ public class TestOmitTf extends LuceneTestCase { // now add some documents with positions, and check // there is no prox after full merge d = new Document(); - f1 = newField("f1", "This field has positions", TextField.TYPE_UNSTORED); + f1 = newTextField("f1", "This field has positions", Field.Store.NO); d.add(f1); for(int i=0;i<30;i++) @@ -279,7 +278,7 @@ public class TestOmitTf extends LuceneTestCase { /* * Verify the index */ - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); searcher.setSimilarity(new SimpleSimilarity()); @@ -430,7 +429,7 @@ public class TestOmitTf extends LuceneTestCase { RandomIndexWriter iw = new RandomIndexWriter(random(), dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); Document doc = new Document(); - FieldType ft = new FieldType(TextField.TYPE_UNSTORED); + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); ft.setIndexOptions(IndexOptions.DOCS_ONLY); ft.freeze(); Field f = newField("foo", "bar", ft); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestParallelAtomicReader.java b/lucene/core/src/test/org/apache/lucene/index/TestParallelAtomicReader.java index 981bdfb..c7760f6 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestParallelAtomicReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestParallelAtomicReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,7 +22,7 @@ import java.util.Random; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.*; import org.apache.lucene.store.Directory; @@ -122,7 +122,7 @@ public class TestParallelAtomicReader extends LuceneTestCase { IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random()))); Document d3 = new Document(); - d3.add(newField("f3", "v1", TextField.TYPE_STORED)); + d3.add(newTextField("f3", "v1", Field.Store.YES)); w2.addDocument(d3); w2.close(); @@ -238,16 +238,16 @@ public class TestParallelAtomicReader extends LuceneTestCase { dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); Document d1 = new Document(); - d1.add(newField("f1", "v1", TextField.TYPE_STORED)); - d1.add(newField("f2", "v1", TextField.TYPE_STORED)); - d1.add(newField("f3", "v1", TextField.TYPE_STORED)); - d1.add(newField("f4", "v1", TextField.TYPE_STORED)); + d1.add(newTextField("f1", "v1", Field.Store.YES)); + d1.add(newTextField("f2", "v1", Field.Store.YES)); + d1.add(newTextField("f3", "v1", Field.Store.YES)); + d1.add(newTextField("f4", "v1", Field.Store.YES)); w.addDocument(d1); Document d2 = new Document(); - d2.add(newField("f1", "v2", TextField.TYPE_STORED)); - d2.add(newField("f2", "v2", TextField.TYPE_STORED)); - d2.add(newField("f3", "v2", TextField.TYPE_STORED)); - d2.add(newField("f4", "v2", TextField.TYPE_STORED)); + d2.add(newTextField("f1", "v2", Field.Store.YES)); + d2.add(newTextField("f2", "v2", Field.Store.YES)); + d2.add(newTextField("f3", "v2", Field.Store.YES)); + d2.add(newTextField("f4", "v2", Field.Store.YES)); w.addDocument(d2); w.close(); @@ -269,12 +269,12 @@ public class TestParallelAtomicReader extends LuceneTestCase { Directory dir1 = newDirectory(); IndexWriter w1 = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); Document d1 = new Document(); - d1.add(newField("f1", "v1", TextField.TYPE_STORED)); - d1.add(newField("f2", "v1", TextField.TYPE_STORED)); + d1.add(newTextField("f1", "v1", Field.Store.YES)); + d1.add(newTextField("f2", "v1", Field.Store.YES)); w1.addDocument(d1); Document d2 = new Document(); - d2.add(newField("f1", "v2", TextField.TYPE_STORED)); - d2.add(newField("f2", "v2", TextField.TYPE_STORED)); + d2.add(newTextField("f1", "v2", Field.Store.YES)); + d2.add(newTextField("f2", "v2", Field.Store.YES)); w1.addDocument(d2); w1.close(); return dir1; @@ -284,12 +284,12 @@ public class TestParallelAtomicReader extends LuceneTestCase { Directory dir2 = newDirectory(); IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); Document d3 = new Document(); - d3.add(newField("f3", "v1", TextField.TYPE_STORED)); - d3.add(newField("f4", "v1", TextField.TYPE_STORED)); + d3.add(newTextField("f3", "v1", Field.Store.YES)); + d3.add(newTextField("f4", "v1", Field.Store.YES)); w2.addDocument(d3); Document d4 = new Document(); - d4.add(newField("f3", "v2", TextField.TYPE_STORED)); - d4.add(newField("f4", "v2", TextField.TYPE_STORED)); + d4.add(newTextField("f3", "v2", Field.Store.YES)); + d4.add(newTextField("f4", "v2", Field.Store.YES)); w2.addDocument(d4); w2.close(); return dir2; diff --git a/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java b/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java index 8d49ec2..68b4179 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,7 +22,7 @@ import java.util.Random; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.*; import org.apache.lucene.store.Directory; @@ -123,7 +123,7 @@ public class TestParallelCompositeReader extends LuceneTestCase { IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random()))); Document d3 = new Document(); - d3.add(newField("f3", "v1", TextField.TYPE_STORED)); + d3.add(newTextField("f3", "v1", Field.Store.YES)); w2.addDocument(d3); w2.close(); @@ -297,28 +297,28 @@ public class TestParallelCompositeReader extends LuceneTestCase { dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); Document d1 = new Document(); - d1.add(newField("f1", "v1", TextField.TYPE_STORED)); - d1.add(newField("f2", "v1", TextField.TYPE_STORED)); - d1.add(newField("f3", "v1", TextField.TYPE_STORED)); - d1.add(newField("f4", "v1", TextField.TYPE_STORED)); + d1.add(newTextField("f1", "v1", Field.Store.YES)); + d1.add(newTextField("f2", "v1", Field.Store.YES)); + d1.add(newTextField("f3", "v1", Field.Store.YES)); + d1.add(newTextField("f4", "v1", Field.Store.YES)); w.addDocument(d1); Document d2 = new Document(); - d2.add(newField("f1", "v2", TextField.TYPE_STORED)); - d2.add(newField("f2", "v2", TextField.TYPE_STORED)); - d2.add(newField("f3", "v2", TextField.TYPE_STORED)); - d2.add(newField("f4", "v2", TextField.TYPE_STORED)); + d2.add(newTextField("f1", "v2", Field.Store.YES)); + d2.add(newTextField("f2", "v2", Field.Store.YES)); + d2.add(newTextField("f3", "v2", Field.Store.YES)); + d2.add(newTextField("f4", "v2", Field.Store.YES)); w.addDocument(d2); Document d3 = new Document(); - d3.add(newField("f1", "v3", TextField.TYPE_STORED)); - d3.add(newField("f2", "v3", TextField.TYPE_STORED)); - d3.add(newField("f3", "v3", TextField.TYPE_STORED)); - d3.add(newField("f4", "v3", TextField.TYPE_STORED)); + d3.add(newTextField("f1", "v3", Field.Store.YES)); + d3.add(newTextField("f2", "v3", Field.Store.YES)); + d3.add(newTextField("f3", "v3", Field.Store.YES)); + d3.add(newTextField("f4", "v3", Field.Store.YES)); w.addDocument(d3); Document d4 = new Document(); - d4.add(newField("f1", "v4", TextField.TYPE_STORED)); - d4.add(newField("f2", "v4", TextField.TYPE_STORED)); - d4.add(newField("f3", "v4", TextField.TYPE_STORED)); - d4.add(newField("f4", "v4", TextField.TYPE_STORED)); + d4.add(newTextField("f1", "v4", Field.Store.YES)); + d4.add(newTextField("f2", "v4", Field.Store.YES)); + d4.add(newTextField("f3", "v4", Field.Store.YES)); + d4.add(newTextField("f4", "v4", Field.Store.YES)); w.addDocument(d4); w.close(); @@ -339,13 +339,13 @@ public class TestParallelCompositeReader extends LuceneTestCase { if (compositeComposite) { rd1 = new MultiReader(DirectoryReader.open(dir1), DirectoryReader.open(dir1)); rd2 = new MultiReader(DirectoryReader.open(dir2), DirectoryReader.open(dir2)); - assertEquals(2, rd1.getSequentialSubReaders().length); - assertEquals(2, rd2.getSequentialSubReaders().length); + assertEquals(2, rd1.getSequentialSubReaders().size()); + assertEquals(2, rd2.getSequentialSubReaders().size()); } else { rd1 = DirectoryReader.open(dir1); rd2 = DirectoryReader.open(dir2); - assertEquals(3, rd1.getSequentialSubReaders().length); - assertEquals(3, rd2.getSequentialSubReaders().length); + assertEquals(3, rd1.getSequentialSubReaders().size()); + assertEquals(3, rd2.getSequentialSubReaders().size()); } ParallelCompositeReader pr = new ParallelCompositeReader(rd1, rd2); return newSearcher(pr); @@ -357,22 +357,22 @@ public class TestParallelCompositeReader extends LuceneTestCase { IndexWriter w1 = new IndexWriter(dir1, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES)); Document d1 = new Document(); - d1.add(newField("f1", "v1", TextField.TYPE_STORED)); - d1.add(newField("f2", "v1", TextField.TYPE_STORED)); + d1.add(newTextField("f1", "v1", Field.Store.YES)); + d1.add(newTextField("f2", "v1", Field.Store.YES)); w1.addDocument(d1); w1.commit(); Document d2 = new Document(); - d2.add(newField("f1", "v2", TextField.TYPE_STORED)); - d2.add(newField("f2", "v2", TextField.TYPE_STORED)); + d2.add(newTextField("f1", "v2", Field.Store.YES)); + d2.add(newTextField("f2", "v2", Field.Store.YES)); w1.addDocument(d2); Document d3 = new Document(); - d3.add(newField("f1", "v3", TextField.TYPE_STORED)); - d3.add(newField("f2", "v3", TextField.TYPE_STORED)); + d3.add(newTextField("f1", "v3", Field.Store.YES)); + d3.add(newTextField("f2", "v3", Field.Store.YES)); w1.addDocument(d3); w1.commit(); Document d4 = new Document(); - d4.add(newField("f1", "v4", TextField.TYPE_STORED)); - d4.add(newField("f2", "v4", TextField.TYPE_STORED)); + d4.add(newTextField("f1", "v4", Field.Store.YES)); + d4.add(newTextField("f2", "v4", Field.Store.YES)); w1.addDocument(d4); w1.close(); return dir1; @@ -384,22 +384,22 @@ public class TestParallelCompositeReader extends LuceneTestCase { IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES)); Document d1 = new Document(); - d1.add(newField("f3", "v1", TextField.TYPE_STORED)); - d1.add(newField("f4", "v1", TextField.TYPE_STORED)); + d1.add(newTextField("f3", "v1", Field.Store.YES)); + d1.add(newTextField("f4", "v1", Field.Store.YES)); w2.addDocument(d1); w2.commit(); Document d2 = new Document(); - d2.add(newField("f3", "v2", TextField.TYPE_STORED)); - d2.add(newField("f4", "v2", TextField.TYPE_STORED)); + d2.add(newTextField("f3", "v2", Field.Store.YES)); + d2.add(newTextField("f4", "v2", Field.Store.YES)); w2.addDocument(d2); Document d3 = new Document(); - d3.add(newField("f3", "v3", TextField.TYPE_STORED)); - d3.add(newField("f4", "v3", TextField.TYPE_STORED)); + d3.add(newTextField("f3", "v3", Field.Store.YES)); + d3.add(newTextField("f4", "v3", Field.Store.YES)); w2.addDocument(d3); w2.commit(); Document d4 = new Document(); - d4.add(newField("f3", "v4", TextField.TYPE_STORED)); - d4.add(newField("f4", "v4", TextField.TYPE_STORED)); + d4.add(newTextField("f3", "v4", Field.Store.YES)); + d4.add(newTextField("f4", "v4", Field.Store.YES)); w2.addDocument(d4); w2.close(); return dir2; @@ -411,22 +411,22 @@ public class TestParallelCompositeReader extends LuceneTestCase { IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES)); Document d1 = new Document(); - d1.add(newField("f3", "v1", TextField.TYPE_STORED)); - d1.add(newField("f4", "v1", TextField.TYPE_STORED)); + d1.add(newTextField("f3", "v1", Field.Store.YES)); + d1.add(newTextField("f4", "v1", Field.Store.YES)); w2.addDocument(d1); w2.commit(); Document d2 = new Document(); - d2.add(newField("f3", "v2", TextField.TYPE_STORED)); - d2.add(newField("f4", "v2", TextField.TYPE_STORED)); + d2.add(newTextField("f3", "v2", Field.Store.YES)); + d2.add(newTextField("f4", "v2", Field.Store.YES)); w2.addDocument(d2); w2.commit(); Document d3 = new Document(); - d3.add(newField("f3", "v3", TextField.TYPE_STORED)); - d3.add(newField("f4", "v3", TextField.TYPE_STORED)); + d3.add(newTextField("f3", "v3", Field.Store.YES)); + d3.add(newTextField("f4", "v3", Field.Store.YES)); w2.addDocument(d3); Document d4 = new Document(); - d4.add(newField("f3", "v4", TextField.TYPE_STORED)); - d4.add(newField("f4", "v4", TextField.TYPE_STORED)); + d4.add(newTextField("f3", "v4", Field.Store.YES)); + d4.add(newTextField("f4", "v4", Field.Store.YES)); w2.addDocument(d4); w2.close(); return dir2; diff --git a/lucene/core/src/test/org/apache/lucene/index/TestParallelReaderEmptyIndex.java b/lucene/core/src/test/org/apache/lucene/index/TestParallelReaderEmptyIndex.java index 1f54d79..aa8f6aa 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestParallelReaderEmptyIndex.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestParallelReaderEmptyIndex.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -27,7 +27,6 @@ import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util._TestUtil; /** * Some tests for {@link ParallelAtomicReader}s with empty indexes @@ -94,14 +93,14 @@ public class TestParallelReaderEmptyIndex extends LuceneTestCase { } IndexWriter iw = new IndexWriter(rd1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random()))); Document doc = new Document(); - Field idField = newField("id", "", TextField.TYPE_UNSTORED); + Field idField = newTextField("id", "", Field.Store.NO); doc.add(idField); - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setStoreTermVectors(true); doc.add(newField("test", "", customType)); idField.setStringValue("1"); iw.addDocument(doc); - doc.add(newField("test", "", TextField.TYPE_UNSTORED)); + doc.add(newTextField("test", "", Field.Store.NO)); idField.setStringValue("2"); iw.addDocument(doc); iw.close(); @@ -115,7 +114,7 @@ public class TestParallelReaderEmptyIndex extends LuceneTestCase { writer.deleteDocuments(new Term("id", "1")); writer.close(); - IndexReader ir = IndexReader.open(rd1); + IndexReader ir = DirectoryReader.open(rd1); assertEquals(2, ir.maxDoc()); assertEquals(1, ir.numDocs()); ir.close(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java b/lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java index 631fbed..b4013b1 100755 --- a/lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,7 +21,7 @@ import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; @@ -44,8 +44,8 @@ public class TestParallelTermEnum extends LuceneTestCase { TEST_VERSION_CURRENT, new MockAnalyzer(random()))); doc = new Document(); - doc.add(newField("field1", "the quick brown fox jumps", TextField.TYPE_STORED)); - doc.add(newField("field2", "the quick brown fox jumps", TextField.TYPE_STORED)); + doc.add(newTextField("field1", "the quick brown fox jumps", Field.Store.YES)); + doc.add(newTextField("field2", "the quick brown fox jumps", Field.Store.YES)); iw1.addDocument(doc); iw1.close(); @@ -54,8 +54,8 @@ public class TestParallelTermEnum extends LuceneTestCase { TEST_VERSION_CURRENT, new MockAnalyzer(random()))); doc = new Document(); - doc.add(newField("field1", "the fox jumps over the lazy dog", TextField.TYPE_STORED)); - doc.add(newField("field3", "the fox jumps over the lazy dog", TextField.TYPE_STORED)); + doc.add(newTextField("field1", "the fox jumps over the lazy dog", Field.Store.YES)); + doc.add(newTextField("field3", "the fox jumps over the lazy dog", Field.Store.YES)); iw2.addDocument(doc); iw2.close(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java b/lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java index 8336bdc..ecf4703 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -28,6 +28,7 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; import org.apache.lucene.index.PayloadProcessorProvider.ReaderPayloadProcessor; @@ -49,7 +50,7 @@ public class TestPayloadProcessorProvider extends LuceneTestCase { } @Override - public ReaderPayloadProcessor getReaderProcessor(AtomicReader reader) throws IOException { + public ReaderPayloadProcessor getReaderProcessor(AtomicReader reader) { if (reader instanceof SegmentReader) { return processors.get(((SegmentReader) reader).directory()); } else { @@ -62,7 +63,7 @@ public class TestPayloadProcessorProvider extends LuceneTestCase { private static final class PerTermPayloadProcessor extends ReaderPayloadProcessor { @Override - public PayloadProcessor getProcessor(String field, BytesRef text) throws IOException { + public PayloadProcessor getProcessor(String field, BytesRef text) { // don't process payloads of terms other than "p:p1" if (!field.equals("p") || !text.bytesEquals(new BytesRef("p1"))) { return null; @@ -78,7 +79,7 @@ public class TestPayloadProcessorProvider extends LuceneTestCase { private static final class DeletePayloadProcessor extends PayloadProcessor { @Override - public void processPayload(BytesRef payload) throws IOException { + public void processPayload(BytesRef payload) { payload.length = 0; } @@ -97,14 +98,14 @@ public class TestPayloadProcessorProvider extends LuceneTestCase { } @Override - public boolean incrementToken() throws IOException { + public boolean incrementToken() { if (called) { return false; } called = true; byte[] p = new byte[] { 1 }; - payload.setPayload(new Payload(p)); + payload.setPayload(new BytesRef(p)); term.append(t); return true; } @@ -138,12 +139,12 @@ public class TestPayloadProcessorProvider extends LuceneTestCase { ); TokenStream payloadTS1 = new PayloadTokenStream("p1"); TokenStream payloadTS2 = new PayloadTokenStream("p2"); - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setOmitNorms(true); for (int i = 0; i < NUM_DOCS; i++) { Document doc = new Document(); doc.add(newField("id", "doc" + i, customType)); - doc.add(newField("content", "doc content " + i, TextField.TYPE_UNSTORED)); + doc.add(newTextField("content", "doc content " + i, Field.Store.NO)); doc.add(new TextField("p", payloadTS1)); doc.add(new TextField("p", payloadTS2)); writer.addDocument(doc); @@ -156,7 +157,7 @@ public class TestPayloadProcessorProvider extends LuceneTestCase { private void verifyPayloadExists(Directory dir, String field, BytesRef text, int numExpected) throws IOException { - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); try { int numPayloads = 0; DocsAndPositionsEnum tpe = MultiFields.getTermPositionsEnum(reader, null, field, text, false); @@ -198,7 +199,7 @@ public class TestPayloadProcessorProvider extends LuceneTestCase { IndexReader[] readers = new IndexReader[dirs.length]; for (int i = 0; i < readers.length; i++) { - readers[i] = IndexReader.open(dirs[i]); + readers[i] = DirectoryReader.open(dirs[i]); } try { writer.addIndexes(readers); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java b/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java index 6b697fb..f88dd03 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -44,46 +44,13 @@ public class TestPayloads extends LuceneTestCase { // Simple tests to test the Payload class public void testPayload() throws Exception { byte[] testData = "This is a test!".getBytes(); - Payload payload = new Payload(testData); - assertEquals("Wrong payload length.", testData.length, payload.length()); - - // test copyTo() - byte[] target = new byte[testData.length - 1]; - try { - payload.copyTo(target, 0); - fail("Expected exception not thrown"); - } catch (Exception expected) { - // expected exception - } - - target = new byte[testData.length + 3]; - payload.copyTo(target, 3); - - for (int i = 0; i < testData.length; i++) { - assertEquals(testData[i], target[i + 3]); - } - - - // test toByteArray() - target = payload.toByteArray(); - assertByteArrayEquals(testData, target); - - // test byteAt() - for (int i = 0; i < testData.length; i++) { - assertEquals(payload.byteAt(i), testData[i]); - } - - try { - payload.byteAt(testData.length + 1); - fail("Expected exception not thrown"); - } catch (Exception expected) { - // expected exception - } + BytesRef payload = new BytesRef(testData); + assertEquals("Wrong payload length.", testData.length, payload.length); - Payload clone = payload.clone(); - assertEquals(payload.length(), clone.length()); - for (int i = 0; i < payload.length(); i++) { - assertEquals(payload.byteAt(i), clone.byteAt(i)); + BytesRef clone = payload.clone(); + assertEquals(payload.length, clone.length); + for (int i = 0; i < payload.length; i++) { + assertEquals(payload.bytes[i + payload.offset], clone.bytes[i + clone.offset]); } } @@ -96,22 +63,22 @@ public class TestPayloads extends LuceneTestCase { IndexWriter writer = new IndexWriter(ram, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)); Document d = new Document(); // this field won't have any payloads - d.add(newField("f1", "This field has no payloads", TextField.TYPE_UNSTORED)); + d.add(newTextField("f1", "This field has no payloads", Field.Store.NO)); // this field will have payloads in all docs, however not for all term positions, // so this field is used to check if the DocumentWriter correctly enables the payloads bit // even if only some term positions have payloads - d.add(newField("f2", "This field has payloads in all docs", TextField.TYPE_UNSTORED)); - d.add(newField("f2", "This field has payloads in all docs NO PAYLOAD", TextField.TYPE_UNSTORED)); + d.add(newTextField("f2", "This field has payloads in all docs", Field.Store.NO)); + d.add(newTextField("f2", "This field has payloads in all docs NO PAYLOAD", Field.Store.NO)); // this field is used to verify if the SegmentMerger enables payloads for a field if it has payloads // enabled in only some documents - d.add(newField("f3", "This field has payloads in some docs", TextField.TYPE_UNSTORED)); + d.add(newTextField("f3", "This field has payloads in some docs", Field.Store.NO)); // only add payload data for field f2 analyzer.setPayloadData("f2", "somedata".getBytes(), 0, 1); writer.addDocument(d); // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); + SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(ram)); FieldInfos fi = reader.getFieldInfos(); assertFalse("Payload field bit should not be set.", fi.fieldInfo("f1").hasPayloads()); assertTrue("Payload field bit should be set.", fi.fieldInfo("f2").hasPayloads()); @@ -124,10 +91,10 @@ public class TestPayloads extends LuceneTestCase { writer = new IndexWriter(ram, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setOpenMode(OpenMode.CREATE)); d = new Document(); - d.add(newField("f1", "This field has no payloads", TextField.TYPE_UNSTORED)); - d.add(newField("f2", "This field has payloads in all docs", TextField.TYPE_UNSTORED)); - d.add(newField("f2", "This field has payloads in all docs", TextField.TYPE_UNSTORED)); - d.add(newField("f3", "This field has payloads in some docs", TextField.TYPE_UNSTORED)); + d.add(newTextField("f1", "This field has no payloads", Field.Store.NO)); + d.add(newTextField("f2", "This field has payloads in all docs", Field.Store.NO)); + d.add(newTextField("f2", "This field has payloads in all docs", Field.Store.NO)); + d.add(newTextField("f3", "This field has payloads in some docs", Field.Store.NO)); // add payload data for field f2 and f3 analyzer.setPayloadData("f2", "somedata".getBytes(), 0, 1); analyzer.setPayloadData("f3", "somedata".getBytes(), 0, 3); @@ -138,7 +105,7 @@ public class TestPayloads extends LuceneTestCase { // flush writer.close(); - reader = getOnlySegmentReader(IndexReader.open(ram)); + reader = getOnlySegmentReader(DirectoryReader.open(ram)); fi = reader.getFieldInfos(); assertFalse("Payload field bit should not be set.", fi.fieldInfo("f1").hasPayloads()); assertTrue("Payload field bit should be set.", fi.fieldInfo("f2").hasPayloads()); @@ -184,7 +151,7 @@ public class TestPayloads extends LuceneTestCase { byte[] payloadData = generateRandomData(payloadDataLength); Document d = new Document(); - d.add(newField(fieldName, content, TextField.TYPE_UNSTORED)); + d.add(newTextField(fieldName, content, Field.Store.NO)); // add the same document multiple times to have the same payload lengths for all // occurrences within two consecutive skip intervals int offset = 0; @@ -213,7 +180,7 @@ public class TestPayloads extends LuceneTestCase { * Verify the index * first we test if all payloads are stored correctly */ - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); byte[] verifyPayloadData = new byte[payloadDataLength]; offset = 0; @@ -318,7 +285,7 @@ public class TestPayloads extends LuceneTestCase { String singleTerm = "lucene"; d = new Document(); - d.add(newField(fieldName, singleTerm, TextField.TYPE_UNSTORED)); + d.add(newTextField(fieldName, singleTerm, Field.Store.NO)); // add a payload whose length is greater than the buffer size of BufferedIndexOutput payloadData = generateRandomData(2000); analyzer.setPayloadData(fieldName, payloadData, 100, 1500); @@ -329,7 +296,7 @@ public class TestPayloads extends LuceneTestCase { // flush writer.close(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); tp = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), fieldName, @@ -478,9 +445,8 @@ public class TestPayloads extends LuceneTestCase { // Some values of the same field are to have payloads and others not if (offset + length <= data.length && !termAttribute.toString().endsWith("NO PAYLOAD")) { - Payload p = new Payload(); + BytesRef p = new BytesRef(data, offset, length); payloadAtt.setPayload(p); - p.setData(data, offset, length); offset += length; } else { payloadAtt.setPayload(null); @@ -530,7 +496,7 @@ public class TestPayloads extends LuceneTestCase { ingesters[i].join(); } writer.close(); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); TermsEnum terms = MultiFields.getFields(reader).terms(field).iterator(null); Bits liveDocs = MultiFields.getLiveDocs(reader); DocsAndPositionsEnum tp = null; @@ -576,7 +542,7 @@ public class TestPayloads extends LuceneTestCase { first = false; clearAttributes(); termAtt.append(term); - payloadAtt.setPayload(new Payload(payload)); + payloadAtt.setPayload(new BytesRef(payload)); return true; } @@ -615,14 +581,14 @@ public class TestPayloads extends LuceneTestCase { RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true)); Document doc = new Document(); - doc.add(new Field("hasMaybepayload", "here we go", TextField.TYPE_STORED)); + doc.add(new TextField("hasMaybepayload", "here we go", Field.Store.YES)); writer.addDocument(doc); writer.close(); writer = new RandomIndexWriter(random(), dir, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true)); doc = new Document(); - doc.add(new Field("hasMaybepayload2", "here we go", TextField.TYPE_STORED)); + doc.add(new TextField("hasMaybepayload2", "here we go", Field.Store.YES)); writer.addDocument(doc); writer.addDocument(doc); writer.forceMerge(1); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java b/lucene/core/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java index dd5bd62..375bd57 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -86,6 +86,7 @@ public class TestPerSegmentDeletes extends LuceneTestCase { // which should apply the delete id:2 writer.deleteDocuments(new Term("id", "2")); writer.flush(false, false); + fsmp = (RangeMergePolicy) writer.getConfig().getMergePolicy(); fsmp.doMerge = true; fsmp.start = 0; fsmp.length = 2; @@ -257,7 +258,7 @@ public class TestPerSegmentDeletes extends LuceneTestCase { @Override public MergeSpecification findMerges(SegmentInfos segmentInfos) - throws CorruptIndexException, IOException { + throws IOException { MergeSpecification ms = new MergeSpecification(); if (doMerge) { OneMerge om = new OneMerge(segmentInfos.asList().subList(start, start + length)); @@ -271,13 +272,13 @@ public class TestPerSegmentDeletes extends LuceneTestCase { @Override public MergeSpecification findForcedMerges(SegmentInfos segmentInfos, int maxSegmentCount, Map segmentsToMerge) - throws CorruptIndexException, IOException { + throws IOException { return null; } @Override public MergeSpecification findForcedDeletesMerges( - SegmentInfos segmentInfos) throws CorruptIndexException, IOException { + SegmentInfos segmentInfos) throws IOException { return null; } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java index db2f256..27e99b2 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java b/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java index 5a6d863..987594c 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,6 +17,7 @@ package org.apache.lucene.index; * limitations under the License. */ +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -27,51 +28,34 @@ import org.apache.lucene.analysis.CannedTokenStream; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockPayloadAnalyzer; import org.apache.lucene.analysis.Token; -import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat; -import org.apache.lucene.codecs.memory.MemoryPostingsFormat; -import org.apache.lucene.codecs.nestedpulsing.NestedPulsingPostingsFormat; -import org.apache.lucene.codecs.pulsing.Pulsing40PostingsFormat; +import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.IntField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldCache; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.English; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.apache.lucene.util._TestUtil; // TODO: we really need to test indexingoffsets, but then getting only docs / docs + freqs. // not all codecs store prx separate... +// TODO: fix sep codec to index offsets so we can greatly reduce this list! +@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom"}) public class TestPostingsOffsets extends LuceneTestCase { IndexWriterConfig iwc; public void setUp() throws Exception { super.setUp(); - - // Currently only SimpleText and Lucene40 can index offsets into postings: - String codecName = Codec.getDefault().getName(); - assumeTrue("Codec does not support offsets: " + codecName, - codecName.equals("SimpleText") || - codecName.equals("Lucene40")); - iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); - - if (codecName.equals("Lucene40")) { - // Sep etc are not implemented - switch(random().nextInt(4)) { - case 0: iwc.setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat())); break; - case 1: iwc.setCodec(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat())); break; - case 2: iwc.setCodec(_TestUtil.alwaysPostingsFormat( - new Pulsing40PostingsFormat(_TestUtil.nextInt(random(), 1, 3)))); break; - case 3: iwc.setCodec(_TestUtil.alwaysPostingsFormat(new NestedPulsingPostingsFormat())); break; - } - } } public void testBasic() throws Exception { @@ -80,7 +64,7 @@ public class TestPostingsOffsets extends LuceneTestCase { RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); Document doc = new Document(); - FieldType ft = new FieldType(TextField.TYPE_UNSTORED); + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); ft.setIndexOptions(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); if (random().nextBoolean()) { ft.setStoreTermVectors(true); @@ -145,16 +129,6 @@ public class TestPostingsOffsets extends LuceneTestCase { Directory dir = newDirectory(); Analyzer analyzer = withPayloads ? new MockPayloadAnalyzer() : new MockAnalyzer(random()); iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer); - if (Codec.getDefault().getName().equals("Lucene40")) { - // sep etc are not implemented - switch(random().nextInt(4)) { - case 0: iwc.setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat())); break; - case 1: iwc.setCodec(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat())); break; - case 2: iwc.setCodec(_TestUtil.alwaysPostingsFormat( - new Pulsing40PostingsFormat(_TestUtil.nextInt(random(), 1, 3)))); break; - case 3: iwc.setCodec(_TestUtil.alwaysPostingsFormat(new NestedPulsingPostingsFormat())); break; - } - } iwc.setMergePolicy(newLogMergePolicy()); // will rely on docids a bit for skipping RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); @@ -171,7 +145,7 @@ public class TestPostingsOffsets extends LuceneTestCase { Document doc = new Document(); doc.add(new Field("numbers", English.intToEnglish(i), ft)); doc.add(new Field("oddeven", (i % 2) == 0 ? "even" : "odd", ft)); - doc.add(new StringField("id", "" + i)); + doc.add(new StringField("id", "" + i, Field.Store.NO)); w.addDocument(doc); } @@ -253,7 +227,7 @@ public class TestPostingsOffsets extends LuceneTestCase { final int numDocs = atLeast(20); //final int numDocs = atLeast(5); - FieldType ft = new FieldType(TextField.TYPE_UNSTORED); + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); // TODO: randomize what IndexOptions we use; also test // changing this up in one IW buffered segment...: @@ -266,7 +240,7 @@ public class TestPostingsOffsets extends LuceneTestCase { for(int docCount=0;docCount tokens = new ArrayList(); final int numTokens = atLeast(100); //final int numTokens = atLeast(20); @@ -376,6 +350,152 @@ public class TestPostingsOffsets extends LuceneTestCase { r.close(); dir.close(); } + + public void testWithUnindexedFields() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter riw = new RandomIndexWriter(random(), dir, iwc); + for (int i = 0; i < 100; i++) { + Document doc = new Document(); + // ensure at least one doc is indexed with offsets + if (i < 99 && random().nextInt(2) == 0) { + // stored only + FieldType ft = new FieldType(); + ft.setIndexed(false); + ft.setStored(true); + doc.add(new Field("foo", "boo!", ft)); + } else { + FieldType ft = new FieldType(TextField.TYPE_STORED); + ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); + if (random().nextBoolean()) { + // store some term vectors for the checkindex cross-check + ft.setStoreTermVectors(true); + ft.setStoreTermVectorPositions(true); + ft.setStoreTermVectorOffsets(true); + } + doc.add(new Field("foo", "bar", ft)); + } + riw.addDocument(doc); + } + CompositeReader ir = riw.getReader(); + SlowCompositeReaderWrapper slow = new SlowCompositeReaderWrapper(ir); + FieldInfos fis = slow.getFieldInfos(); + assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, fis.fieldInfo("foo").getIndexOptions()); + slow.close(); + ir.close(); + riw.close(); + dir.close(); + } + + public void testAddFieldTwice() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + FieldType customType3 = new FieldType(TextField.TYPE_STORED); + customType3.setStoreTermVectors(true); + customType3.setStoreTermVectorPositions(true); + customType3.setStoreTermVectorOffsets(true); + customType3.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); + doc.add(new Field("content3", "here is more content with aaa aaa aaa", customType3)); + doc.add(new Field("content3", "here is more content with aaa aaa aaa", customType3)); + iw.addDocument(doc); + iw.close(); + dir.close(); // checkindex + } + + // NOTE: the next two tests aren't that good as we need an EvilToken... + public void testNegativeOffsets() throws Exception { + try { + checkTokens(new Token[] { + makeToken("foo", 1, -1, -1) + }); + fail(); + } catch (IllegalArgumentException expected) { + //expected + } + } + + public void testIllegalOffsets() throws Exception { + try { + checkTokens(new Token[] { + makeToken("foo", 1, 1, 0) + }); + fail(); + } catch (IllegalArgumentException expected) { + //expected + } + } + + public void testBackwardsOffsets() throws Exception { + try { + checkTokens(new Token[] { + makeToken("foo", 1, 0, 3), + makeToken("foo", 1, 4, 7), + makeToken("foo", 0, 3, 6) + }); + fail(); + } catch (IllegalArgumentException expected) { + // expected + } + } + + public void testStackedTokens() throws Exception { + checkTokens(new Token[] { + makeToken("foo", 1, 0, 3), + makeToken("foo", 0, 0, 3), + makeToken("foo", 0, 0, 3) + }); + } + + public void testLegalbutVeryLargeOffsets() throws Exception { + Directory dir = newDirectory(); + IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, null)); + Document doc = new Document(); + Token t1 = new Token("foo", 0, Integer.MAX_VALUE-500); + if (random().nextBoolean()) { + t1.setPayload(new BytesRef("test")); + } + Token t2 = new Token("foo", Integer.MAX_VALUE-500, Integer.MAX_VALUE); + TokenStream tokenStream = new CannedTokenStream( + new Token[] { t1, t2 } + ); + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); + ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); + // store some term vectors for the checkindex cross-check + ft.setStoreTermVectors(true); + ft.setStoreTermVectorPositions(true); + ft.setStoreTermVectorOffsets(true); + Field field = new Field("foo", tokenStream, ft); + doc.add(field); + iw.addDocument(doc); + iw.close(); + dir.close(); + } + // TODO: more tests with other possibilities + + private void checkTokens(Token[] tokens) throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter riw = new RandomIndexWriter(random(), dir, iwc); + boolean success = false; + try { + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); + ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); + // store some term vectors for the checkindex cross-check + ft.setStoreTermVectors(true); + ft.setStoreTermVectorPositions(true); + ft.setStoreTermVectorOffsets(true); + + Document doc = new Document(); + doc.add(new Field("body", new CannedTokenStream(tokens), ft)); + riw.addDocument(doc); + success = true; + } finally { + if (success) { + IOUtils.close(riw, dir); + } else { + IOUtils.closeWhileHandlingException(riw, dir); + } + } + } private Token makeToken(String text, int posIncr, int startOffset, int endOffset) { final Token t = new Token(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPrefixCodedTerms.java b/lucene/core/src/test/org/apache/lucene/index/TestPrefixCodedTerms.java index 94508da..9400b74 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPrefixCodedTerms.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPrefixCodedTerms.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/index/TestRandomStoredFields.java b/lucene/core/src/test/org/apache/lucene/index/TestRandomStoredFields.java index de1b0ed..fc74f90 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestRandomStoredFields.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestRandomStoredFields.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java b/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java index a61222a..25d1587 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,7 +21,6 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.store.AlreadyClosedException; @@ -42,7 +41,7 @@ public class TestReaderClosed extends LuceneTestCase { .setMaxBufferedDocs(_TestUtil.nextInt(random(), 50, 1000))); Document doc = new Document(); - Field field = newField("field", "", StringField.TYPE_UNSTORED); + Field field = newStringField("field", "", Field.Store.NO); doc.add(field); // we generate aweful prefixes: good for testing. diff --git a/lucene/core/src/test/org/apache/lucene/index/TestRollback.java b/lucene/core/src/test/org/apache/lucene/index/TestRollback.java index 9b356e2..9f8dd48 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestRollback.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestRollback.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,7 +19,7 @@ package org.apache.lucene.index; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.StringField; +import org.apache.lucene.document.Field; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; @@ -31,7 +31,7 @@ public class TestRollback extends LuceneTestCase { RandomIndexWriter rw = new RandomIndexWriter(random(), dir); for (int i = 0; i < 5; i++) { Document doc = new Document(); - doc.add(newField("pk", Integer.toString(i), StringField.TYPE_STORED)); + doc.add(newStringField("pk", Integer.toString(i), Field.Store.YES)); rw.addDocument(doc); } rw.close(); @@ -42,13 +42,13 @@ public class TestRollback extends LuceneTestCase { for (int i = 0; i < 3; i++) { Document doc = new Document(); String value = Integer.toString(i); - doc.add(newField("pk", value, StringField.TYPE_STORED)); - doc.add(newField("text", "foo", StringField.TYPE_STORED)); + doc.add(newStringField("pk", value, Field.Store.YES)); + doc.add(newStringField("text", "foo", Field.Store.YES)); w.updateDocument(new Term("pk", value), doc); } w.rollback(); - IndexReader r = IndexReader.open(dir); + IndexReader r = DirectoryReader.open(dir); assertEquals("index should contain same number of docs post rollback", 5, r.numDocs()); r.close(); dir.close(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java index 10e147c..cd131a1 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -37,11 +37,11 @@ public class TestRollingUpdates extends LuceneTestCase { Random random = new Random(random().nextLong()); final MockDirectoryWrapper dir = newDirectory(); dir.setCheckIndexOnClose(false); // we use a custom codec provider - final LineFileDocs docs = new LineFileDocs(random, defaultCodecSupportsDocValues()); + final LineFileDocs docs = new LineFileDocs(random, true); //provider.register(new MemoryCodec()); - if ( (!"Lucene3x".equals(Codec.getDefault().getName())) && random().nextBoolean()) { - Codec.setDefault(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat(random().nextBoolean()))); + if (random().nextBoolean()) { + Codec.setDefault(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat(random().nextBoolean(), random.nextFloat()))); } final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); @@ -110,7 +110,7 @@ public class TestRollingUpdates extends LuceneTestCase { w.close(); } - IndexReader open = IndexReader.open(dir); + IndexReader open = DirectoryReader.open(dir); assertEquals(1, open.numDocs()); open.close(); docs.close(); @@ -134,11 +134,11 @@ public class TestRollingUpdates extends LuceneTestCase { DirectoryReader open = null; for (int i = 0; i < num; i++) { Document doc = new Document();// docs.nextDoc(); - doc.add(newField("id", "test", StringField.TYPE_UNSTORED)); + doc.add(newStringField("id", "test", Field.Store.NO)); writer.updateDocument(new Term("id", "test"), doc); if (random().nextInt(3) == 0) { if (open == null) { - open = IndexReader.open(writer, true); + open = DirectoryReader.open(writer, true); } DirectoryReader reader = DirectoryReader.openIfChanged(open); if (reader != null) { diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java b/lucene/core/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java index 58d6365..5d627f1 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -72,7 +72,7 @@ final class BugReproTokenStream extends TokenStream { private final int incs[] = new int[]{1, 0, 1, 0}; @Override - public boolean incrementToken() throws IOException { + public boolean incrementToken() { if (nextTokenIndex < tokenCount) { termAtt.setEmpty().append(terms[nextTokenIndex]); offsetAtt.setOffset(starts[nextTokenIndex], ends[nextTokenIndex]); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java index 64f1f28..950c0fd 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -25,9 +25,11 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Constants; +import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; +import org.apache.lucene.util.packed.PackedInts; public class TestSegmentMerger extends LuceneTestCase { @@ -139,4 +141,41 @@ public class TestSegmentMerger extends LuceneTestCase { TestSegmentReader.checkNorms(mergedReader); mergedReader.close(); } + + private static boolean equals(MergeState.DocMap map1, MergeState.DocMap map2) { + if (map1.maxDoc() != map2.maxDoc()) { + return false; + } + for (int i = 0; i < map1.maxDoc(); ++i) { + if (map1.get(i) != map2.get(i)) { + return false; + } + } + return true; + } + + public void testBuildDocMap() { + final int maxDoc = 128; + final FixedBitSet liveDocs = new FixedBitSet(maxDoc); + + MergeState.DocMap docMap1 = MergeState.DocMap.buildDelCountDocmap(maxDoc, maxDoc, liveDocs, PackedInts.COMPACT); + MergeState.DocMap docMap2 = MergeState.DocMap.buildDirectDocMap(maxDoc, 0, liveDocs, PackedInts.COMPACT); + assertTrue(equals(docMap1, docMap2)); + + liveDocs.set(1); + for (int i = 7; i < 79; ++i) { + liveDocs.set(i); + } + liveDocs.set(80); + liveDocs.set(88); + int numDocs = liveDocs.cardinality(); + docMap1 = MergeState.DocMap.buildDelCountDocmap(maxDoc, maxDoc - numDocs, liveDocs, PackedInts.COMPACT); + docMap2 = MergeState.DocMap.buildDirectDocMap(maxDoc, numDocs, liveDocs, PackedInts.COMPACT); + assertTrue(equals(docMap1, docMap2)); + + liveDocs.set(0, maxDoc); + docMap1 = MergeState.DocMap.buildDelCountDocmap(maxDoc, 0, liveDocs, PackedInts.COMPACT); + docMap2 = MergeState.DocMap.buildDirectDocMap(maxDoc, maxDoc, liveDocs, PackedInts.COMPACT); + assertTrue(equals(docMap1, docMap2)); + } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java index 31c77b6..2f23c84 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermDocs.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermDocs.java index 27f4bc3..675161b 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermDocs.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermDocs.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,7 +21,7 @@ import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; @@ -131,7 +131,7 @@ public class TestSegmentTermDocs extends LuceneTestCase { writer.forceMerge(1); writer.close(); - IndexReader reader = IndexReader.open(dir, indexDivisor); + IndexReader reader = DirectoryReader.open(dir, indexDivisor); DocsEnum tdocs = _TestUtil.docs(random(), reader, ta.field(), @@ -280,7 +280,7 @@ public class TestSegmentTermDocs extends LuceneTestCase { private void addDoc(IndexWriter writer, String value) throws IOException { Document doc = new Document(); - doc.add(newField("content", value, TextField.TYPE_UNSTORED)); + doc.add(newTextField("content", value, Field.Store.NO)); writer.addDocument(doc); } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermEnum.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermEnum.java index c2b6d32..4554243 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermEnum.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermEnum.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,13 +19,13 @@ package org.apache.lucene.index; import java.io.IOException; +import org.apache.lucene.document.Field; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util._TestUtil; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.store.Directory; @@ -78,7 +78,7 @@ public class TestSegmentTermEnum extends LuceneTestCase { IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()))); addDoc(writer, "aaa bbb"); writer.close(); - SegmentReader reader = getOnlySegmentReader(IndexReader.open(dir)); + SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(dir)); TermsEnum terms = reader.fields().terms("content").iterator(null); assertNotNull(terms.next()); assertEquals("aaa", terms.term().utf8ToString()); @@ -102,7 +102,7 @@ public class TestSegmentTermEnum extends LuceneTestCase { private void verifyDocFreq() throws IOException { - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); TermsEnum termEnum = MultiFields.getTerms(reader, "content").iterator(null); // create enumeration of all terms @@ -135,7 +135,7 @@ public class TestSegmentTermEnum extends LuceneTestCase { private void addDoc(IndexWriter writer, String value) throws IOException { Document doc = new Document(); - doc.add(newField("content", value, TextField.TYPE_UNSTORED)); + doc.add(newTextField("content", value, Field.Store.NO)); writer.addDocument(doc); } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSizeBoundedForceMerge.java b/lucene/core/src/test/org/apache/lucene/index/TestSizeBoundedForceMerge.java index 01aea18..9ca95c9 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestSizeBoundedForceMerge.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestSizeBoundedForceMerge.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -36,14 +36,14 @@ public class TestSizeBoundedForceMerge extends LuceneTestCase { for (int i = 0; i < numDocs; i++) { Document doc = new Document(); if (withID) { - doc.add(new Field("id", "" + i, StringField.TYPE_UNSTORED)); + doc.add(new StringField("id", "" + i, Field.Store.NO)); } writer.addDocument(doc); } writer.commit(); } - private static IndexWriterConfig newWriterConfig() throws IOException { + private static IndexWriterConfig newWriterConfig() { IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null); conf.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); conf.setRAMBufferSizeMB(IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java index 7d0ac66..bebf6c5 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -54,7 +54,7 @@ public class TestSnapshotDeletionPolicy extends LuceneTestCase { } protected void checkMaxDoc(IndexCommit commit, int expectedMaxDoc) throws Exception { - IndexReader reader = IndexReader.open(commit); + IndexReader reader = DirectoryReader.open(commit); try { assertEquals(expectedMaxDoc, reader.maxDoc()); } finally { @@ -245,7 +245,7 @@ public class TestSnapshotDeletionPolicy extends LuceneTestCase { assertSnapshotExists(dir, sdp, numSnapshots); // open a reader on a snapshot - should succeed. - IndexReader.open(sdp.getSnapshot("snapshot0")).close(); + DirectoryReader.open(sdp.getSnapshot("snapshot0")).close(); // open a new IndexWriter w/ no snapshots to keep and assert that all snapshots are gone. sdp = getDeletionPolicy(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestStressAdvance.java b/lucene/core/src/test/org/apache/lucene/index/TestStressAdvance.java index 5ac8c50..ff58f8f 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestStressAdvance.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestStressAdvance.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -37,9 +37,9 @@ public class TestStressAdvance extends LuceneTestCase { RandomIndexWriter w = new RandomIndexWriter(random(), dir); final Set aDocs = new HashSet(); final Document doc = new Document(); - final Field f = newField("field", "", StringField.TYPE_UNSTORED); + final Field f = newStringField("field", "", Field.Store.NO); doc.add(f); - final Field idField = newField("id", "", StringField.TYPE_STORED); + final Field idField = newStringField("id", "", Field.Store.YES); doc.add(idField); int num = atLeast(4097); if (VERBOSE) { diff --git a/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing.java b/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing.java index 178c8d4..9129bd7 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing.java @@ -78,8 +78,8 @@ public class TestStressIndexing extends LuceneTestCase { for(int j=0; j<10; j++) { Document d = new Document(); int n = random().nextInt(); - d.add(newField("id", Integer.toString(nextID++), StringField.TYPE_STORED)); - d.add(newField("contents", English.intToEnglish(n), TextField.TYPE_UNSTORED)); + d.add(newStringField("id", Integer.toString(nextID++), Field.Store.YES)); + d.add(newTextField("contents", English.intToEnglish(n), Field.Store.NO)); writer.addDocument(d); } @@ -103,7 +103,7 @@ public class TestStressIndexing extends LuceneTestCase { @Override public void doWork() throws Throwable { for (int i=0; i<100; i++) { - IndexReader ir = IndexReader.open(directory); + IndexReader ir = DirectoryReader.open(directory); IndexSearcher is = new IndexSearcher(ir); ir.close(); } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java b/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java index 06c2afd..e9caa08 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java @@ -267,21 +267,21 @@ public class TestStressIndexing2 extends LuceneTestCase { } public void verifyEquals(Random r, DirectoryReader r1, Directory dir2, String idField) throws Throwable { - DirectoryReader r2 = IndexReader.open(dir2); + DirectoryReader r2 = DirectoryReader.open(dir2); verifyEquals(r1, r2, idField); r2.close(); } public void verifyEquals(Directory dir1, Directory dir2, String idField) throws Throwable { - DirectoryReader r1 = IndexReader.open(dir1); - DirectoryReader r2 = IndexReader.open(dir2); + DirectoryReader r1 = DirectoryReader.open(dir1); + DirectoryReader r2 = DirectoryReader.open(dir2); verifyEquals(r1, r2, idField); r1.close(); r2.close(); } private static void printDocs(DirectoryReader r) throws Throwable { - IndexReader[] subs = r.getSequentialSubReaders(); + List subs = r.getSequentialSubReaders(); for(IndexReader sub : subs) { // TODO: improve this Bits liveDocs = ((AtomicReader)sub).getLiveDocs(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestStressNRT.java b/lucene/core/src/test/org/apache/lucene/index/TestStressNRT.java index 5e1d356..9a78986 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestStressNRT.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestStressNRT.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -28,8 +28,8 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.StringField; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; @@ -109,7 +109,7 @@ public class TestStressNRT extends LuceneTestCase { final RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); writer.setDoRandomForceMergeAssert(false); writer.commit(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); for (int i=0; i terms, Map termToID, int id) throws IOException { Document doc = new Document(); - doc.add(new IntField("id", id)); + doc.add(new IntField("id", id, Field.Store.NO)); if (VERBOSE) { System.out.println("TEST: addDoc id:" + id + " terms=" + terms); } for (String s2 : terms) { - doc.add(newField("f", s2, StringField.TYPE_UNSTORED)); + doc.add(newStringField("f", s2, Field.Store.NO)); termToID.put(new BytesRef(s2), id); } w.addDocument(doc); @@ -184,8 +182,9 @@ public class TestTermsEnum extends LuceneTestCase { final Directory dir = newDirectory(); final RandomIndexWriter w = new RandomIndexWriter(random(), dir); - + final int numTerms = atLeast(300); + //final int numTerms = 50; final Set terms = new HashSet(); final Collection pendingTerms = new ArrayList(); @@ -257,8 +256,16 @@ public class TestTermsEnum extends LuceneTestCase { acceptTerms.add(s2); sortedAcceptTerms.add(new BytesRef(s2)); } - a = DaciukMihovAutomatonBuilder.build(sortedAcceptTerms); + a = BasicAutomata.makeStringUnion(sortedAcceptTerms); + } + + if (random().nextBoolean()) { + if (VERBOSE) { + System.out.println("TEST: reduce the automaton"); + } + a.reduce(); } + final CompiledAutomaton c = new CompiledAutomaton(a, true, false); final BytesRef[] acceptTermsArray = new BytesRef[acceptTerms.size()]; @@ -321,7 +328,7 @@ public class TestTermsEnum extends LuceneTestCase { final BytesRef expected = termsArray[loc]; final BytesRef actual = te.next(); if (VERBOSE) { - System.out.println("TEST: next() expected=" + expected.utf8ToString() + " actual=" + actual.utf8ToString()); + System.out.println("TEST: next() expected=" + expected.utf8ToString() + " actual=" + (actual == null ? "null" : actual.utf8ToString())); } assertEquals(expected, actual); assertEquals(1, te.docFreq()); @@ -358,7 +365,7 @@ public class TestTermsEnum extends LuceneTestCase { final RandomIndexWriter w = new RandomIndexWriter(random(), d, iwc); for(String term : terms) { Document doc = new Document(); - Field f = newField(FIELD, term, StringField.TYPE_UNSTORED); + Field f = newStringField(FIELD, term, Field.Store.NO); doc.add(f); w.addDocument(doc); } @@ -497,9 +504,9 @@ public class TestTermsEnum extends LuceneTestCase { d = newDirectory(); final RandomIndexWriter w = new RandomIndexWriter(random(), d); Document doc = new Document(); - doc.add(newField("field", "one two three", TextField.TYPE_UNSTORED)); + doc.add(newTextField("field", "one two three", Field.Store.NO)); doc = new Document(); - doc.add(newField("field2", "one two three", TextField.TYPE_UNSTORED)); + doc.add(newTextField("field2", "one two three", Field.Store.NO)); w.addDocument(doc); w.commit(); w.deleteDocuments(new Term("field", "one")); @@ -517,7 +524,7 @@ public class TestTermsEnum extends LuceneTestCase { } private String getRandomString() { - //return _TestUtil.randomSimpleString(random); + //return _TestUtil.randomSimpleString(random()); return _TestUtil.randomRealisticUnicodeString(random()); } @@ -713,4 +720,55 @@ public class TestTermsEnum extends LuceneTestCase { } } } + + public void testIntersectBasic() throws Exception { + Directory dir = newDirectory(); + IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); + iwc.setMergePolicy(new LogDocMergePolicy()); + RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); + Document doc = new Document(); + doc.add(newTextField("field", "aaa", Field.Store.NO)); + w.addDocument(doc); + + doc = new Document(); + doc.add(newStringField("field", "bbb", Field.Store.NO)); + w.addDocument(doc); + + doc = new Document(); + doc.add(newTextField("field", "ccc", Field.Store.NO)); + w.addDocument(doc); + + w.forceMerge(1); + DirectoryReader r = w.getReader(); + w.close(); + AtomicReader sub = getOnlySegmentReader(r); + Terms terms = sub.fields().terms("field"); + Automaton automaton = new RegExp(".*", RegExp.NONE).toAutomaton(); + CompiledAutomaton ca = new CompiledAutomaton(automaton, false, false); + TermsEnum te = terms.intersect(ca, null); + assertEquals("aaa", te.next().utf8ToString()); + assertEquals(0, te.docs(null, null, false).nextDoc()); + assertEquals("bbb", te.next().utf8ToString()); + assertEquals(1, te.docs(null, null, false).nextDoc()); + assertEquals("ccc", te.next().utf8ToString()); + assertEquals(2, te.docs(null, null, false).nextDoc()); + assertNull(te.next()); + + te = terms.intersect(ca, new BytesRef("abc")); + assertEquals("bbb", te.next().utf8ToString()); + assertEquals(1, te.docs(null, null, false).nextDoc()); + assertEquals("ccc", te.next().utf8ToString()); + assertEquals(2, te.docs(null, null, false).nextDoc()); + assertNull(te.next()); + + te = terms.intersect(ca, new BytesRef("aaa")); + assertEquals("bbb", te.next().utf8ToString()); + assertEquals(1, te.docs(null, null, false).nextDoc()); + assertEquals("ccc", te.next().utf8ToString()); + assertEquals(2, te.docs(null, null, false).nextDoc()); + assertNull(te.next()); + + r.close(); + dir.close(); + } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum2.java b/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum2.java index d322add..8b9dc7e 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum2.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum2.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -25,10 +25,8 @@ import java.util.TreeSet; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; -import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; import org.apache.lucene.index.TermsEnum.SeekStatus; import org.apache.lucene.search.AutomatonQuery; import org.apache.lucene.search.CheckHits; @@ -37,13 +35,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; -import org.apache.lucene.util.automaton.Automaton; -import org.apache.lucene.util.automaton.AutomatonTestUtil; -import org.apache.lucene.util.automaton.BasicOperations; -import org.apache.lucene.util.automaton.CompiledAutomaton; -import org.apache.lucene.util.automaton.DaciukMihovAutomatonBuilder; -import org.apache.lucene.util.automaton.RegExp; -import org.apache.lucene.util.automaton.SpecialOperations; +import org.apache.lucene.util.automaton.*; public class TestTermsEnum2 extends LuceneTestCase { private Directory dir; @@ -55,16 +47,14 @@ public class TestTermsEnum2 extends LuceneTestCase { public void setUp() throws Exception { super.setUp(); - // we generate aweful regexps: good for testing. - // but for preflex codec, the test can be very slow, so use less iterations. - numIterations = Codec.getDefault().getName().equals("Lucene3x") ? 10 * RANDOM_MULTIPLIER : atLeast(50); + numIterations = atLeast(50); dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)) .setMaxBufferedDocs(_TestUtil.nextInt(random(), 50, 1000))); Document doc = new Document(); - Field field = newField("field", "", StringField.TYPE_STORED); + Field field = newStringField("field", "", Field.Store.YES); doc.add(field); terms = new TreeSet(); @@ -76,7 +66,7 @@ public class TestTermsEnum2 extends LuceneTestCase { writer.addDocument(doc); } - termsAutomaton = DaciukMihovAutomatonBuilder.build(terms); + termsAutomaton = BasicAutomata.makeStringUnion(terms); reader = writer.getReader(); searcher = newSearcher(reader); @@ -101,7 +91,7 @@ public class TestTermsEnum2 extends LuceneTestCase { } } - Automaton alternate = DaciukMihovAutomatonBuilder.build(matchedTerms); + Automaton alternate = BasicAutomata.makeStringUnion(matchedTerms); //System.out.println("match " + matchedTerms.size() + " " + alternate.getNumberOfStates() + " states, sigma=" + alternate.getStartPoints().length); //AutomatonTestUtil.minimizeSimple(alternate); //System.out.println("minmize done"); @@ -168,7 +158,7 @@ public class TestTermsEnum2 extends LuceneTestCase { found.add(BytesRef.deepCopyOf(te.term())); } - Automaton actual = DaciukMihovAutomatonBuilder.build(found); + Automaton actual = BasicAutomata.makeStringUnion(found); assertTrue(BasicOperations.sameLanguage(expected, actual)); } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestThreadedForceMerge.java b/lucene/core/src/test/org/apache/lucene/index/TestThreadedForceMerge.java index 84be8d0..3b78629 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestThreadedForceMerge.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestThreadedForceMerge.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -129,8 +129,8 @@ public class TestThreadedForceMerge extends LuceneTestCase { TEST_VERSION_CURRENT, ANALYZER).setOpenMode( OpenMode.APPEND).setMaxBufferedDocs(2)); - DirectoryReader reader = IndexReader.open(directory); - assertEquals("reader=" + reader, 1, reader.getSequentialSubReaders().length); + DirectoryReader reader = DirectoryReader.open(directory); + assertEquals("reader=" + reader, 1, reader.getSequentialSubReaders().size()); assertEquals(expectedDocCount, reader.numDocs()); reader.close(); } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java index e876d0b..beb6570 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,8 +19,7 @@ package org.apache.lucene.index; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; @@ -39,7 +38,7 @@ public class TestTieredMergePolicy extends LuceneTestCase { IndexWriter w = new IndexWriter(dir, conf); for(int i=0;i<80;i++) { Document doc = new Document(); - doc.add(newField("content", "aaa " + (i%4), TextField.TYPE_UNSTORED)); + doc.add(newTextField("content", "aaa " + (i%4), Field.Store.NO)); w.addDocument(doc); } assertEquals(80, w.maxDoc()); @@ -57,7 +56,7 @@ public class TestTieredMergePolicy extends LuceneTestCase { if (VERBOSE) { System.out.println("\nTEST: forceMergeDeletes2"); } - tmp.setForceMergeDeletesPctAllowed(10.0); + ((TieredMergePolicy) w.getConfig().getMergePolicy()).setForceMergeDeletesPctAllowed(10.0); w.forceMergeDeletes(); assertEquals(60, w.maxDoc()); assertEquals(60, w.numDocs()); @@ -85,7 +84,7 @@ public class TestTieredMergePolicy extends LuceneTestCase { final int numDocs = _TestUtil.nextInt(random(), 20, 100); for(int i=0;i prepareCommitData = null; + Map commitData = null; + + public TwoPhaseCommitImpl(boolean failOnPrepare, boolean failOnCommit, boolean failOnRollback) { + this.failOnPrepare = failOnPrepare; + this.failOnCommit = failOnCommit; + this.failOnRollback = failOnRollback; + } + + public void prepareCommit() throws IOException { + prepareCommit(null); + } + + public void prepareCommit(Map commitData) throws IOException { + this.prepareCommitData = commitData; + assertFalse("commit should not have been called before all prepareCommit were", commitCalled); + if (failOnPrepare) { + throw new IOException("failOnPrepare"); + } + } + + public void commit() throws IOException { + commit(null); + } + + public void commit(Map commitData) throws IOException { + this.commitData = commitData; + commitCalled = true; + if (failOnCommit) { + throw new RuntimeException("failOnCommit"); + } + } + + public void rollback() throws IOException { + rollbackCalled = true; + if (failOnRollback) { + throw new Error("failOnRollback"); + } + } + } + + @Override + public void setUp() throws Exception { + super.setUp(); + TwoPhaseCommitImpl.commitCalled = false; // reset count before every test + } + + public void testPrepareThenCommit() throws Exception { + // tests that prepareCommit() is called on all objects before commit() + TwoPhaseCommitImpl[] objects = new TwoPhaseCommitImpl[2]; + for (int i = 0; i < objects.length; i++) { + objects[i] = new TwoPhaseCommitImpl(false, false, false); + } + + // following call will fail if commit() is called before all prepare() were + TwoPhaseCommitTool.execute(objects); + } + + public void testRollback() throws Exception { + // tests that rollback is called if failure occurs at any stage + int numObjects = random().nextInt(8) + 3; // between [3, 10] + TwoPhaseCommitImpl[] objects = new TwoPhaseCommitImpl[numObjects]; + for (int i = 0; i < objects.length; i++) { + boolean failOnPrepare = random().nextBoolean(); + // we should not hit failures on commit usually + boolean failOnCommit = random().nextDouble() < 0.05; + boolean railOnRollback = random().nextBoolean(); + objects[i] = new TwoPhaseCommitImpl(failOnPrepare, failOnCommit, railOnRollback); + } + + boolean anyFailure = false; + try { + TwoPhaseCommitTool.execute(objects); + } catch (Throwable t) { + anyFailure = true; + } + + if (anyFailure) { + // if any failure happened, ensure that rollback was called on all. + for (TwoPhaseCommitImpl tpc : objects) { + assertTrue("rollback was not called while a failure occurred during the 2-phase commit", tpc.rollbackCalled); + } + } + } + + public void testWrapper() throws Exception { + // tests that TwoPhaseCommitWrapper delegates prepare/commit w/ commitData + TwoPhaseCommitImpl impl = new TwoPhaseCommitImpl(false, false, false); + HashMap commitData = new HashMap(); + TwoPhaseCommitWrapper wrapper = new TwoPhaseCommitWrapper(impl, commitData); + + wrapper.prepareCommit(); + assertSame(commitData, impl.prepareCommitData); + + // wrapper should ignore passed commitData + wrapper.prepareCommit(new HashMap()); + assertSame(commitData, impl.prepareCommitData); + + wrapper.commit(); + assertSame(commitData, impl.commitData); + + // wrapper should ignore passed commitData + wrapper.commit(new HashMap()); + assertSame(commitData, impl.commitData); + } + + public void testNullTPCs() throws Exception { + int numObjects = random().nextInt(4) + 3; // between [3, 6] + TwoPhaseCommit[] tpcs = new TwoPhaseCommit[numObjects]; + boolean setNull = false; + for (int i = 0; i < tpcs.length; i++) { + boolean isNull = random().nextDouble() < 0.3; + if (isNull) { + setNull = true; + tpcs[i] = null; + } else { + tpcs[i] = new TwoPhaseCommitImpl(false, false, false); + } + } + + if (!setNull) { + // none of the TPCs were picked to be null, pick one at random + int idx = random().nextInt(numObjects); + tpcs[idx] = null; + } + + // following call would fail if TPCTool won't handle null TPCs properly + TwoPhaseCommitTool.execute(tpcs); + } + +} diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTypePromotion.java b/lucene/core/src/test/org/apache/lucene/index/TestTypePromotion.java index ddec37f..f5dbcd0 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestTypePromotion.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestTypePromotion.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF @@ -19,6 +19,7 @@ package org.apache.lucene.index; import java.io.IOException; import java.util.EnumSet; +import java.util.List; import java.util.Random; import org.apache.lucene.analysis.MockAnalyzer; @@ -41,9 +42,7 @@ import org.apache.lucene.store.ByteArrayDataOutput; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; -@SuppressCodecs("Lucene3x") public class TestTypePromotion extends LuceneTestCase { private static EnumSet INTEGERS = EnumSet.of(Type.VAR_INTS, @@ -51,7 +50,7 @@ public class TestTypePromotion extends LuceneTestCase { Type.FIXED_INTS_64, Type.FIXED_INTS_8); private static EnumSet FLOATS = EnumSet.of(Type.FLOAT_32, - Type.FLOAT_64); + Type.FLOAT_64, Type.FIXED_INTS_8); private static EnumSet UNSORTED_BYTES = EnumSet.of( Type.BYTES_FIXED_DEREF, Type.BYTES_FIXED_STRAIGHT, @@ -69,8 +68,7 @@ public class TestTypePromotion extends LuceneTestCase { Int, Float, Byte } - private void runTest(EnumSet types, TestType type) - throws CorruptIndexException, IOException { + private void runTest(EnumSet types, TestType type) throws IOException { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); @@ -78,12 +76,13 @@ public class TestTypePromotion extends LuceneTestCase { int num_2 = atLeast(200); int num_3 = atLeast(200); long[] values = new long[num_1 + num_2 + num_3]; + Type[] sourceType = new Type[num_1 + num_2 + num_3]; index(writer, - randomValueType(types, random()), values, 0, num_1); + randomValueType(types, random()), values, sourceType, 0, num_1); writer.commit(); index(writer, - randomValueType(types, random()), values, num_1, num_2); + randomValueType(types, random()), values, sourceType, num_1, num_2); writer.commit(); if (random().nextInt(4) == 0) { @@ -94,38 +93,38 @@ public class TestTypePromotion extends LuceneTestCase { IndexWriter writer_2 = new IndexWriter(dir_2, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); index(writer_2, - randomValueType(types, random()), values, num_1 + num_2, num_3); + randomValueType(types, random()), values, sourceType, num_1 + num_2, num_3); writer_2.commit(); writer_2.close(); if (rarely()) { writer.addIndexes(dir_2); } else { // do a real merge here - IndexReader open = maybeWrapReader(IndexReader.open(dir_2)); + IndexReader open = maybeWrapReader(DirectoryReader.open(dir_2)); writer.addIndexes(open); open.close(); } dir_2.close(); } else { index(writer, - randomValueType(types, random()), values, num_1 + num_2, num_3); + randomValueType(types, random()), values, sourceType, num_1 + num_2, num_3); } writer.forceMerge(1); writer.close(); - assertValues(type, dir, values); + assertValues(type, dir, values, sourceType); dir.close(); } - private void assertValues(TestType type, Directory dir, long[] values) - throws CorruptIndexException, IOException { + private void assertValues(TestType type, Directory dir, long[] values, Type[] sourceType) + throws IOException { DirectoryReader reader = DirectoryReader.open(dir); - assertEquals(1, reader.getSequentialSubReaders().length); + assertEquals(1, reader.getSequentialSubReaders().size()); IndexReaderContext topReaderContext = reader.getTopReaderContext(); - AtomicReaderContext[] children = topReaderContext.leaves(); - assertEquals(1, children.length); - DocValues docValues = children[0].reader().docValues("promote"); + List leaves = topReaderContext.leaves(); + assertEquals(1, leaves.size()); + DocValues docValues = leaves.get(0).reader().docValues("promote"); Source directSource = docValues.getDirectSource(); for (int i = 0; i < values.length; i++) { int id = Integer.parseInt(reader.document(i).get("id")); @@ -159,7 +158,13 @@ public class TestTypePromotion extends LuceneTestCase { assertEquals(msg + " byteSize: " + bytes.length, values[id], value); break; case Float: - assertEquals(msg, values[id], Double.doubleToRawLongBits(directSource.getFloat(i))); + if (sourceType[id] == Type.FLOAT_32 + || sourceType[id] == Type.FLOAT_64) { + assertEquals(msg, values[id], + Double.doubleToRawLongBits(directSource.getFloat(i))); + } else { + assertEquals(msg, values[id], directSource.getFloat(i), 0.0d); + } break; case Int: assertEquals(msg, values[id], directSource.getInt(i)); @@ -174,8 +179,8 @@ public class TestTypePromotion extends LuceneTestCase { } public void index(IndexWriter writer, - Type valueType, long[] values, int offset, int num) - throws CorruptIndexException, IOException { + Type valueType, long[] values, Type[] sourceTypes, int offset, int num) + throws IOException { final Field valField; if (VERBOSE) { @@ -228,7 +233,8 @@ public class TestTypePromotion extends LuceneTestCase { for (int i = offset; i < offset + num; i++) { Document doc = new Document(); - doc.add(new Field("id", i + "", TextField.TYPE_STORED)); + doc.add(new TextField("id", i + "", Field.Store.YES)); + sourceTypes[i] = valueType; switch (valueType) { case VAR_INTS: // TODO: can we do nextLong()? @@ -315,8 +321,7 @@ public class TestTypePromotion extends LuceneTestCase { runTest(INTEGERS, TestType.Int); } - public void testPromotFloatingPoint() throws CorruptIndexException, - IOException { + public void testPromotFloatingPoint() throws IOException { runTest(FLOATS, TestType.Float); } @@ -328,8 +333,9 @@ public class TestTypePromotion extends LuceneTestCase { int num_1 = atLeast(200); int num_2 = atLeast(200); long[] values = new long[num_1 + num_2]; + Type[] sourceType = new Type[num_1 + num_2]; index(writer, - randomValueType(INTEGERS, random()), values, 0, num_1); + randomValueType(INTEGERS, random()), values, sourceType, 0, num_1); writer.commit(); if (random().nextInt(4) == 0) { @@ -338,21 +344,21 @@ public class TestTypePromotion extends LuceneTestCase { IndexWriter writer_2 = new IndexWriter(dir_2, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); index(writer_2, - randomValueType(random().nextBoolean() ? UNSORTED_BYTES : SORTED_BYTES, random()), values, num_1, num_2); + randomValueType(random().nextBoolean() ? UNSORTED_BYTES : SORTED_BYTES, random()), values, sourceType, num_1, num_2); writer_2.commit(); writer_2.close(); if (random().nextBoolean()) { writer.addIndexes(dir_2); } else { // do a real merge here - IndexReader open = IndexReader.open(dir_2); + IndexReader open = DirectoryReader.open(dir_2); writer.addIndexes(open); open.close(); } dir_2.close(); } else { index(writer, - randomValueType(random().nextBoolean() ? UNSORTED_BYTES : SORTED_BYTES, random()), values, num_1, num_2); + randomValueType(random().nextBoolean() ? UNSORTED_BYTES : SORTED_BYTES, random()), values, sourceType, num_1, num_2); writer.commit(); } writer.close(); @@ -365,12 +371,12 @@ public class TestTypePromotion extends LuceneTestCase { writer.forceMerge(1); writer.close(); DirectoryReader reader = DirectoryReader.open(dir); - assertEquals(1, reader.getSequentialSubReaders().length); + assertEquals(1, reader.getSequentialSubReaders().size()); IndexReaderContext topReaderContext = reader.getTopReaderContext(); - AtomicReaderContext[] children = topReaderContext.leaves(); - DocValues docValues = children[0].reader().docValues("promote"); + List leaves = topReaderContext.leaves(); + DocValues docValues = leaves.get(0).reader().docValues("promote"); assertNotNull(docValues); - assertValues(TestType.Byte, dir, values); + assertValues(TestType.Byte, dir, values, sourceType); assertEquals(Type.BYTES_VAR_STRAIGHT, docValues.getType()); reader.close(); dir.close(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java b/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java index c5ed174..911e890 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -24,9 +24,7 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.TextField; import org.apache.lucene.search.similarities.DefaultSimilarity; -import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; @@ -49,7 +47,7 @@ public class TestUniqueTermCount extends LuceneTestCase { config.setSimilarity(new TestSimilarity()); RandomIndexWriter writer = new RandomIndexWriter(random(), dir, config); Document doc = new Document(); - Field foo = newField("foo", "", TextField.TYPE_UNSTORED); + Field foo = newTextField("foo", "", Field.Store.NO); doc.add(foo); for (int i = 0; i < 100; i++) { foo.setStringValue(addValue()); diff --git a/lucene/core/src/test/org/apache/lucene/index/index.30.cfs.zip b/lucene/core/src/test/org/apache/lucene/index/index.30.cfs.zip deleted file mode 100644 index d5978c8..0000000 Binary files a/lucene/core/src/test/org/apache/lucene/index/index.30.cfs.zip and /dev/null differ diff --git a/lucene/core/src/test/org/apache/lucene/index/index.30.nocfs.zip b/lucene/core/src/test/org/apache/lucene/index/index.30.nocfs.zip deleted file mode 100644 index 28cd83b..0000000 Binary files a/lucene/core/src/test/org/apache/lucene/index/index.30.nocfs.zip and /dev/null differ diff --git a/lucene/core/src/test/org/apache/lucene/index/index.31.cfs.zip b/lucene/core/src/test/org/apache/lucene/index/index.31.cfs.zip deleted file mode 100644 index 8f123a7..0000000 Binary files a/lucene/core/src/test/org/apache/lucene/index/index.31.cfs.zip and /dev/null differ diff --git a/lucene/core/src/test/org/apache/lucene/index/index.31.nocfs.zip b/lucene/core/src/test/org/apache/lucene/index/index.31.nocfs.zip deleted file mode 100644 index 21434e1..0000000 Binary files a/lucene/core/src/test/org/apache/lucene/index/index.31.nocfs.zip and /dev/null differ diff --git a/lucene/core/src/test/org/apache/lucene/index/index.31.optimized.cfs.zip b/lucene/core/src/test/org/apache/lucene/index/index.31.optimized.cfs.zip deleted file mode 100644 index 200c710..0000000 Binary files a/lucene/core/src/test/org/apache/lucene/index/index.31.optimized.cfs.zip and /dev/null differ diff --git a/lucene/core/src/test/org/apache/lucene/index/index.31.optimized.nocfs.zip b/lucene/core/src/test/org/apache/lucene/index/index.31.optimized.nocfs.zip deleted file mode 100644 index 9a158f1..0000000 Binary files a/lucene/core/src/test/org/apache/lucene/index/index.31.optimized.nocfs.zip and /dev/null differ diff --git a/lucene/core/src/test/org/apache/lucene/index/index.32.cfs.zip b/lucene/core/src/test/org/apache/lucene/index/index.32.cfs.zip deleted file mode 100644 index eff3153..0000000 Binary files a/lucene/core/src/test/org/apache/lucene/index/index.32.cfs.zip and /dev/null differ diff --git a/lucene/core/src/test/org/apache/lucene/index/index.32.nocfs.zip b/lucene/core/src/test/org/apache/lucene/index/index.32.nocfs.zip deleted file mode 100644 index 0b345da..0000000 Binary files a/lucene/core/src/test/org/apache/lucene/index/index.32.nocfs.zip and /dev/null differ diff --git a/lucene/core/src/test/org/apache/lucene/index/index.34.cfs.zip b/lucene/core/src/test/org/apache/lucene/index/index.34.cfs.zip deleted file mode 100644 index 257e9d8..0000000 Binary files a/lucene/core/src/test/org/apache/lucene/index/index.34.cfs.zip and /dev/null differ diff --git a/lucene/core/src/test/org/apache/lucene/index/index.34.nocfs.zip b/lucene/core/src/test/org/apache/lucene/index/index.34.nocfs.zip deleted file mode 100644 index 935d6a1..0000000 Binary files a/lucene/core/src/test/org/apache/lucene/index/index.34.nocfs.zip and /dev/null differ diff --git a/lucene/core/src/test/org/apache/lucene/index/index.36.surrogates.zip b/lucene/core/src/test/org/apache/lucene/index/index.36.surrogates.zip deleted file mode 100644 index 6bd7f20..0000000 Binary files a/lucene/core/src/test/org/apache/lucene/index/index.36.surrogates.zip and /dev/null differ diff --git a/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip b/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip new file mode 100644 index 0000000..4974749 Binary files /dev/null and b/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip differ diff --git a/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip b/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip new file mode 100644 index 0000000..9699080 Binary files /dev/null and b/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip differ diff --git a/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip b/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip new file mode 100644 index 0000000..209c436 Binary files /dev/null and b/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip differ diff --git a/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip b/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip new file mode 100644 index 0000000..0eaffd0 Binary files /dev/null and b/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip differ diff --git a/lucene/core/src/test/org/apache/lucene/index/unsupported.30.cfs.zip b/lucene/core/src/test/org/apache/lucene/index/unsupported.30.cfs.zip new file mode 100644 index 0000000..d5978c8 Binary files /dev/null and b/lucene/core/src/test/org/apache/lucene/index/unsupported.30.cfs.zip differ diff --git a/lucene/core/src/test/org/apache/lucene/index/unsupported.30.nocfs.zip b/lucene/core/src/test/org/apache/lucene/index/unsupported.30.nocfs.zip new file mode 100644 index 0000000..28cd83b Binary files /dev/null and b/lucene/core/src/test/org/apache/lucene/index/unsupported.30.nocfs.zip differ diff --git a/lucene/core/src/test/org/apache/lucene/index/unsupported.31.cfs.zip b/lucene/core/src/test/org/apache/lucene/index/unsupported.31.cfs.zip new file mode 100644 index 0000000..8f123a7 Binary files /dev/null and b/lucene/core/src/test/org/apache/lucene/index/unsupported.31.cfs.zip differ diff --git a/lucene/core/src/test/org/apache/lucene/index/unsupported.31.nocfs.zip b/lucene/core/src/test/org/apache/lucene/index/unsupported.31.nocfs.zip new file mode 100644 index 0000000..21434e1 Binary files /dev/null and b/lucene/core/src/test/org/apache/lucene/index/unsupported.31.nocfs.zip differ diff --git a/lucene/core/src/test/org/apache/lucene/index/unsupported.32.cfs.zip b/lucene/core/src/test/org/apache/lucene/index/unsupported.32.cfs.zip new file mode 100644 index 0000000..eff3153 Binary files /dev/null and b/lucene/core/src/test/org/apache/lucene/index/unsupported.32.cfs.zip differ diff --git a/lucene/core/src/test/org/apache/lucene/index/unsupported.32.nocfs.zip b/lucene/core/src/test/org/apache/lucene/index/unsupported.32.nocfs.zip new file mode 100644 index 0000000..0b345da Binary files /dev/null and b/lucene/core/src/test/org/apache/lucene/index/unsupported.32.nocfs.zip differ diff --git a/lucene/core/src/test/org/apache/lucene/index/unsupported.34.cfs.zip b/lucene/core/src/test/org/apache/lucene/index/unsupported.34.cfs.zip new file mode 100644 index 0000000..257e9d8 Binary files /dev/null and b/lucene/core/src/test/org/apache/lucene/index/unsupported.34.cfs.zip differ diff --git a/lucene/core/src/test/org/apache/lucene/index/unsupported.34.nocfs.zip b/lucene/core/src/test/org/apache/lucene/index/unsupported.34.nocfs.zip new file mode 100644 index 0000000..935d6a1 Binary files /dev/null and b/lucene/core/src/test/org/apache/lucene/index/unsupported.34.nocfs.zip differ diff --git a/lucene/core/src/test/org/apache/lucene/search/BaseTestRangeFilter.java b/lucene/core/src/test/org/apache/lucene/search/BaseTestRangeFilter.java index 9064630..72da35b 100644 --- a/lucene/core/src/test/org/apache/lucene/search/BaseTestRangeFilter.java +++ b/lucene/core/src/test/org/apache/lucene/search/BaseTestRangeFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,7 +23,6 @@ import java.util.Random; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.RandomIndexWriter; @@ -55,9 +54,7 @@ public class BaseTestRangeFilter extends LuceneTestCase { this.minR = minR; this.maxR = maxR; this.allowNegativeRandomInts = allowNegativeRandomInts; - try { - index = newDirectory(random); - } catch (IOException e) { throw new RuntimeException(e); } + index = newDirectory(random); } } @@ -117,9 +114,9 @@ public class BaseTestRangeFilter extends LuceneTestCase { /* build an index */ Document doc = new Document(); - Field idField = newField(random, "id", "", StringField.TYPE_STORED); - Field randField = newField(random, "rand", "", StringField.TYPE_STORED); - Field bodyField = newField(random, "body", "", StringField.TYPE_UNSTORED); + Field idField = newStringField(random, "id", "", Field.Store.YES); + Field randField = newStringField(random, "rand", "", Field.Store.YES); + Field bodyField = newStringField(random, "body", "", Field.Store.NO); doc.add(idField); doc.add(randField); doc.add(bodyField); diff --git a/lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java b/lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java index 2abdf76..861a79b 100644 --- a/lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java +++ b/lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -94,12 +94,12 @@ public final class FieldCacheRewriteMethod extends MultiTermQuery.RewriteMethod TermsEnum termsEnum = query.getTermsEnum(new Terms() { @Override - public Comparator getComparator() throws IOException { + public Comparator getComparator() { return BytesRef.getUTF8SortedAsUnicodeComparator(); } @Override - public TermsEnum iterator(TermsEnum reuse) throws IOException { + public TermsEnum iterator(TermsEnum reuse) { return fcsi.getTermsEnum(); } @@ -109,17 +109,17 @@ public final class FieldCacheRewriteMethod extends MultiTermQuery.RewriteMethod } @Override - public long getSumDocFreq() throws IOException { + public long getSumDocFreq() { return -1; } @Override - public int getDocCount() throws IOException { + public int getDocCount() { return -1; } @Override - public long size() throws IOException { + public long size() { return -1; } }); diff --git a/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java b/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java index 612490d..17a122c 100644 --- a/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java +++ b/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -42,18 +42,17 @@ final class JustCompileSearch { static final class JustCompileCollector extends Collector { @Override - public void collect(int doc) throws IOException { + public void collect(int doc) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @Override - public void setNextReader(AtomicReaderContext context) - throws IOException { + public void setNextReader(AtomicReaderContext context) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @Override - public void setScorer(Scorer scorer) throws IOException { + public void setScorer(Scorer scorer) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @@ -67,7 +66,7 @@ final class JustCompileSearch { static final class JustCompileDocIdSet extends DocIdSet { @Override - public DocIdSetIterator iterator() throws IOException { + public DocIdSetIterator iterator() { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @@ -81,12 +80,12 @@ final class JustCompileSearch { } @Override - public int nextDoc() throws IOException { + public int nextDoc() { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @Override - public int advance(int target) throws IOException { + public int advance(int target) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } } @@ -115,12 +114,12 @@ final class JustCompileSearch { } @Override - public int compareBottom(int doc) throws IOException { + public int compareBottom(int doc) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @Override - public void copy(int slot, int doc) throws IOException { + public void copy(int slot, int doc) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @@ -130,8 +129,7 @@ final class JustCompileSearch { } @Override - public FieldComparator setNextReader(AtomicReaderContext context) - throws IOException { + public FieldComparator setNextReader(AtomicReaderContext context) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @@ -150,7 +148,7 @@ final class JustCompileSearch { @Override public FieldComparator newComparator(String fieldname, int numHits, - int sortPos, boolean reversed) throws IOException { + int sortPos, boolean reversed) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @@ -161,7 +159,7 @@ final class JustCompileSearch { // still added here in case someone will add abstract methods in the future. @Override - public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException { + public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) { return null; } } @@ -200,7 +198,7 @@ final class JustCompileSearch { } @Override - protected float phraseFreq() throws IOException { + protected float phraseFreq() { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @@ -227,13 +225,12 @@ final class JustCompileSearch { } @Override - public boolean score(Collector collector, int max, int firstDocID) - throws IOException { + public boolean score(Collector collector, int max, int firstDocID) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @Override - public float score() throws IOException { + public float score() { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @@ -243,12 +240,12 @@ final class JustCompileSearch { } @Override - public int nextDoc() throws IOException { + public int nextDoc() { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @Override - public int advance(int target) throws IOException { + public int advance(int target) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @@ -266,12 +263,12 @@ final class JustCompileSearch { } @Override - public ExactSimScorer exactSimScorer(SimWeight stats, AtomicReaderContext context) throws IOException { + public ExactSimScorer exactSimScorer(SimWeight stats, AtomicReaderContext context) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @Override - public SloppySimScorer sloppySimScorer(SimWeight stats, AtomicReaderContext context) throws IOException { + public SloppySimScorer sloppySimScorer(SimWeight stats, AtomicReaderContext context) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @@ -288,18 +285,17 @@ final class JustCompileSearch { } @Override - public void collect(int doc) throws IOException { + public void collect(int doc) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @Override - public void setNextReader(AtomicReaderContext context) - throws IOException { + public void setNextReader(AtomicReaderContext context) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @Override - public void setScorer(Scorer scorer) throws IOException { + public void setScorer(Scorer scorer) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @@ -328,7 +324,7 @@ final class JustCompileSearch { static final class JustCompileWeight extends Weight { @Override - public Explanation explain(AtomicReaderContext context, int doc) throws IOException { + public Explanation explain(AtomicReaderContext context, int doc) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @@ -343,14 +339,13 @@ final class JustCompileSearch { } @Override - public float getValueForNormalization() throws IOException { + public float getValueForNormalization() { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @Override public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder, - boolean topScorer, Bits acceptDocs) - throws IOException { + boolean topScorer, Bits acceptDocs) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } diff --git a/lucene/core/src/test/org/apache/lucene/search/MockFilter.java b/lucene/core/src/test/org/apache/lucene/search/MockFilter.java index 4b982d2..56b6dce 100644 --- a/lucene/core/src/test/org/apache/lucene/search/MockFilter.java +++ b/lucene/core/src/test/org/apache/lucene/search/MockFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/search/MultiCollectorTest.java b/lucene/core/src/test/org/apache/lucene/search/MultiCollectorTest.java index 189a20d..2a63f49 100644 --- a/lucene/core/src/test/org/apache/lucene/search/MultiCollectorTest.java +++ b/lucene/core/src/test/org/apache/lucene/search/MultiCollectorTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/search/SingleDocTestFilter.java b/lucene/core/src/test/org/apache/lucene/search/SingleDocTestFilter.java index 78fd1ad..df4fd07 100644 --- a/lucene/core/src/test/org/apache/lucene/search/SingleDocTestFilter.java +++ b/lucene/core/src/test/org/apache/lucene/search/SingleDocTestFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQuery.java index 36736c0..bf51048 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,7 +21,6 @@ import java.io.IOException; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.RandomIndexWriter; @@ -48,9 +47,9 @@ public class TestAutomatonQuery extends LuceneTestCase { directory = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), directory); Document doc = new Document(); - Field titleField = newField("title", "some title", TextField.TYPE_UNSTORED); - Field field = newField(FN, "this is document one 2345", TextField.TYPE_UNSTORED); - Field footerField = newField("footer", "a footer", TextField.TYPE_UNSTORED); + Field titleField = newTextField("title", "some title", Field.Store.NO); + Field field = newTextField(FN, "this is document one 2345", Field.Store.NO); + Field footerField = newTextField("footer", "a footer", Field.Store.NO); doc.add(titleField); doc.add(field); doc.add(footerField); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQueryUnicode.java b/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQueryUnicode.java index d03fbac..0606ad3 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQueryUnicode.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQueryUnicode.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,7 +21,6 @@ import java.io.IOException; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; @@ -48,9 +47,9 @@ public class TestAutomatonQueryUnicode extends LuceneTestCase { directory = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), directory); Document doc = new Document(); - Field titleField = newField("title", "some title", TextField.TYPE_UNSTORED); - Field field = newField(FN, "", TextField.TYPE_UNSTORED); - Field footerField = newField("footer", "a footer", TextField.TYPE_UNSTORED); + Field titleField = newTextField("title", "some title", Field.Store.NO); + Field field = newTextField(FN, "", Field.Store.NO); + Field footerField = newTextField("footer", "a footer", Field.Store.NO); doc.add(titleField); doc.add(field); doc.add(footerField); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java b/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java index 5dc6bb1..060cdb2 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,7 +22,8 @@ import java.util.Random; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexReader; @@ -59,11 +60,11 @@ public class TestBoolean2 extends LuceneTestCase { RandomIndexWriter writer= new RandomIndexWriter(random(), directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); for (int i = 0; i < docFields.length; i++) { Document doc = new Document(); - doc.add(newField(field, docFields[i], TextField.TYPE_UNSTORED)); + doc.add(newTextField(field, docFields[i], Field.Store.NO)); writer.addDocument(doc); } writer.close(); - littleReader = IndexReader.open(directory); + littleReader = DirectoryReader.open(directory); searcher = new IndexSearcher(littleReader); // Make big index @@ -91,12 +92,12 @@ public class TestBoolean2 extends LuceneTestCase { newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())) .setMaxBufferedDocs(_TestUtil.nextInt(random(), 50, 1000))); Document doc = new Document(); - doc.add(newField("field2", "xxx", TextField.TYPE_UNSTORED)); + doc.add(newTextField("field2", "xxx", Field.Store.NO)); for(int i=0;i doc, Analyzer analyzer) - throws CorruptIndexException, IOException { + throws IOException { super.updateDocument(term, doc, analyzer); try { if (waitAfterUpdate) { @@ -398,7 +396,7 @@ public class TestNRTManager extends ThreadedIndexingAndSearchingTestCase { final SearcherFactory theEvilOne = new SearcherFactory() { @Override - public IndexSearcher newSearcher(IndexReader ignored) throws IOException { + public IndexSearcher newSearcher(IndexReader ignored) { return new IndexSearcher(other); } }; diff --git a/lucene/core/src/test/org/apache/lucene/search/TestNot.java b/lucene/core/src/test/org/apache/lucene/search/TestNot.java index 5b909dc..a7591b3 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestNot.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestNot.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,6 +17,7 @@ package org.apache.lucene.search; * limitations under the License. */ +import org.apache.lucene.document.Field; import org.apache.lucene.index.Term; import org.apache.lucene.util.LuceneTestCase; @@ -24,7 +25,6 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.store.Directory; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; /** Similarity unit test. * @@ -37,7 +37,7 @@ public class TestNot extends LuceneTestCase { RandomIndexWriter writer = new RandomIndexWriter(random(), store); Document d1 = new Document(); - d1.add(newField("field", "a b", TextField.TYPE_STORED)); + d1.add(newTextField("field", "a b", Field.Store.YES)); writer.addDocument(d1); IndexReader reader = writer.getReader(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java b/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java index cdb966a..456ec91 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,10 +19,12 @@ package org.apache.lucene.search; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.FloatField; import org.apache.lucene.document.IntField; import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.RandomIndexWriter; @@ -61,7 +63,7 @@ public class TestNumericRangeQuery32 extends LuceneTestCase { .setMaxBufferedDocs(_TestUtil.nextInt(random(), 100, 1000)) .setMergePolicy(newLogMergePolicy())); - final FieldType storedInt = new FieldType(IntField.TYPE); + final FieldType storedInt = new FieldType(IntField.TYPE_NOT_STORED); storedInt.setStored(true); storedInt.freeze(); @@ -77,7 +79,7 @@ public class TestNumericRangeQuery32 extends LuceneTestCase { final FieldType storedIntNone = new FieldType(storedInt); storedIntNone.setNumericPrecisionStep(Integer.MAX_VALUE); - final FieldType unstoredInt = IntField.TYPE; + final FieldType unstoredInt = IntField.TYPE_NOT_STORED; final FieldType unstoredInt8 = new FieldType(unstoredInt); unstoredInt8.setNumericPrecisionStep(8); @@ -302,29 +304,29 @@ public class TestNumericRangeQuery32 extends LuceneTestCase { RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random()))); Document doc = new Document(); - doc.add(new FloatField("float", Float.NEGATIVE_INFINITY)); - doc.add(new IntField("int", Integer.MIN_VALUE)); + doc.add(new FloatField("float", Float.NEGATIVE_INFINITY, Field.Store.NO)); + doc.add(new IntField("int", Integer.MIN_VALUE, Field.Store.NO)); writer.addDocument(doc); doc = new Document(); - doc.add(new FloatField("float", Float.POSITIVE_INFINITY)); - doc.add(new IntField("int", Integer.MAX_VALUE)); + doc.add(new FloatField("float", Float.POSITIVE_INFINITY, Field.Store.NO)); + doc.add(new IntField("int", Integer.MAX_VALUE, Field.Store.NO)); writer.addDocument(doc); doc = new Document(); - doc.add(new FloatField("float", 0.0f)); - doc.add(new IntField("int", 0)); + doc.add(new FloatField("float", 0.0f, Field.Store.NO)); + doc.add(new IntField("int", 0, Field.Store.NO)); writer.addDocument(doc); for (float f : TestNumericUtils.FLOAT_NANs) { doc = new Document(); - doc.add(new FloatField("float", f)); + doc.add(new FloatField("float", f, Field.Store.NO)); writer.addDocument(doc); } writer.close(); - IndexReader r = IndexReader.open(dir); + IndexReader r = DirectoryReader.open(dir); IndexSearcher s = new IndexSearcher(r); Query q=NumericRangeQuery.newIntRange("int", null, null, true, true); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java b/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java index f228e51..82dc587 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,9 +20,11 @@ package org.apache.lucene.search; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.DoubleField; +import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.LongField; import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.RandomIndexWriter; @@ -61,7 +63,7 @@ public class TestNumericRangeQuery64 extends LuceneTestCase { .setMaxBufferedDocs(_TestUtil.nextInt(random(), 100, 1000)) .setMergePolicy(newLogMergePolicy())); - final FieldType storedLong = new FieldType(LongField.TYPE); + final FieldType storedLong = new FieldType(LongField.TYPE_NOT_STORED); storedLong.setStored(true); storedLong.freeze(); @@ -80,7 +82,7 @@ public class TestNumericRangeQuery64 extends LuceneTestCase { final FieldType storedLongNone = new FieldType(storedLong); storedLongNone.setNumericPrecisionStep(Integer.MAX_VALUE); - final FieldType unstoredLong = LongField.TYPE; + final FieldType unstoredLong = LongField.TYPE_NOT_STORED; final FieldType unstoredLong8 = new FieldType(unstoredLong); unstoredLong8.setNumericPrecisionStep(8); @@ -327,29 +329,29 @@ public class TestNumericRangeQuery64 extends LuceneTestCase { RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); Document doc = new Document(); - doc.add(new DoubleField("double", Double.NEGATIVE_INFINITY)); - doc.add(new LongField("long", Long.MIN_VALUE)); + doc.add(new DoubleField("double", Double.NEGATIVE_INFINITY, Field.Store.NO)); + doc.add(new LongField("long", Long.MIN_VALUE, Field.Store.NO)); writer.addDocument(doc); doc = new Document(); - doc.add(new DoubleField("double", Double.POSITIVE_INFINITY)); - doc.add(new LongField("long", Long.MAX_VALUE)); + doc.add(new DoubleField("double", Double.POSITIVE_INFINITY, Field.Store.NO)); + doc.add(new LongField("long", Long.MAX_VALUE, Field.Store.NO)); writer.addDocument(doc); doc = new Document(); - doc.add(new DoubleField("double", 0.0)); - doc.add(new LongField("long", 0L)); + doc.add(new DoubleField("double", 0.0, Field.Store.NO)); + doc.add(new LongField("long", 0L, Field.Store.NO)); writer.addDocument(doc); for (double d : TestNumericUtils.DOUBLE_NANs) { doc = new Document(); - doc.add(new DoubleField("double", d)); + doc.add(new DoubleField("double", d, Field.Store.NO)); writer.addDocument(doc); } writer.close(); - IndexReader r = IndexReader.open(dir); + IndexReader r = DirectoryReader.open(dir); IndexSearcher s = new IndexSearcher(r); Query q=NumericRangeQuery.newLongRange("long", null, null, true, true); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPhrasePrefixQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestPhrasePrefixQuery.java index 551f15b..8fe492e 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPhrasePrefixQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPhrasePrefixQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,9 +17,9 @@ package org.apache.lucene.search; * limitations under the License. */ +import org.apache.lucene.document.Field; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.IndexReader; @@ -47,11 +47,11 @@ public class TestPhrasePrefixQuery extends LuceneTestCase { Document doc3 = new Document(); Document doc4 = new Document(); Document doc5 = new Document(); - doc1.add(newField("body", "blueberry pie", TextField.TYPE_STORED)); - doc2.add(newField("body", "blueberry strudel", TextField.TYPE_STORED)); - doc3.add(newField("body", "blueberry pizza", TextField.TYPE_STORED)); - doc4.add(newField("body", "blueberry chewing gum", TextField.TYPE_STORED)); - doc5.add(newField("body", "piccadilly circus", TextField.TYPE_STORED)); + doc1.add(newTextField("body", "blueberry pie", Field.Store.YES)); + doc2.add(newTextField("body", "blueberry strudel", Field.Store.YES)); + doc3.add(newTextField("body", "blueberry pizza", Field.Store.YES)); + doc4.add(newTextField("body", "blueberry chewing gum", Field.Store.YES)); + doc5.add(newTextField("body", "piccadilly circus", Field.Store.YES)); writer.addDocument(doc1); writer.addDocument(doc2); writer.addDocument(doc3); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java index 3b51b5d..f59e3de 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -72,19 +72,19 @@ public class TestPhraseQuery extends LuceneTestCase { RandomIndexWriter writer = new RandomIndexWriter(random(), directory, analyzer); Document doc = new Document(); - doc.add(newField("field", "one two three four five", TextField.TYPE_STORED)); - doc.add(newField("repeated", "this is a repeated field - first part", TextField.TYPE_STORED)); - IndexableField repeatedField = newField("repeated", "second part of a repeated field", TextField.TYPE_STORED); + doc.add(newTextField("field", "one two three four five", Field.Store.YES)); + doc.add(newTextField("repeated", "this is a repeated field - first part", Field.Store.YES)); + IndexableField repeatedField = newTextField("repeated", "second part of a repeated field", Field.Store.YES); doc.add(repeatedField); - doc.add(newField("palindrome", "one two three two one", TextField.TYPE_STORED)); + doc.add(newTextField("palindrome", "one two three two one", Field.Store.YES)); writer.addDocument(doc); doc = new Document(); - doc.add(newField("nonexist", "phrase exist notexist exist found", TextField.TYPE_STORED)); + doc.add(newTextField("nonexist", "phrase exist notexist exist found", Field.Store.YES)); writer.addDocument(doc); doc = new Document(); - doc.add(newField("nonexist", "phrase exist notexist exist found", TextField.TYPE_STORED)); + doc.add(newTextField("nonexist", "phrase exist notexist exist found", Field.Store.YES)); writer.addDocument(doc); reader = writer.getReader(); @@ -226,7 +226,7 @@ public class TestPhraseQuery extends LuceneTestCase { RandomIndexWriter writer = new RandomIndexWriter(random(), directory, newIndexWriterConfig( Version.LUCENE_40, stopAnalyzer)); Document doc = new Document(); - doc.add(newField("field", "the stop words are here", TextField.TYPE_STORED)); + doc.add(newTextField("field", "the stop words are here", Field.Store.YES)); writer.addDocument(doc); IndexReader reader = writer.getReader(); writer.close(); @@ -260,12 +260,12 @@ public class TestPhraseQuery extends LuceneTestCase { RandomIndexWriter writer = new RandomIndexWriter(random(), directory); Document doc = new Document(); - doc.add(newField("source", "marketing info", TextField.TYPE_STORED)); + doc.add(newTextField("source", "marketing info", Field.Store.YES)); writer.addDocument(doc); doc = new Document(); - doc.add(newField("contents", "foobar", TextField.TYPE_STORED)); - doc.add(newField("source", "marketing info", TextField.TYPE_STORED)); + doc.add(newTextField("contents", "foobar", Field.Store.YES)); + doc.add(newTextField("source", "marketing info", Field.Store.YES)); writer.addDocument(doc); IndexReader reader = writer.getReader(); @@ -295,15 +295,15 @@ public class TestPhraseQuery extends LuceneTestCase { writer = new RandomIndexWriter(random(), directory, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())).setOpenMode(OpenMode.CREATE)); doc = new Document(); - doc.add(newField("contents", "map entry woo", TextField.TYPE_STORED)); + doc.add(newTextField("contents", "map entry woo", Field.Store.YES)); writer.addDocument(doc); doc = new Document(); - doc.add(newField("contents", "woo map entry", TextField.TYPE_STORED)); + doc.add(newTextField("contents", "woo map entry", Field.Store.YES)); writer.addDocument(doc); doc = new Document(); - doc.add(newField("contents", "map foobarword entry woo", TextField.TYPE_STORED)); + doc.add(newTextField("contents", "map foobarword entry woo", Field.Store.YES)); writer.addDocument(doc); reader = writer.getReader(); @@ -348,15 +348,15 @@ public class TestPhraseQuery extends LuceneTestCase { .setSimilarity(new DefaultSimilarity())); Document doc = new Document(); - doc.add(newField("field", "foo firstname lastname foo", TextField.TYPE_STORED)); + doc.add(newTextField("field", "foo firstname lastname foo", Field.Store.YES)); writer.addDocument(doc); Document doc2 = new Document(); - doc2.add(newField("field", "foo firstname zzz lastname foo", TextField.TYPE_STORED)); + doc2.add(newTextField("field", "foo firstname zzz lastname foo", Field.Store.YES)); writer.addDocument(doc2); Document doc3 = new Document(); - doc3.add(newField("field", "foo firstname zzz yyy lastname foo", TextField.TYPE_STORED)); + doc3.add(newTextField("field", "foo firstname zzz yyy lastname foo", Field.Store.YES)); writer.addDocument(doc3); IndexReader reader = writer.getReader(); @@ -611,7 +611,7 @@ public class TestPhraseQuery extends LuceneTestCase { RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).setMergePolicy(newLogMergePolicy())); List> docs = new ArrayList>(); Document d = new Document(); - Field f = newField("f", "", TextField.TYPE_UNSTORED); + Field f = newTextField("f", "", Field.Store.NO); d.add(f); Random r = random(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java b/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java index 9196e9c..89571d6 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -27,6 +27,7 @@ import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.MultiFields; @@ -61,7 +62,7 @@ public class TestPositionIncrement extends LuceneTestCase { return new TokenStreamComponents(new Tokenizer(reader) { // TODO: use CannedTokenStream private final String[] TOKENS = {"1", "2", "3", "4", "5"}; - private final int[] INCREMENTS = {0, 2, 1, 0, 1}; + private final int[] INCREMENTS = {1, 2, 1, 0, 1}; private int i = 0; PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class); @@ -91,7 +92,7 @@ public class TestPositionIncrement extends LuceneTestCase { Directory store = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), store, analyzer); Document d = new Document(); - d.add(newField("field", "bogus", TextField.TYPE_STORED)); + d.add(newTextField("field", "bogus", Field.Store.YES)); writer.addDocument(d); IndexReader reader = writer.getReader(); writer.close(); @@ -207,7 +208,7 @@ public class TestPositionIncrement extends LuceneTestCase { RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new MockPayloadAnalyzer()); Document doc = new Document(); doc.add(new TextField("content", new StringReader( - "a a b c d e a f g h i j a b k k"))); + "a a b c d e a f g h i j a b k k"), Field.Store.NO)); writer.addDocument(doc); final IndexReader readerFromWriter = writer.getReader(); @@ -222,8 +223,7 @@ public class TestPositionIncrement extends LuceneTestCase { assertTrue(tp.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); // "a" occurs 4 times assertEquals(4, tp.freq()); - int expected = 0; - assertEquals(expected, tp.nextPosition()); + assertEquals(0, tp.nextPosition()); assertEquals(1, tp.nextPosition()); assertEquals(3, tp.nextPosition()); assertEquals(6, tp.nextPosition()); @@ -254,7 +254,7 @@ public class TestPositionIncrement extends LuceneTestCase { for (byte[] bytes : payloads) { count++; if (VERBOSE) { - System.out.println(" payload: " + new String(bytes)); + System.out.println(" payload: " + new String(bytes, "UTF-8")); } } } @@ -281,7 +281,7 @@ public class TestPositionIncrement extends LuceneTestCase { Collection pls = psu.getPayloadsForQuery(snq); count = pls.size(); for (byte[] bytes : pls) { - String s = new String(bytes); + String s = new String(bytes, "UTF-8"); //System.out.println(s); sawZero |= s.equals("pos: 0"); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPositiveScoresOnlyCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestPositiveScoresOnlyCollector.java index 3a4c075..4fae5ff 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPositiveScoresOnlyCollector.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPositiveScoresOnlyCollector.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -35,17 +35,17 @@ public class TestPositiveScoresOnlyCollector extends LuceneTestCase { super(weight); } - @Override public float score() throws IOException { + @Override public float score() { return idx == scores.length ? Float.NaN : scores[idx]; } @Override public int docID() { return idx; } - @Override public int nextDoc() throws IOException { + @Override public int nextDoc() { return ++idx != scores.length ? idx : NO_MORE_DOCS; } - @Override public int advance(int target) throws IOException { + @Override public int advance(int target) { idx = target; return idx < scores.length ? idx : NO_MORE_DOCS; } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPrefixFilter.java b/lucene/core/src/test/org/apache/lucene/search/TestPrefixFilter.java index 1064d24..b41b6dd 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPrefixFilter.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPrefixFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,13 +17,13 @@ package org.apache.lucene.search; * limitations under the License. */ +import org.apache.lucene.document.Field; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.store.Directory; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.document.Document; -import org.apache.lucene.document.StringField; /** * Tests {@link PrefixFilter} class. @@ -40,7 +40,7 @@ public class TestPrefixFilter extends LuceneTestCase { RandomIndexWriter writer = new RandomIndexWriter(random(), directory); for (int i = 0; i < categories.length; i++) { Document doc = new Document(); - doc.add(newField("category", categories[i], StringField.TYPE_STORED)); + doc.add(newStringField("category", categories[i], Field.Store.YES)); writer.addDocument(doc); } IndexReader reader = writer.getReader(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPrefixInBooleanQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestPrefixInBooleanQuery.java index 727a6a6..b8a0e11 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPrefixInBooleanQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPrefixInBooleanQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,7 +19,6 @@ package org.apache.lucene.search; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; @@ -28,7 +27,6 @@ import org.apache.lucene.util.LuceneTestCase; import org.junit.AfterClass; import org.junit.BeforeClass; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeaks; /** * https://issues.apache.org/jira/browse/LUCENE-1974 @@ -52,7 +50,7 @@ public class TestPrefixInBooleanQuery extends LuceneTestCase { RandomIndexWriter writer = new RandomIndexWriter(random(), directory); Document doc = new Document(); - Field field = newField(FIELD, "meaninglessnames", StringField.TYPE_UNSTORED); + Field field = newStringField(FIELD, "meaninglessnames", Field.Store.NO); doc.add(field); for (int i = 0; i < 5137; ++i) { diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPrefixQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestPrefixQuery.java index f149cd9..06f0595 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPrefixQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPrefixQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,6 +17,7 @@ package org.apache.lucene.search; * limitations under the License. */ +import org.apache.lucene.document.Field; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.index.IndexReader; @@ -25,7 +26,6 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.document.Document; -import org.apache.lucene.document.StringField; /** * Tests {@link PrefixQuery} class. @@ -41,7 +41,7 @@ public class TestPrefixQuery extends LuceneTestCase { RandomIndexWriter writer = new RandomIndexWriter(random(), directory); for (int i = 0; i < categories.length; i++) { Document doc = new Document(); - doc.add(newField("category", categories[i], StringField.TYPE_STORED)); + doc.add(newStringField("category", categories[i], Field.Store.YES)); writer.addDocument(doc); } IndexReader reader = writer.getReader(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPrefixRandom.java b/lucene/core/src/test/org/apache/lucene/search/TestPrefixRandom.java index 2699293..6a4228f 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPrefixRandom.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPrefixRandom.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,10 +21,8 @@ import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; -import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; import org.apache.lucene.index.FilteredTermsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; @@ -56,13 +54,10 @@ public class TestPrefixRandom extends LuceneTestCase { .setMaxBufferedDocs(_TestUtil.nextInt(random(), 50, 1000))); Document doc = new Document(); - Field field = newField("field", "", StringField.TYPE_UNSTORED); + Field field = newStringField("field", "", Field.Store.NO); doc.add(field); - // we generate aweful prefixes: good for testing. - // but for preflex codec, the test can be very slow, so use less iterations. - final String codec = Codec.getDefault().getName(); - int num = codec.equals("Lucene3x") ? 200 * RANDOM_MULTIPLIER : atLeast(1000); + int num = atLeast(1000); for (int i = 0; i < num; i++) { field.setStringValue(_TestUtil.randomUnicodeString(random(), 10)); writer.addDocument(doc); @@ -96,7 +91,7 @@ public class TestPrefixRandom extends LuceneTestCase { private class SimplePrefixTermsEnum extends FilteredTermsEnum { private final BytesRef prefix; - private SimplePrefixTermsEnum(TermsEnum tenum, BytesRef prefix) throws IOException { + private SimplePrefixTermsEnum(TermsEnum tenum, BytesRef prefix) { super(tenum); this.prefix = prefix; setInitialSeekTerm(new BytesRef("")); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestQueryWrapperFilter.java b/lucene/core/src/test/org/apache/lucene/search/TestQueryWrapperFilter.java index 38fb547..ae22278 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestQueryWrapperFilter.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestQueryWrapperFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,8 +21,6 @@ import java.util.Set; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; @@ -37,7 +35,7 @@ public class TestQueryWrapperFilter extends LuceneTestCase { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); - doc.add(newField("field", "value", TextField.TYPE_UNSTORED)); + doc.add(newTextField("field", "value", Field.Store.NO)); writer.addDocument(doc); IndexReader reader = writer.getReader(); writer.close(); @@ -100,9 +98,9 @@ public class TestQueryWrapperFilter extends LuceneTestCase { } else { v = "b"; } - final Field f = newField("field", v, StringField.TYPE_UNSTORED); + final Field f = newStringField("field", v, Field.Store.NO); doc.add(f); - doc.add(newField("id", ""+i, StringField.TYPE_STORED)); + doc.add(newStringField("id", ""+i, Field.Store.YES)); w.addDocument(doc); } @@ -131,7 +129,7 @@ public class TestQueryWrapperFilter extends LuceneTestCase { RandomIndexWriter writer = new RandomIndexWriter(random(), dir); for (int i = 0; i < 1000; i++) { Document doc = new Document(); - doc.add(newField("field", English.intToEnglish(i), StringField.TYPE_UNSTORED)); + doc.add(newStringField("field", English.intToEnglish(i), Field.Store.NO)); writer.addDocument(doc); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java index 6a7d020..870db4f 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,7 +21,7 @@ import java.io.IOException; import java.util.Arrays; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; @@ -48,9 +48,7 @@ public class TestRegexpQuery extends LuceneTestCase { directory = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), directory); Document doc = new Document(); - doc.add(newField(FN, - "the quick brown fox jumps over the lazy ??? dog 493432 49344", - TextField.TYPE_UNSTORED)); + doc.add(newTextField(FN, "the quick brown fox jumps over the lazy ??? dog 493432 49344", Field.Store.NO)); writer.addDocument(doc); reader = writer.getReader(); writer.close(); @@ -104,7 +102,7 @@ public class TestRegexpQuery extends LuceneTestCase { BasicAutomata.makeString("brown"), BasicAutomata.makeString("bob"))); - public Automaton getAutomaton(String name) throws IOException { + public Automaton getAutomaton(String name) { if (name.equals("quickBrown")) return quickBrownAutomaton; else return null; } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestRegexpRandom.java b/lucene/core/src/test/org/apache/lucene/search/TestRegexpRandom.java index 8916c0b..f156262 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestRegexpRandom.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestRegexpRandom.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/search/TestRegexpRandom2.java b/lucene/core/src/test/org/apache/lucene/search/TestRegexpRandom2.java index dddf443..54ab48a 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestRegexpRandom2.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestRegexpRandom2.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -24,10 +24,8 @@ import java.util.List; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; -import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; import org.apache.lucene.index.FilteredTermsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; @@ -66,7 +64,7 @@ public class TestRegexpRandom2 extends LuceneTestCase { newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)) .setMaxBufferedDocs(_TestUtil.nextInt(random(), 50, 1000))); Document doc = new Document(); - Field field = newField(fieldName, "", StringField.TYPE_UNSTORED); + Field field = newStringField(fieldName, "", Field.Store.NO); doc.add(field); List terms = new ArrayList(); int num = atLeast(200); @@ -118,7 +116,7 @@ public class TestRegexpRandom2 extends LuceneTestCase { CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton); CharsRef utf16 = new CharsRef(10); - private SimpleAutomatonTermsEnum(TermsEnum tenum) throws IOException { + private SimpleAutomatonTermsEnum(TermsEnum tenum) { super(tenum); setInitialSeekTerm(new BytesRef("")); } @@ -139,9 +137,7 @@ public class TestRegexpRandom2 extends LuceneTestCase { /** test a bunch of random regular expressions */ public void testRegexps() throws Exception { - // we generate aweful regexps: good for testing. - // but for preflex codec, the test can be very slow, so use less iterations. - int num = Codec.getDefault().getName().equals("Lucene3x") ? 100 * RANDOM_MULTIPLIER : atLeast(1000); + int num = atLeast(1000); for (int i = 0; i < num; i++) { String reg = AutomatonTestUtil.randomRegexp(random()); if (VERBOSE) { diff --git a/lucene/core/src/test/org/apache/lucene/search/TestScoreCachingWrappingScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestScoreCachingWrappingScorer.java index 55ddd74..f9823ca 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestScoreCachingWrappingScorer.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestScoreCachingWrappingScorer.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -37,7 +37,7 @@ public class TestScoreCachingWrappingScorer extends LuceneTestCase { super(weight); } - @Override public float score() throws IOException { + @Override public float score() { // advance idx on purpose, so that consecutive calls to score will get // different results. This is to emulate computation of a score. If // ScoreCachingWrappingScorer is used, this should not be called more than @@ -47,11 +47,11 @@ public class TestScoreCachingWrappingScorer extends LuceneTestCase { @Override public int docID() { return doc; } - @Override public int nextDoc() throws IOException { + @Override public int nextDoc() { return ++doc < scores.length ? doc : NO_MORE_DOCS; } - @Override public int advance(int target) throws IOException { + @Override public int advance(int target) { doc = target; return doc < scores.length ? doc : NO_MORE_DOCS; } @@ -85,11 +85,10 @@ public class TestScoreCachingWrappingScorer extends LuceneTestCase { ++idx; } - @Override public void setNextReader(AtomicReaderContext context) - throws IOException { + @Override public void setNextReader(AtomicReaderContext context) { } - @Override public void setScorer(Scorer scorer) throws IOException { + @Override public void setScorer(Scorer scorer) { this.scorer = new ScoreCachingWrappingScorer(scorer); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java b/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java index 4953c39..197d02b 100755 --- a/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java @@ -1,5 +1,6 @@ package org.apache.lucene.search; +import org.apache.lucene.document.Field; import org.apache.lucene.util.Bits; import org.apache.lucene.util.DocIdBitSet; import org.apache.lucene.util.LuceneTestCase; @@ -8,6 +9,7 @@ import java.util.BitSet; import java.io.IOException; import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; @@ -15,9 +17,8 @@ import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.store.Directory; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.StringField; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -51,7 +52,7 @@ public class TestScorerPerf extends LuceneTestCase { IndexWriter iw = new IndexWriter(d, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random()))); iw.addDocument(new Document()); iw.close(); - r = IndexReader.open(d); + r = DirectoryReader.open(d); s = new IndexSearcher(r); } @@ -69,7 +70,7 @@ public class TestScorerPerf extends LuceneTestCase { Document d = new Document(); for (int j=0; j subs = ((CompositeReader) mockSearcher.getIndexReader()).getSequentialSubReaders(); + assertEquals(numNodes, subs.size()); int docCount = 0; for(int nodeID=0;nodeID 0); - } - } else { - assertFalse(interval.payloadAvailable()); - } - } - - /** - * this test indexes random numbers within a range into a field and checks - * their occurrences by searching for a number from that range selected at - * random. All positions for that number are safed up front and compared to - * the terms scorers positions. - * - */ - public void testRandomPositons() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); - int numDocs = 131; - int max = 1051; - int term = random().nextInt(max); - Integer[][] positionsInDoc = new Integer[numDocs][]; - for (int i = 0; i < numDocs; i++) { - Document doc = new Document(); - ArrayList positions = new ArrayList(); - StringBuilder builder = new StringBuilder(); - for (int j = 0; j < 3049; j++) { - int nextInt = random().nextInt(max); - builder.append(nextInt).append(" "); - if (nextInt == term) { - positions.add(Integer.valueOf(j)); - } - } - doc.add(newField(fieldName, builder.toString(), TextField.TYPE_STORED)); - positionsInDoc[i] = positions.toArray(new Integer[0]); - writer.addDocument(doc); - } - - IndexReader reader = writer.getReader(); - writer.close(); - IndexSearcher searcher = new IndexSearcher(reader); - - for (int i = 0; i < 39 * RANDOM_MULTIPLIER; i++) { - TermQuery one = new TermQuery(new Term(fieldName, "" + term)); - IndexReaderContext topReaderContext = reader.getTopReaderContext(); - AtomicReaderContext[] leaves = topReaderContext.leaves(); - Weight weight = one.createWeight(searcher); - for (AtomicReaderContext atomicReaderContext : leaves) { - Scorer scorer = weight.scorer(atomicReaderContext, true, true, null); - assertNotNull(scorer); - int initDoc = 0; - int maxDoc = atomicReaderContext.reader().maxDoc(); - // initially advance or do next doc - if (random().nextBoolean()) { - initDoc = scorer.nextDoc(); - } else { - initDoc = scorer.advance(random().nextInt(maxDoc)); - } - // now run through the scorer and check if all positions are there... - do { - int docID = scorer.docID(); - if (docID == Scorer.NO_MORE_DOCS) { - break; - } - PositionIntervalIterator positions = scorer.positions(false, false); - Integer[] pos = positionsInDoc[atomicReaderContext.docBase + docID]; - - assertEquals((float) pos.length, positions.getScorer().freq(), 0.0f); - // number of positions read should be random - don't read all of them - // allways - final int howMany = random().nextInt(20) == 0 ? pos.length - - random().nextInt(pos.length) : pos.length; - PositionInterval interval = null; - assertEquals(scorer.docID(), positions.advanceTo(scorer.docID())); - for (int j = 0; j < howMany; j++) { - assertNotNull((interval = positions.next())); - assertEquals("iteration: " + i + " initDoc: " + initDoc + " doc: " - + docID + " base: " + atomicReaderContext.docBase - + " positions: " + Arrays.toString(pos), pos[j].intValue(), - interval.begin); - assertEquals(pos[j].intValue(), interval.end); - } - if (howMany == pos.length) { - assertNull((interval = positions.next())); - } - - if (random().nextInt(10) == 0) { // once is a while advance - scorer.advance(docID + 1 + random().nextInt((maxDoc - docID))); - } - - } while (scorer.nextDoc() != Scorer.NO_MORE_DOCS); - } - - } - reader.close(); - dir.close(); - } - - /** - * tests retrieval of positions for terms that have a large number of - * occurrences to force test of buffer refill during positions iteration. - */ - public void testLargeNumberOfPositions() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir, - - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); - int howMany = 1000; - for (int i = 0; i < 39; i++) { - Document doc = new Document(); - StringBuilder builder = new StringBuilder(); - for (int j = 0; j < howMany; j++) { - if (j % 2 == 0) { - builder.append("even "); - } else { - builder.append("odd "); - } - } - doc.add(newField(fieldName, builder.toString(),TextField.TYPE_STORED)); - writer.addDocument(doc); - } - - // now do seaches - IndexReader reader = writer.getReader(); - writer.close(); - IndexSearcher searcher = new IndexSearcher(reader); - - for (int i = 0; i < 39 * RANDOM_MULTIPLIER; i++) { - TermQuery one = new TermQuery(new Term(fieldName, "even")); - IndexReaderContext topReaderContext = reader.getTopReaderContext(); - AtomicReaderContext[] leaves = topReaderContext.leaves(); - Weight weight = one.createWeight(searcher); - PositionInterval interval = null; - for (AtomicReaderContext atomicReaderContext : leaves) { - Scorer scorer = weight.scorer(atomicReaderContext, true, true, null); - assertNotNull(scorer); - - int initDoc = 0; - int maxDoc = atomicReaderContext.reader().maxDoc(); - // initially advance or do next doc - if (random().nextBoolean()) { - initDoc = scorer.nextDoc(); - } else { - initDoc = scorer.advance(random().nextInt(maxDoc)); - } - String msg = "Iteration: " + i + " initDoc: " + initDoc; - PositionIntervalIterator positions = scorer.positions(false, false); - assertEquals(howMany / 2.f, positions.getScorer().freq(), 0.0); - assertEquals(scorer.docID(), positions.advanceTo(scorer.docID())); - for (int j = 0; j < howMany; j += 2) { - assertNotNull("next returned nullat index: " + j + " with freq: " - + positions.getScorer().freq() + " -- " + msg, - (interval = positions.next())); - assertEquals("position missmatch index: " + j + " with freq: " - + positions.getScorer().freq() + " -- " + msg, j, interval.begin); - } - assertNull("next returned nonNull -- " + msg, - (interval = positions.next())); - - } - } - reader.close(); - dir.close(); - } - -} \ No newline at end of file diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermRangeFilter.java b/lucene/core/src/test/org/apache/lucene/search/TestTermRangeFilter.java index 3f44ca0..ff57a85 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTermRangeFilter.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTermRangeFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermRangeQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestTermRangeQuery.java index 397389f..4cfb0ab 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTermRangeQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTermRangeQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -24,8 +24,8 @@ import java.util.Set; import org.apache.lucene.analysis.*; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.document.Document; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.MultiFields; @@ -55,21 +55,21 @@ public class TestTermRangeQuery extends LuceneTestCase { public void testExclusive() throws Exception { Query query = TermRangeQuery.newStringRange("content", "A", "C", false, false); initializeIndex(new String[] {"A", "B", "C", "D"}); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs; assertEquals("A,B,C,D, only B in range", 1, hits.length); reader.close(); initializeIndex(new String[] {"A", "B", "D"}); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); searcher = new IndexSearcher(reader); hits = searcher.search(query, null, 1000).scoreDocs; assertEquals("A,B,D, only B in range", 1, hits.length); reader.close(); addDoc("C"); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); searcher = new IndexSearcher(reader); hits = searcher.search(query, null, 1000).scoreDocs; assertEquals("C added, still only B in range", 1, hits.length); @@ -80,21 +80,21 @@ public class TestTermRangeQuery extends LuceneTestCase { Query query = TermRangeQuery.newStringRange("content", "A", "C", true, true); initializeIndex(new String[]{"A", "B", "C", "D"}); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs; assertEquals("A,B,C,D - A,B,C in range", 3, hits.length); reader.close(); initializeIndex(new String[]{"A", "B", "D"}); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); searcher = new IndexSearcher(reader); hits = searcher.search(query, null, 1000).scoreDocs; assertEquals("A,B,D - A and B in range", 2, hits.length); reader.close(); addDoc("C"); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); searcher = new IndexSearcher(reader); hits = searcher.search(query, null, 1000).scoreDocs; assertEquals("C added - A, B, C in range", 3, hits.length); @@ -103,7 +103,7 @@ public class TestTermRangeQuery extends LuceneTestCase { public void testAllDocs() throws Exception { initializeIndex(new String[]{"A", "B", "C", "D"}); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); TermRangeQuery query = new TermRangeQuery("content", null, null, true, true); Terms terms = MultiFields.getTerms(searcher.getIndexReader(), "content"); @@ -127,7 +127,7 @@ public class TestTermRangeQuery extends LuceneTestCase { public void testTopTermsRewrite() throws Exception { initializeIndex(new String[]{"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K"}); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); TermRangeQuery query = TermRangeQuery.newStringRange("content", "B", "J", true, true); checkBooleanTerms(searcher, query, "B", "C", "D", "E", "F", "G", "H", "I", "J"); @@ -262,8 +262,8 @@ public class TestTermRangeQuery extends LuceneTestCase { private void insertDoc(IndexWriter writer, String content) throws IOException { Document doc = new Document(); - doc.add(newField("id", "id" + docCount, StringField.TYPE_STORED)); - doc.add(newField("content", content, TextField.TYPE_UNSTORED)); + doc.add(newStringField("id", "id" + docCount, Field.Store.YES)); + doc.add(newTextField("content", content, Field.Store.NO)); writer.addDocument(doc); docCount++; @@ -276,7 +276,7 @@ public class TestTermRangeQuery extends LuceneTestCase { Query query = TermRangeQuery.newStringRange("content", null, "C", false, false); initializeIndex(new String[] {"A", "B", "", "C", "D"}, analyzer); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); int numHits = searcher.search(query, null, 1000).totalHits; // When Lucene-38 is fixed, use the assert on the next line: @@ -286,7 +286,7 @@ public class TestTermRangeQuery extends LuceneTestCase { reader.close(); initializeIndex(new String[] {"A", "B", "", "D"}, analyzer); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); searcher = new IndexSearcher(reader); numHits = searcher.search(query, null, 1000).totalHits; // When Lucene-38 is fixed, use the assert on the next line: @@ -295,7 +295,7 @@ public class TestTermRangeQuery extends LuceneTestCase { //assertEquals("A,B,,D => A, B & are in range", 2, hits.length()); reader.close(); addDoc("C"); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); searcher = new IndexSearcher(reader); numHits = searcher.search(query, null, 1000).totalHits; // When Lucene-38 is fixed, use the assert on the next line: @@ -311,7 +311,7 @@ public class TestTermRangeQuery extends LuceneTestCase { Analyzer analyzer = new SingleCharAnalyzer(); Query query = TermRangeQuery.newStringRange("content", null, "C", true, true); initializeIndex(new String[]{"A", "B", "","C", "D"}, analyzer); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); int numHits = searcher.search(query, null, 1000).totalHits; // When Lucene-38 is fixed, use the assert on the next line: @@ -320,7 +320,7 @@ public class TestTermRangeQuery extends LuceneTestCase { //assertEquals("A,B,,C,D => A,B,,C in range", 3, hits.length()); reader.close(); initializeIndex(new String[]{"A", "B", "", "D"}, analyzer); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); searcher = new IndexSearcher(reader); numHits = searcher.search(query, null, 1000).totalHits; // When Lucene-38 is fixed, use the assert on the next line: @@ -329,7 +329,7 @@ public class TestTermRangeQuery extends LuceneTestCase { //assertEquals("A,B,,D => A, B and in range", 2, hits.length()); reader.close(); addDoc("C"); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); searcher = new IndexSearcher(reader); numHits = searcher.search(query, null, 1000).totalHits; // When Lucene-38 is fixed, use the assert on the next line: diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java index 1e01f3a..f5b62a3 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,7 +23,7 @@ import java.util.List; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; @@ -54,7 +54,7 @@ public class TestTermScorer extends LuceneTestCase { for (int i = 0; i < values.length; i++) { Document doc = new Document(); doc - .add(newField(FIELD, values[i], TextField.TYPE_STORED)); + .add(newTextField(FIELD, values[i], Field.Store.YES)); writer.addDocument(doc); } indexReader = SlowCompositeReaderWrapper.wrap(writer.getReader()); @@ -89,7 +89,7 @@ public class TestTermScorer extends LuceneTestCase { private Scorer scorer; @Override - public void setScorer(Scorer scorer) throws IOException { + public void setScorer(Scorer scorer) { this.scorer = scorer; } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermVectors.java b/lucene/core/src/test/org/apache/lucene/search/TestTermVectors.java index e8cec93..02e2dd0 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTermVectors.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTermVectors.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -69,7 +69,7 @@ public class TestTermVectors extends LuceneTestCase { } doc.add(new Field("field", English.intToEnglish(i), ft)); //test no term vectors too - doc.add(new Field("noTV", English.intToEnglish(i), TextField.TYPE_STORED)); + doc.add(new TextField("noTV", English.intToEnglish(i), Field.Store.YES)); writer.addDocument(doc); } reader = writer.getReader(); @@ -341,7 +341,7 @@ public class TestTermVectors extends LuceneTestCase { } for (int i = 0; i < 100; i++) { Document doc = new Document(); - doc.add(newField("field", English.intToEnglish(i), TextField.TYPE_STORED)); + doc.add(newTextField("field", English.intToEnglish(i), Field.Store.YES)); writer.addDocument(doc); } if (VERBOSE) { @@ -401,7 +401,7 @@ public class TestTermVectors extends LuceneTestCase { ft5.setStoreTermVectorOffsets(true); ft5.setStoreTermVectorPositions(true); - doc.add(newField("field", "one", TextField.TYPE_STORED)); + doc.add(newTextField("field", "one", Field.Store.YES)); doc.add(newField("field", "one", ft2)); doc.add(newField("field", "one", ft3)); doc.add(newField("field", "one", ft4)); @@ -468,7 +468,7 @@ public class TestTermVectors extends LuceneTestCase { } private void verifyIndex(Directory dir) throws IOException { - IndexReader r = IndexReader.open(dir); + IndexReader r = DirectoryReader.open(dir); int numDocs = r.numDocs(); for (int i = 0; i < numDocs; i++) { assertNotNull("term vectors should not have been null for document " + i, r.getTermVectors(i).terms("c")); @@ -519,7 +519,7 @@ public class TestTermVectors extends LuceneTestCase { IndexWriter writer = createWriter(target); for (Directory dir : input) { - IndexReader r = IndexReader.open(dir); + IndexReader r = DirectoryReader.open(dir); writer.addIndexes(r); r.close(); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTimeLimitingCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestTimeLimitingCollector.java index 1e2695b..537e30f 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTimeLimitingCollector.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTimeLimitingCollector.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,7 +22,7 @@ import java.util.BitSet; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; @@ -116,7 +116,7 @@ public class TestTimeLimitingCollector extends LuceneTestCase { private void add(String value, RandomIndexWriter iw) throws IOException { Document d = new Document(); - d.add(newField(FIELD_NAME, value, TextField.TYPE_UNSTORED)); + d.add(newTextField(FIELD_NAME, value, Field.Store.NO)); iw.addDocument(d); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java index b9fccef..f8f728f 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -55,19 +55,18 @@ public class TestTopDocsCollector extends LuceneTestCase { } @Override - public void collect(int doc) throws IOException { + public void collect(int doc) { ++totalHits; pq.insertWithOverflow(new ScoreDoc(doc + base, scores[idx++])); } @Override - public void setNextReader(AtomicReaderContext context) - throws IOException { + public void setNextReader(AtomicReaderContext context) { base = context.docBase; } @Override - public void setScorer(Scorer scorer) throws IOException { + public void setScorer(Scorer scorer) { // Don't do anything. Assign scores in random } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java index fe4dd71..4c49c3d 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,32 +19,32 @@ package org.apache.lucene.search; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.FloatField; import org.apache.lucene.document.IntField; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.CompositeReaderContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.ReaderUtil; import org.apache.lucene.util._TestUtil; public class TestTopDocsMerge extends LuceneTestCase { private static class ShardSearcher extends IndexSearcher { - private final AtomicReaderContext[] ctx; + private final List ctx; public ShardSearcher(AtomicReaderContext ctx, IndexReaderContext parent) { super(parent); - this.ctx = new AtomicReaderContext[] {ctx}; + this.ctx = Collections.singletonList(ctx); } public void search(Weight weight, Collector collector) throws IOException { @@ -57,7 +57,7 @@ public class TestTopDocsMerge extends LuceneTestCase { @Override public String toString() { - return "ShardSearcher(" + ctx[0] + ")"; + return "ShardSearcher(" + ctx.get(0) + ")"; } } @@ -95,9 +95,9 @@ public class TestTopDocsMerge extends LuceneTestCase { for(int docIDX=0;docIDX leaves = topReaderContext.leaves(); + assertEquals(1, leaves.size()); + for (int i = 0; i < leaves.size(); i++) { + Scorer scorer = weight.scorer(leaves.get(i), true, true, leaves.get(i).reader().getLiveDocs()); { int nextDoc = scorer.nextDoc(); assertEquals(0, nextDoc); diff --git a/lucene/core/src/test/org/apache/lucene/search/positions/TestConjunctionPositionsIterator.java b/lucene/core/src/test/org/apache/lucene/search/positions/TestConjunctionPositionsIterator.java index 02c0634..457cf87 100644 --- a/lucene/core/src/test/org/apache/lucene/search/positions/TestConjunctionPositionsIterator.java +++ b/lucene/core/src/test/org/apache/lucene/search/positions/TestConjunctionPositionsIterator.java @@ -15,6 +15,7 @@ package org.apache.lucene.search.positions; * See the License for the specific language governing permissions and * limitations under the License. */ + import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; @@ -26,6 +27,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import java.io.IOException; +import java.util.List; public class TestConjunctionPositionsIterator extends LuceneTestCase { @@ -98,10 +100,10 @@ public class TestConjunctionPositionsIterator extends LuceneTestCase { query.add(new BooleanClause(new TermQuery(new Term("field", "hot!")), Occur.MUST)); Weight weight = query.createWeight(searcher); IndexReaderContext topReaderContext = searcher.getTopReaderContext(); - AtomicReaderContext[] leaves = topReaderContext.leaves(); - assertEquals(1, leaves.length); - for (int i = 0; i < leaves.length; i++) { - Scorer scorer = weight.scorer(leaves[0], true, true, leaves[0].reader().getLiveDocs()); + List leaves = topReaderContext.leaves(); + assertEquals(1, leaves.size()); + for (int i = 0; i < leaves.size(); i++) { + Scorer scorer = weight.scorer(leaves.get(i), true, true, leaves.get(i).reader().getLiveDocs()); { int nextDoc = scorer.nextDoc(); assertEquals(0, nextDoc); diff --git a/lucene/core/src/test/org/apache/lucene/search/positions/TestDisjunctionPositionIterator.java b/lucene/core/src/test/org/apache/lucene/search/positions/TestDisjunctionPositionIterator.java index 0c36bb6..48e14ed 100644 --- a/lucene/core/src/test/org/apache/lucene/search/positions/TestDisjunctionPositionIterator.java +++ b/lucene/core/src/test/org/apache/lucene/search/positions/TestDisjunctionPositionIterator.java @@ -11,6 +11,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import java.io.IOException; +import java.util.List; public class TestDisjunctionPositionIterator extends LuceneTestCase { private static final void addDocs(RandomIndexWriter writer) @@ -101,10 +102,10 @@ public class TestDisjunctionPositionIterator extends LuceneTestCase { Weight weight = query.createWeight(searcher); IndexReaderContext topReaderContext = searcher.getTopReaderContext(); - AtomicReaderContext[] leaves = topReaderContext.leaves(); - assertEquals(1, leaves.length); - Scorer scorer = weight.scorer(leaves[0], - true, true, leaves[0].reader().getLiveDocs()); + List leaves = topReaderContext.leaves(); + assertEquals(1, leaves.size()); + Scorer scorer = weight.scorer(leaves.get(0), + true, true, leaves.get(0).reader().getLiveDocs()); for (int i = 0; i < 2; i++) { @@ -156,11 +157,11 @@ public class TestDisjunctionPositionIterator extends LuceneTestCase { query.add(new BooleanClause(query2, Occur.SHOULD)); Weight weight = query.createWeight(searcher); IndexReaderContext topReaderContext = searcher.getTopReaderContext(); - AtomicReaderContext[] leaves = topReaderContext.leaves(); - assertEquals(1, leaves.length); - for (int i = 0; i < leaves.length; i++) { - Scorer scorer = weight.scorer(leaves[0], - true, true, leaves[0].reader().getLiveDocs()); + List leaves = topReaderContext.leaves(); + assertEquals(1, leaves.size()); + for (int i = 0; i < leaves.size(); i++) { + Scorer scorer = weight.scorer(leaves.get(i), + true, true, leaves.get(i).reader().getLiveDocs()); { int nextDoc = scorer.nextDoc(); assertEquals(0, nextDoc); diff --git a/lucene/core/src/test/org/apache/lucene/search/positions/TestOrderedConjunctionPositionsIterator.java b/lucene/core/src/test/org/apache/lucene/search/positions/TestOrderedConjunctionPositionsIterator.java index 45eb6e6..94b6985 100644 --- a/lucene/core/src/test/org/apache/lucene/search/positions/TestOrderedConjunctionPositionsIterator.java +++ b/lucene/core/src/test/org/apache/lucene/search/positions/TestOrderedConjunctionPositionsIterator.java @@ -12,6 +12,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import java.io.IOException; +import java.util.List; public class TestOrderedConjunctionPositionsIterator extends LuceneTestCase { @@ -103,10 +104,10 @@ public class TestOrderedConjunctionPositionsIterator extends LuceneTestCase { query.add(new BooleanClause(new TermQuery(new Term("field", "hot!")), Occur.MUST)); Weight weight = query.createWeight(searcher); IndexReaderContext topReaderContext = searcher.getTopReaderContext(); - AtomicReaderContext[] leaves = topReaderContext.leaves(); - assertEquals(1, leaves.length); - for (int i = 0; i < leaves.length; i++) { - Scorer scorer = weight.scorer(leaves[0], true, true, leaves[0].reader() + List leaves = topReaderContext.leaves(); + assertEquals(1, leaves.size()); + for (int i = 0; i < leaves.size(); i++) { + Scorer scorer = weight.scorer(leaves.get(i), true, true, leaves.get(i).reader() .getLiveDocs()); { int nextDoc = scorer.nextDoc(); diff --git a/lucene/core/src/test/org/apache/lucene/search/positions/TestPositionOffsets.java b/lucene/core/src/test/org/apache/lucene/search/positions/TestPositionOffsets.java index 7d7545b..6b9ae40 100644 --- a/lucene/core/src/test/org/apache/lucene/search/positions/TestPositionOffsets.java +++ b/lucene/core/src/test/org/apache/lucene/search/positions/TestPositionOffsets.java @@ -16,8 +16,6 @@ package org.apache.lucene.search.positions; * limitations under the License. */ -import java.io.IOException; - import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat; @@ -27,24 +25,15 @@ import org.apache.lucene.codecs.pulsing.Pulsing40PostingsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexReaderContext; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.Weight; +import org.apache.lucene.index.*; +import org.apache.lucene.search.*; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; +import java.io.IOException; +import java.util.List; + public class TestPositionOffsets extends LuceneTestCase { // What am I testing here? @@ -92,28 +81,32 @@ public class TestPositionOffsets extends LuceneTestCase { writer.addDocument(doc); } - public void testTermQueryWithOffsets() throws IOException { + private void testQuery(Query query, int[][] expectedOffsets) throws IOException { + testQuery(query, expectedOffsets, true); + } + + private void testQuery(Query query, int[][] expectedOffsets, boolean needsOffsets) throws IOException { Directory directory = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), directory, iwc); - addDocs(writer, true); + addDocs(writer, needsOffsets); IndexReader reader = writer.getReader(); IndexSearcher searcher = new IndexSearcher(reader); writer.close(); - Query query = new TermQuery(new Term("field", "porridge")); Weight weight = query.createWeight(searcher); IndexReaderContext topReaderContext = searcher.getTopReaderContext(); - AtomicReaderContext[] leaves = topReaderContext.leaves(); - assertEquals(1, leaves.length); - Scorer scorer = weight.scorer(leaves[0], - true, true, leaves[0].reader().getLiveDocs()); + List leaves = topReaderContext.leaves(); + assertEquals(1, leaves.size()); + Scorer scorer = weight.scorer(leaves.get(0), + true, true, leaves.get(0).reader().getLiveDocs()); int nextDoc = scorer.nextDoc(); assertEquals(0, nextDoc); - PositionIntervalIterator positions = scorer.positions(false, true); - int[] startOffsets = new int[] { 6, 26, 47, 164, 184 }; - int[] endOffsets = new int[] { 14, 34, 55, 172, 192 }; + PositionIntervalIterator positions = scorer.positions(false, needsOffsets); + + int startOffsets[] = expectedOffsets[0]; + int endOffsets[] = expectedOffsets[1]; assertEquals(0, positions.advanceTo(nextDoc)); for (int i = 0; i < startOffsets.length; i++) { @@ -128,79 +121,68 @@ public class TestPositionOffsets extends LuceneTestCase { directory.close(); } - public void testTermQueryWithoutOffsets() throws IOException { - Directory directory = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), directory, iwc); - addDocs(writer, false); - - IndexReader reader = writer.getReader(); - IndexSearcher searcher = new IndexSearcher(reader); - writer.close(); + public void testTermQueryWithOffsets() throws IOException { Query query = new TermQuery(new Term("field", "porridge")); + int[] startOffsets = new int[] { 6, 26, 47, 164, 184 }; + int[] endOffsets = new int[] { 14, 34, 55, 172, 192 }; + testQuery(query, new int[][] { startOffsets, endOffsets }); + } - Weight weight = query.createWeight(searcher); - IndexReaderContext topReaderContext = searcher.getTopReaderContext(); - AtomicReaderContext[] leaves = topReaderContext.leaves(); - assertEquals(1, leaves.length); - Scorer scorer = weight.scorer(leaves[0], - true, true, leaves[0].reader().getLiveDocs()); - - int nextDoc = scorer.nextDoc(); - assertEquals(0, nextDoc); - PositionIntervalIterator positions = scorer.positions(false, false); + public void testTermQueryWithoutOffsets() throws IOException { + Query query = new TermQuery(new Term("field", "porridge")); int[] startOffsets = new int[] { -1, -1, -1, -1, -1 }; int[] endOffsets = new int[] { -1, -1, -1, -1, -1 }; - - assertEquals(0, positions.advanceTo(nextDoc)); - for (int i = 0; i < startOffsets.length; i++) { - PositionIntervalIterator.PositionInterval interval = positions.next(); - assertEquals(startOffsets[i], interval.offsetBegin); - assertEquals(endOffsets[i], interval.offsetEnd); - } - - assertNull(positions.next()); - - reader.close(); - directory.close(); + testQuery(query, new int[][] { startOffsets, endOffsets }, false); } public void testBooleanQueryWithOffsets() throws IOException { - Directory directory = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), directory, iwc); - addDocs(writer, true); - - IndexReader reader = writer.getReader(); - IndexSearcher searcher = new IndexSearcher(reader); - writer.close(); BooleanQuery query = new BooleanQuery(); query.add(new BooleanClause(new TermQuery(new Term("field", "porridge")), BooleanClause.Occur.MUST)); query.add(new BooleanClause(new TermQuery(new Term("field", "nine")), BooleanClause.Occur.MUST)); - - Weight weight = query.createWeight(searcher); - IndexReaderContext topReaderContext = searcher.getTopReaderContext(); - AtomicReaderContext[] leaves = topReaderContext.leaves(); - assertEquals(1, leaves.length); - Scorer scorer = weight.scorer(leaves[0], - true, true, leaves[0].reader().getLiveDocs()); - - int nextDoc = scorer.nextDoc(); - assertEquals(0, nextDoc); - PositionIntervalIterator positions = scorer.positions(false, true); int[] startOffsetsConj = new int[] { 6, 26, 47, 67, 143}; int[] endOffsetsConj = new int[] { 71, 71, 71, 172, 172}; - assertEquals(0, positions.advanceTo(nextDoc)); - PositionIntervalIterator.PositionInterval interval; - int i = 0; - while((interval = positions.next()) != null) { - assertEquals(startOffsetsConj[i], interval.offsetBegin); - assertEquals(endOffsetsConj[i], interval.offsetEnd); - i++; - } - assertEquals(i, startOffsetsConj.length); - assertNull(positions.next()); + testQuery(query, new int[][] { startOffsetsConj, endOffsetsConj }); + } - reader.close(); - directory.close(); + public void testExactPhraseQuery() throws IOException { + PhraseQuery query = new PhraseQuery(); + query.add(new Term("field", "pease")); + query.add(new Term("field", "porridge")); + query.add(new Term("field", "hot!")); + int[] startOffsetsBlock = new int[] { 0, 158 }; + int[] endOffsetsBlock = new int[] { 19, 177 }; + testQuery(query, new int[][] { startOffsetsBlock, endOffsetsBlock }); + } + + public void testSloppyPhraseQuery() throws IOException { + PhraseQuery query = new PhraseQuery(); + query.add(new Term("field", "pease")); + query.add(new Term("field", "hot!")); + query.setSlop(1); + int[] startOffsetsBlock = new int[] { 0, 158 }; + int[] endOffsetsBlock = new int[] { 19, 177 }; + testQuery(query, new int[][] { startOffsetsBlock, endOffsetsBlock }); + } + + public void testManyTermSloppyPhraseQuery() throws IOException { + PhraseQuery query = new PhraseQuery(); + query.add(new Term("field", "pease")); + query.add(new Term("field", "porridge")); + query.add(new Term("field", "pot")); + query.setSlop(2); + int[] startOffsetsBlock = new int[] { 41 }; + int[] endOffsetsBlock = new int[] { 66 }; + testQuery(query, new int[][] { startOffsetsBlock, endOffsetsBlock }); + } + + public void testMultiTermPhraseQuery() throws IOException { + MultiPhraseQuery query = new MultiPhraseQuery(); + query.add(new Term("field", "pease")); + query.add(new Term("field", "porridge")); + query.add(new Term[] { new Term("field", "hot!"), new Term("field", "cold!")}); + int[] startOffsetsBlock = new int[] { 0, 20, 158, 178 }; + int[] endOffsetsBlock = new int[] { 19, 40, 177, 198 }; + testQuery(query, new int[][] { startOffsetsBlock, endOffsetsBlock }); } - + } \ No newline at end of file diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarity2.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarity2.java index e83c0cd..730877a 100644 --- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarity2.java +++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarity2.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,26 +17,21 @@ package org.apache.lucene.search.similarities; * limitations under the License. */ -import java.io.IOException; import java.util.ArrayList; import java.util.List; -import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.Collector; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; -import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.spans.SpanOrQuery; @@ -99,7 +94,7 @@ public class TestSimilarity2 extends LuceneTestCase { Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir); Document doc = new Document(); - doc.add(newField("foo", "bar", TextField.TYPE_UNSTORED)); + doc.add(newTextField("foo", "bar", Field.Store.NO)); iw.addDocument(doc); IndexReader ir = iw.getReader(); iw.close(); @@ -121,7 +116,7 @@ public class TestSimilarity2 extends LuceneTestCase { Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir); Document doc = new Document(); - doc.add(newField("foo", "bar", TextField.TYPE_UNSTORED)); + doc.add(newTextField("foo", "bar", Field.Store.NO)); iw.addDocument(doc); IndexReader ir = iw.getReader(); iw.close(); @@ -143,7 +138,7 @@ public class TestSimilarity2 extends LuceneTestCase { Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir); Document doc = new Document(); - FieldType ft = new FieldType(TextField.TYPE_UNSTORED); + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); ft.setOmitNorms(true); ft.freeze(); doc.add(newField("foo", "bar", ft)); @@ -167,7 +162,7 @@ public class TestSimilarity2 extends LuceneTestCase { Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir); Document doc = new Document(); - FieldType ft = new FieldType(TextField.TYPE_UNSTORED); + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); ft.setIndexOptions(IndexOptions.DOCS_ONLY); ft.freeze(); Field f = newField("foo", "bar", ft); @@ -192,7 +187,7 @@ public class TestSimilarity2 extends LuceneTestCase { Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir); Document doc = new Document(); - FieldType ft = new FieldType(TextField.TYPE_UNSTORED); + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); ft.setIndexOptions(IndexOptions.DOCS_ONLY); ft.setOmitNorms(true); ft.freeze(); @@ -222,7 +217,7 @@ public class TestSimilarity2 extends LuceneTestCase { Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir); Document doc = new Document(); - FieldType ft = new FieldType(TextField.TYPE_UNSTORED); + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); doc.add(newField("foo", "bar", ft)); iw.addDocument(doc); IndexReader ir = iw.getReader(); diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java index f53a83e..dc3017e 100644 --- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java +++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,8 +21,8 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexReader; @@ -48,7 +48,7 @@ import org.apache.lucene.util.LuceneTestCase; * items in the list. If a test case fails, the name of the Similarity that * caused the failure is returned as part of the assertion error message.

      *

      Unit testing is performed by constructing statistics manually and calling - * the {@link SimilarityBase#score(BasicStats, float, int)} method of the + * the {@link SimilarityBase#score(BasicStats, float, float)} method of the * Similarities. The statistics represent corner cases of corpus distributions. *

      *

      For the integration tests, a small (8-document) collection is indexed. The @@ -112,7 +112,7 @@ public class TestSimilarityBase extends LuceneTestCase { FieldType ft = new FieldType(TextField.TYPE_STORED); ft.setIndexed(false); d.add(newField(FIELD_ID, Integer.toString(i), ft)); - d.add(newField(FIELD_BODY, docs[i], TextField.TYPE_STORED)); + d.add(newTextField(FIELD_BODY, docs[i], Field.Store.YES)); writer.addDocument(d); } @@ -181,12 +181,11 @@ public class TestSimilarityBase extends LuceneTestCase { } /** * The generic test core called by all unit test methods. It calls the - * {@link SimilarityBase#score(BasicStats, float, int)} method of all + * {@link SimilarityBase#score(BasicStats, float, float)} method of all * Similarities in {@link #sims} and checks if the score is valid; i.e. it * is a finite positive real number. */ - private void unitTestCore(BasicStats stats, float freq, int docLen) - throws IOException { + private void unitTestCore(BasicStats stats, float freq, int docLen) { for (SimilarityBase sim : sims) { BasicStats realStats = (BasicStats) sim.computeWeight(stats.getTotalBoost(), toCollectionStats(stats), @@ -514,12 +513,11 @@ public class TestSimilarityBase extends LuceneTestCase { /** * The generic test core called by all correctness test methods. It calls the - * {@link SimilarityBase#score(BasicStats, float, int)} method of all + * {@link SimilarityBase#score(BasicStats, float, float)} method of all * Similarities in {@link #sims} and compares the score against the manually * computed {@code gold}. */ - private void correctnessTestCore(SimilarityBase sim, float gold) - throws IOException { + private void correctnessTestCore(SimilarityBase sim, float gold) { BasicStats stats = createStats(); BasicStats realStats = (BasicStats) sim.computeWeight(stats.getTotalBoost(), toCollectionStats(stats), @@ -559,9 +557,6 @@ public class TestSimilarityBase extends LuceneTestCase { /** Test whether all similarities return document 3 before documents 7 and 8. */ public void testHeartRanking() throws IOException { - assumeFalse("PreFlex codec does not support the stats necessary for this test!", - "Lucene3x".equals(Codec.getDefault().getName())); - Query q = new TermQuery(new Term(FIELD_BODY, "heart")); for (SimilarityBase sim : sims) { diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java b/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java index d33ae3e..1442e26 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,10 +23,10 @@ import java.util.Map; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; import org.apache.lucene.search.Weight; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.TermContext; /** * Holds all implementations of classes in the o.a.l.s.spans package as a @@ -52,12 +52,12 @@ final class JustCompileSearchSpans { } @Override - public boolean next() throws IOException { + public boolean next() { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @Override - public boolean skipTo(int target) throws IOException { + public boolean skipTo(int target) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @@ -67,7 +67,7 @@ final class JustCompileSearchSpans { } @Override - public Collection getPayload() throws IOException { + public Collection getPayload() { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @@ -86,7 +86,7 @@ final class JustCompileSearchSpans { } @Override - public Spans getSpans(AtomicReaderContext context, Bits acceptDocs, Map termContexts) throws IOException { + public Spans getSpans(AtomicReaderContext context, Bits acceptDocs, Map termContexts) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @@ -100,7 +100,7 @@ final class JustCompileSearchSpans { static final class JustCompilePayloadSpans extends Spans { @Override - public Collection getPayload() throws IOException { + public Collection getPayload() { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @@ -120,12 +120,12 @@ final class JustCompileSearchSpans { } @Override - public boolean next() throws IOException { + public boolean next() { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @Override - public boolean skipTo(int target) throws IOException { + public boolean skipTo(int target) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @@ -144,7 +144,7 @@ final class JustCompileSearchSpans { } @Override - protected boolean setFreqCurrentDoc() throws IOException { + protected boolean setFreqCurrentDoc() { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } } diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java b/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java index eda8730..1db7b02 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,16 +21,16 @@ import java.io.IOException; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.TreeSet; import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReaderContext; +import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.util.ReaderUtil; -import org.apache.lucene.util.TermContext; /** * @@ -42,14 +42,16 @@ import org.apache.lucene.util.TermContext; public class MultiSpansWrapper extends Spans { // can't be package private due to payloads private SpanQuery query; - private AtomicReaderContext[] leaves; + private List leaves; private int leafOrd = 0; private Spans current; private Map termContexts; + private final int numLeaves; - private MultiSpansWrapper(AtomicReaderContext[] leaves, SpanQuery query, Map termContexts) { + private MultiSpansWrapper(List leaves, SpanQuery query, Map termContexts) { this.query = query; this.leaves = leaves; + this.numLeaves = leaves.size(); this.termContexts = termContexts; } @@ -61,27 +63,30 @@ public class MultiSpansWrapper extends Spans { // can't be package private due t for (Term term : terms) { termContexts.put(term, TermContext.build(topLevelReaderContext, term, true)); } - AtomicReaderContext[] leaves = topLevelReaderContext.leaves(); - if(leaves.length == 1) { - return query.getSpans(leaves[0], leaves[0].reader().getLiveDocs(), termContexts); + final List leaves = topLevelReaderContext.leaves(); + if(leaves.size() == 1) { + final AtomicReaderContext ctx = leaves.get(0); + return query.getSpans(ctx, ctx.reader().getLiveDocs(), termContexts); } return new MultiSpansWrapper(leaves, query, termContexts); } @Override public boolean next() throws IOException { - if (leafOrd >= leaves.length) { + if (leafOrd >= numLeaves) { return false; } if (current == null) { - current = query.getSpans(leaves[leafOrd], leaves[leafOrd].reader().getLiveDocs(), termContexts); + final AtomicReaderContext ctx = leaves.get(leafOrd); + current = query.getSpans(ctx, ctx.reader().getLiveDocs(), termContexts); } while(true) { if (current.next()) { return true; } - if (++leafOrd < leaves.length) { - current = query.getSpans(leaves[leafOrd], leaves[leafOrd].reader().getLiveDocs(), termContexts); + if (++leafOrd < numLeaves) { + final AtomicReaderContext ctx = leaves.get(leafOrd); + current = query.getSpans(ctx, ctx.reader().getLiveDocs(), termContexts); } else { current = null; break; @@ -92,27 +97,30 @@ public class MultiSpansWrapper extends Spans { // can't be package private due t @Override public boolean skipTo(int target) throws IOException { - if (leafOrd >= leaves.length) { + if (leafOrd >= numLeaves) { return false; } int subIndex = ReaderUtil.subIndex(target, leaves); assert subIndex >= leafOrd; if (subIndex != leafOrd) { - current = query.getSpans(leaves[subIndex], leaves[subIndex].reader().getLiveDocs(), termContexts); + final AtomicReaderContext ctx = leaves.get(subIndex); + current = query.getSpans(ctx, ctx.reader().getLiveDocs(), termContexts); leafOrd = subIndex; } else if (current == null) { - current = query.getSpans(leaves[leafOrd], leaves[leafOrd].reader().getLiveDocs(), termContexts); + final AtomicReaderContext ctx = leaves.get(leafOrd); + current = query.getSpans(ctx, ctx.reader().getLiveDocs(), termContexts); } while (true) { - if (current.skipTo(target - leaves[leafOrd].docBase)) { + if (current.skipTo(target - leaves.get(leafOrd).docBase)) { return true; } - if (++leafOrd < leaves.length) { - current = query.getSpans(leaves[leafOrd], leaves[leafOrd].reader().getLiveDocs(), termContexts); + if (++leafOrd < numLeaves) { + final AtomicReaderContext ctx = leaves.get(leafOrd); + current = query.getSpans(ctx, ctx.reader().getLiveDocs(), termContexts); } else { - current = null; - break; + current = null; + break; } } @@ -124,7 +132,7 @@ public class MultiSpansWrapper extends Spans { // can't be package private due t if (current == null) { return DocIdSetIterator.NO_MORE_DOCS; } - return current.doc() + leaves[leafOrd].docBase; + return current.doc() + leaves.get(leafOrd).docBase; } @Override diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestBasics.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestBasics.java index 155f93b..29771eb 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/TestBasics.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestBasics.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -28,9 +28,8 @@ import org.apache.lucene.analysis.*; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Payload; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; @@ -42,6 +41,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryUtils; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.English; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; @@ -81,7 +81,7 @@ public class TestBasics extends LuceneTestCase { @Override public boolean incrementToken() throws IOException { if (input.incrementToken()) { - payloadAttr.setPayload(new Payload(("pos: " + pos).getBytes())); + payloadAttr.setPayload(new BytesRef(("pos: " + pos).getBytes())); pos++; return true; } else { @@ -115,7 +115,7 @@ public class TestBasics extends LuceneTestCase { //writer.infoStream = System.out; for (int i = 0; i < 2000; i++) { Document doc = new Document(); - doc.add(newField("field", English.intToEnglish(i), TextField.TYPE_STORED)); + doc.add(newTextField("field", English.intToEnglish(i), Field.Store.YES)); writer.addDocument(doc); } reader = writer.getReader(); @@ -411,8 +411,8 @@ public class TestBasics extends LuceneTestCase { @Test public void testSpanPayloadCheck() throws Exception { SpanTermQuery term1 = new SpanTermQuery(new Term("field", "five")); - Payload pay = new Payload(("pos: " + 5).getBytes()); - SpanQuery query = new SpanPayloadCheckQuery(term1, Collections.singletonList(pay.getData())); + BytesRef pay = new BytesRef(("pos: " + 5).getBytes()); + SpanQuery query = new SpanPayloadCheckQuery(term1, Collections.singletonList(pay.bytes)); checkHits(query, new int[] {1125, 1135, 1145, 1155, 1165, 1175, 1185, 1195, 1225, 1235, 1245, 1255, 1265, 1275, 1285, 1295, 1325, 1335, 1345, 1355, 1365, 1375, 1385, 1395, 1425, 1435, 1445, 1455, 1465, 1475, 1485, 1495, 1525, 1535, 1545, 1555, 1565, 1575, 1585, 1595, 1625, 1635, 1645, 1655, 1665, 1675, 1685, 1695, 1725, 1735, 1745, 1755, 1765, 1775, 1785, 1795, 1825, 1835, 1845, 1855, 1865, 1875, 1885, 1895, 1925, 1935, 1945, 1955, 1965, 1975, 1985, 1995}); assertTrue(searcher.explain(query, 1125).getValue() > 0.0f); @@ -421,16 +421,16 @@ public class TestBasics extends LuceneTestCase { SpanNearQuery snq; SpanQuery[] clauses; List list; - Payload pay2; + BytesRef pay2; clauses = new SpanQuery[2]; clauses[0] = term1; clauses[1] = term2; snq = new SpanNearQuery(clauses, 0, true); - pay = new Payload(("pos: " + 0).getBytes()); - pay2 = new Payload(("pos: " + 1).getBytes()); + pay = new BytesRef(("pos: " + 0).getBytes()); + pay2 = new BytesRef(("pos: " + 1).getBytes()); list = new ArrayList(); - list.add(pay.getData()); - list.add(pay2.getData()); + list.add(pay.bytes); + list.add(pay2.bytes); query = new SpanNearPayloadCheckQuery(snq, list); checkHits(query, new int[] {500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599}); @@ -439,13 +439,13 @@ public class TestBasics extends LuceneTestCase { clauses[1] = term2; clauses[2] = new SpanTermQuery(new Term("field", "five")); snq = new SpanNearQuery(clauses, 0, true); - pay = new Payload(("pos: " + 0).getBytes()); - pay2 = new Payload(("pos: " + 1).getBytes()); - Payload pay3 = new Payload(("pos: " + 2).getBytes()); + pay = new BytesRef(("pos: " + 0).getBytes()); + pay2 = new BytesRef(("pos: " + 1).getBytes()); + BytesRef pay3 = new BytesRef(("pos: " + 2).getBytes()); list = new ArrayList(); - list.add(pay.getData()); - list.add(pay2.getData()); - list.add(pay3.getData()); + list.add(pay.bytes); + list.add(pay2.bytes); + list.add(pay3.bytes); query = new SpanNearPayloadCheckQuery(snq, list); checkHits(query, new int[] {505}); @@ -470,14 +470,14 @@ public class TestBasics extends LuceneTestCase { checkHits(query, new int[]{1103, 1203,1303,1403,1503,1603,1703,1803,1903}); Collection payloads = new ArrayList(); - Payload pay = new Payload(("pos: " + 0).getBytes()); - Payload pay2 = new Payload(("pos: " + 1).getBytes()); - Payload pay3 = new Payload(("pos: " + 3).getBytes()); - Payload pay4 = new Payload(("pos: " + 4).getBytes()); - payloads.add(pay.getData()); - payloads.add(pay2.getData()); - payloads.add(pay3.getData()); - payloads.add(pay4.getData()); + BytesRef pay = new BytesRef(("pos: " + 0).getBytes()); + BytesRef pay2 = new BytesRef(("pos: " + 1).getBytes()); + BytesRef pay3 = new BytesRef(("pos: " + 3).getBytes()); + BytesRef pay4 = new BytesRef(("pos: " + 4).getBytes()); + payloads.add(pay.bytes); + payloads.add(pay2.bytes); + payloads.add(pay3.bytes); + payloads.add(pay4.bytes); query = new SpanNearPayloadCheckQuery(oneThousHunThree, payloads); checkHits(query, new int[]{1103, 1203,1303,1403,1503,1603,1703,1803,1903}); diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java index 36e7822..0971b20 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,7 +23,6 @@ import java.util.Set; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; @@ -48,7 +47,7 @@ public class TestFieldMaskingSpanQuery extends LuceneTestCase { } protected static Field field(String name, String value) { - return newField(name, value, TextField.TYPE_UNSTORED); + return newTextField(name, value, Field.Store.NO); } protected static IndexSearcher searcher; diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java index 5b9fabb..8ebea6a 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,7 +19,7 @@ package org.apache.lucene.search.spans; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; @@ -62,7 +62,7 @@ public class TestNearSpansOrdered extends LuceneTestCase { RandomIndexWriter writer= new RandomIndexWriter(random(), directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); for (int i = 0; i < docFields.length; i++) { Document doc = new Document(); - doc.add(newField(FIELD, docFields[i], TextField.TYPE_UNSTORED)); + doc.add(newTextField(FIELD, docFields[i], Field.Store.NO)); writer.addDocument(doc); } reader = writer.getReader(); @@ -196,8 +196,8 @@ public class TestNearSpansOrdered extends LuceneTestCase { SpanQuery q = makeQuery(); Weight w = searcher.createNormalizedWeight(q); IndexReaderContext topReaderContext = searcher.getTopReaderContext(); - AtomicReaderContext[] leaves = topReaderContext.leaves(); - Scorer s = w.scorer(leaves[0], true, false, leaves[0].reader().getLiveDocs()); + AtomicReaderContext leave = topReaderContext.leaves().get(0); + Scorer s = w.scorer(leave, true, false, leave.reader().getLiveDocs()); assertEquals(1, s.advance(1)); } diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java index 534bb9d..e5c079f 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java @@ -23,16 +23,19 @@ import java.util.Collection; import java.util.HashSet; import java.util.Set; -import org.apache.lucene.analysis.*; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.MockTokenizer; +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Payload; import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermQuery; @@ -42,7 +45,7 @@ import org.apache.lucene.search.payloads.PayloadSpanUtil; import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; public class TestPayloadSpans extends LuceneTestCase { @@ -110,7 +113,7 @@ public class TestPayloadSpans extends LuceneTestCase { newIndexWriterConfig(TEST_VERSION_CURRENT, new PayloadAnalyzer()).setSimilarity(similarity)); Document doc = new Document(); - doc.add(newField(PayloadHelper.FIELD, "one two three one four three", TextField.TYPE_STORED)); + doc.add(newTextField(PayloadHelper.FIELD, "one two three one four three", Field.Store.YES)); writer.addDocument(doc); IndexReader reader = writer.getReader(); writer.close(); @@ -247,14 +250,13 @@ public class TestPayloadSpans extends LuceneTestCase { directory.close(); } - public void testShrinkToAfterShortestMatch() throws CorruptIndexException, - LockObtainFailedException, IOException { + public void testShrinkToAfterShortestMatch() throws IOException { Directory directory = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new TestPayloadAnalyzer())); Document doc = new Document(); - doc.add(new TextField("content", new StringReader("a b c d e f g h i j a k"))); + doc.add(new TextField("content", new StringReader("a b c d e f g h i j a k"), Field.Store.NO)); writer.addDocument(doc); IndexReader reader = writer.getReader(); @@ -285,14 +287,13 @@ public class TestPayloadSpans extends LuceneTestCase { directory.close(); } - public void testShrinkToAfterShortestMatch2() throws CorruptIndexException, - LockObtainFailedException, IOException { + public void testShrinkToAfterShortestMatch2() throws IOException { Directory directory = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new TestPayloadAnalyzer())); Document doc = new Document(); - doc.add(new TextField("content", new StringReader("a b a d k f a h i k a k"))); + doc.add(new TextField("content", new StringReader("a b a d k f a h i k a k"), Field.Store.NO)); writer.addDocument(doc); IndexReader reader = writer.getReader(); IndexSearcher is = newSearcher(reader); @@ -321,14 +322,13 @@ public class TestPayloadSpans extends LuceneTestCase { directory.close(); } - public void testShrinkToAfterShortestMatch3() throws CorruptIndexException, - LockObtainFailedException, IOException { + public void testShrinkToAfterShortestMatch3() throws IOException { Directory directory = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new TestPayloadAnalyzer())); Document doc = new Document(); - doc.add(new TextField("content", new StringReader("j k a l f k k p a t a k l k t a"))); + doc.add(new TextField("content", new StringReader("j k a l f k k p a t a k l k t a"), Field.Store.NO)); writer.addDocument(doc); IndexReader reader = writer.getReader(); IndexSearcher is = newSearcher(reader); @@ -369,7 +369,7 @@ public class TestPayloadSpans extends LuceneTestCase { newIndexWriterConfig(TEST_VERSION_CURRENT, new PayloadAnalyzer()).setSimilarity(similarity)); Document doc = new Document(); - doc.add(newField(PayloadHelper.FIELD,"xx rr yy mm pp", TextField.TYPE_STORED)); + doc.add(newTextField(PayloadHelper.FIELD, "xx rr yy mm pp", Field.Store.YES)); writer.addDocument(doc); IndexReader reader = writer.getReader(); @@ -432,7 +432,7 @@ public class TestPayloadSpans extends LuceneTestCase { for(int i = 0; i < docs.length; i++) { doc = new Document(); String docText = docs[i]; - doc.add(newField(PayloadHelper.FIELD,docText, TextField.TYPE_STORED)); + doc.add(newTextField(PayloadHelper.FIELD, docText, Field.Store.YES)); writer.addDocument(doc); } @@ -505,9 +505,9 @@ public class TestPayloadSpans extends LuceneTestCase { if (!nopayload.contains(token)) { if (entities.contains(token)) { - payloadAtt.setPayload(new Payload((token + ":Entity:"+ pos ).getBytes())); + payloadAtt.setPayload(new BytesRef((token + ":Entity:"+ pos ).getBytes())); } else { - payloadAtt.setPayload(new Payload((token + ":Noise:" + pos ).getBytes())); + payloadAtt.setPayload(new BytesRef((token + ":Noise:" + pos ).getBytes())); } } pos += posIncrAtt.getPositionIncrement(); diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanExplanations.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanExplanations.java index c5da9d5..e752f1d 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanExplanations.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanExplanations.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanExplanationsOfNonMatches.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanExplanationsOfNonMatches.java index ed6406d..c4bbe41 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanExplanationsOfNonMatches.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanExplanationsOfNonMatches.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanFirstQuery.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanFirstQuery.java index f00b35b..b522bbf 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanFirstQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanFirstQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,7 +21,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; @@ -43,10 +43,10 @@ public class TestSpanFirstQuery extends LuceneTestCase { RandomIndexWriter writer = new RandomIndexWriter(random(), dir, analyzer); Document doc = new Document(); - doc.add(newField("field", "the quick brown fox", TextField.TYPE_UNSTORED)); + doc.add(newTextField("field", "the quick brown fox", Field.Store.NO)); writer.addDocument(doc); Document doc2 = new Document(); - doc2.add(newField("field", "quick brown fox", TextField.TYPE_UNSTORED)); + doc2.add(newTextField("field", "quick brown fox", Field.Store.NO)); writer.addDocument(doc2); IndexReader reader = writer.getReader(); @@ -75,10 +75,10 @@ public class TestSpanFirstQuery extends LuceneTestCase { RandomIndexWriter writer = new RandomIndexWriter(random(), dir, analyzer); Document doc = new Document(); - doc.add(newField("field", "the quick brown fox", TextField.TYPE_UNSTORED)); + doc.add(newTextField("field", "the quick brown fox", Field.Store.NO)); writer.addDocument(doc); Document doc2 = new Document(); - doc2.add(newField("field", "quick brown fox", TextField.TYPE_UNSTORED)); + doc2.add(newTextField("field", "quick brown fox", Field.Store.NO)); writer.addDocument(doc2); IndexReader reader = writer.getReader(); diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanMultiTermQueryWrapper.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanMultiTermQueryWrapper.java index 3ac1b01..9a310a9 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanMultiTermQueryWrapper.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanMultiTermQueryWrapper.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,7 +19,6 @@ package org.apache.lucene.search.spans; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; @@ -43,7 +42,7 @@ public class TestSpanMultiTermQueryWrapper extends LuceneTestCase { directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory); Document doc = new Document(); - Field field = newField("field", "", TextField.TYPE_UNSTORED); + Field field = newTextField("field", "", Field.Store.NO); doc.add(field); field.setStringValue("quick brown fox"); diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanSearchEquivalence.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanSearchEquivalence.java index ff9d06d..94c5db3 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanSearchEquivalence.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanSearchEquivalence.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java index 0ed3d7e..71e52fc 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,17 +18,19 @@ package org.apache.lucene.search.spans; */ import java.io.IOException; +import java.util.List; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanClause.Occur; @@ -47,7 +49,6 @@ import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.ReaderUtil; public class TestSpans extends LuceneTestCase { private IndexSearcher searcher; @@ -63,7 +64,7 @@ public class TestSpans extends LuceneTestCase { RandomIndexWriter writer= new RandomIndexWriter(random(), directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); for (int i = 0; i < docFields.length; i++) { Document doc = new Document(); - doc.add(newField(field, docFields[i], TextField.TYPE_STORED)); + doc.add(newTextField(field, docFields[i], Field.Store.YES)); writer.addDocument(doc); } reader = writer.getReader(); @@ -429,10 +430,10 @@ public class TestSpans extends LuceneTestCase { boolean ordered = true; int slop = 1; IndexReaderContext topReaderContext = searcher.getTopReaderContext(); - AtomicReaderContext[] leaves = topReaderContext.leaves(); + List leaves = topReaderContext.leaves(); int subIndex = ReaderUtil.subIndex(11, leaves); - for (int i = 0; i < leaves.length; i++) { - + for (int i = 0, c = leaves.size(); i < c; i++) { + final AtomicReaderContext ctx = leaves.get(i); final Similarity sim = new DefaultSimilarity() { @Override @@ -452,13 +453,13 @@ public class TestSpans extends LuceneTestCase { slop, ordered); - spanScorer = searcher.createNormalizedWeight(snq).scorer(leaves[i], true, false, leaves[i].reader().getLiveDocs()); + spanScorer = searcher.createNormalizedWeight(snq).scorer(ctx, true, false, ctx.reader().getLiveDocs()); } finally { searcher.setSimilarity(oldSim); } if (i == subIndex) { assertTrue("first doc", spanScorer.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); - assertEquals("first doc number", spanScorer.docID() + leaves[i].docBase, 11); + assertEquals("first doc number", spanScorer.docID() + ctx.docBase, 11); float score = spanScorer.score(); assertTrue("first doc score should be zero, " + score, score == 0.0f); } else { @@ -470,8 +471,8 @@ public class TestSpans extends LuceneTestCase { // LUCENE-1404 private void addDoc(IndexWriter writer, String id, String text) throws IOException { final Document doc = new Document(); - doc.add( newField("id", id, StringField.TYPE_STORED) ); - doc.add( newField("text", text, TextField.TYPE_STORED) ); + doc.add( newStringField("id", id, Field.Store.YES) ); + doc.add( newTextField("text", text, Field.Store.YES) ); writer.addDocument(doc); } @@ -509,7 +510,7 @@ public class TestSpans extends LuceneTestCase { writer.close(); // Get searcher - final IndexReader reader = IndexReader.open(dir); + final IndexReader reader = DirectoryReader.open(dir); final IndexSearcher searcher = newSearcher(reader); // Control (make sure docs indexed) diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java index cab704c..dc8a0f0 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,14 +19,13 @@ package org.apache.lucene.search.spans; import java.io.IOException; +import org.apache.lucene.document.Field; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenFilter; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; @@ -92,8 +91,8 @@ public class TestSpansAdvanced extends LuceneTestCase { final String text) throws IOException { final Document document = new Document(); - document.add(newField(FIELD_ID, id, StringField.TYPE_STORED)); - document.add(newField(FIELD_TEXT, text, TextField.TYPE_STORED)); + document.add(newStringField(FIELD_ID, id, Field.Store.YES)); + document.add(newTextField(FIELD_TEXT, text, Field.Store.YES)); writer.addDocument(document); } diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java index 9aaba8b..584be01 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,6 +22,7 @@ import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenFilter; import org.apache.lucene.analysis.MockTokenizer; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; @@ -75,7 +76,7 @@ public class TestSpansAdvanced2 extends TestSpansAdvanced { * @throws Exception */ public void testVerifyIndex() throws Exception { - final IndexReader reader = IndexReader.open(mDirectory); + final IndexReader reader = DirectoryReader.open(mDirectory); assertEquals(8, reader.numDocs()); reader.close(); } diff --git a/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java b/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java index eff89f0..581ebc8 100755 --- a/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java +++ b/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -28,7 +28,8 @@ import java.util.Random; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -252,14 +253,14 @@ public class TestBufferedIndexInput extends LuceneTestCase { ); for(int i=0;i<37;i++) { Document doc = new Document(); - doc.add(newField("content", "aaa bbb ccc ddd" + i, TextField.TYPE_STORED)); - doc.add(newField("id", "" + i, TextField.TYPE_STORED)); + doc.add(newTextField("content", "aaa bbb ccc ddd" + i, Field.Store.YES)); + doc.add(newTextField("id", "" + i, Field.Store.YES)); writer.addDocument(doc); } dir.allIndexInputs.clear(); - IndexReader reader = IndexReader.open(writer, true); + IndexReader reader = DirectoryReader.open(writer, true); Term aaa = new Term("content", "aaa"); Term bbb = new Term("content", "bbb"); @@ -267,7 +268,7 @@ public class TestBufferedIndexInput extends LuceneTestCase { dir.tweakBufferSizes(); writer.deleteDocuments(new Term("id", "0")); - reader = IndexReader.open(writer, true); + reader = DirectoryReader.open(writer, true); IndexSearcher searcher = newSearcher(reader); ScoreDoc[] hits = searcher.search(new TermQuery(bbb), null, 1000).scoreDocs; dir.tweakBufferSizes(); @@ -277,7 +278,7 @@ public class TestBufferedIndexInput extends LuceneTestCase { dir.tweakBufferSizes(); writer.deleteDocuments(new Term("id", "4")); - reader = IndexReader.open(writer, true); + reader = DirectoryReader.open(writer, true); searcher = newSearcher(reader); hits = searcher.search(new TermQuery(bbb), null, 1000).scoreDocs; diff --git a/lucene/core/src/test/org/apache/lucene/store/TestByteArrayDataInput.java b/lucene/core/src/test/org/apache/lucene/store/TestByteArrayDataInput.java index fb14c45..5d06cf5 100644 --- a/lucene/core/src/test/org/apache/lucene/store/TestByteArrayDataInput.java +++ b/lucene/core/src/test/org/apache/lucene/store/TestByteArrayDataInput.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/store/TestCopyBytes.java b/lucene/core/src/test/org/apache/lucene/store/TestCopyBytes.java index 62f1abb..f723f93 100644 --- a/lucene/core/src/test/org/apache/lucene/store/TestCopyBytes.java +++ b/lucene/core/src/test/org/apache/lucene/store/TestCopyBytes.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java index 12ab64a..c0ea75d 100644 --- a/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java +++ b/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java index d118648..32ecf12 100644 --- a/lucene/core/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java +++ b/lucene/core/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -26,6 +26,7 @@ import java.util.Set; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsWriter; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -56,7 +57,7 @@ public class TestFileSwitchDirectory extends LuceneTestCase { setMergePolicy(newLogMergePolicy(false)).setCodec(Codec.forName("Lucene40")) ); TestIndexWriterReader.createIndexNoClose(true, "ram", writer); - IndexReader reader = IndexReader.open(writer, true); + IndexReader reader = DirectoryReader.open(writer, true); assertEquals(100, reader.maxDoc()); writer.commit(); // we should see only fdx,fdt files here @@ -94,7 +95,7 @@ public class TestFileSwitchDirectory extends LuceneTestCase { public void testNoDir() throws Throwable { Directory dir = newFSSwitchDirectory(Collections.emptySet()); try { - IndexReader.open(dir); + DirectoryReader.open(dir); fail("did not hit expected exception"); } catch (NoSuchDirectoryException nsde) { // expected diff --git a/lucene/core/src/test/org/apache/lucene/store/TestHugeRamFile.java b/lucene/core/src/test/org/apache/lucene/store/TestHugeRamFile.java index 89790c3..6840f51 100755 --- a/lucene/core/src/test/org/apache/lucene/store/TestHugeRamFile.java +++ b/lucene/core/src/test/org/apache/lucene/store/TestHugeRamFile.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/store/TestLock.java b/lucene/core/src/test/org/apache/lucene/store/TestLock.java index 4c83ba5..42fac34 100644 --- a/lucene/core/src/test/org/apache/lucene/store/TestLock.java +++ b/lucene/core/src/test/org/apache/lucene/store/TestLock.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/store/TestLockFactory.java b/lucene/core/src/test/org/apache/lucene/store/TestLockFactory.java index 277dd21..7faa32d 100755 --- a/lucene/core/src/test/org/apache/lucene/store/TestLockFactory.java +++ b/lucene/core/src/test/org/apache/lucene/store/TestLockFactory.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -25,7 +25,8 @@ import java.util.Map; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -347,7 +348,7 @@ public class TestLockFactory extends LuceneTestCase { Query query = new TermQuery(new Term("content", "aaa")); for(int i=0;i 0 ? 1 : 0; - - int encodedLen1 = IndexableBinaryStringTools.getEncodedLength( - originalArray1, 0, numBytes1); - if (encodedLen1 > encoded1.length) - encoded1 = new char[ArrayUtil.oversize(encodedLen1, RamUsageEstimator.NUM_BYTES_CHAR)]; - IndexableBinaryStringTools.encode(originalArray1, 0, numBytes1, encoded1, - 0, encodedLen1); - - int encodedLen2 = IndexableBinaryStringTools.getEncodedLength(original2, - 0, numBytes2); - if (encodedLen2 > encoded2.length) - encoded2 = new char[ArrayUtil.oversize(encodedLen2, RamUsageEstimator.NUM_BYTES_CHAR)]; - IndexableBinaryStringTools.encode(original2, 0, numBytes2, encoded2, 0, - encodedLen2); - - int encodedComparison = new String(encoded1, 0, encodedLen1) - .compareTo(new String(encoded2, 0, encodedLen2)); - encodedComparison = encodedComparison < 0 ? -1 - : encodedComparison > 0 ? 1 : 0; - - assertEquals("Test #" + (testNum + 1) - + ": Original bytes and encoded chars compare differently:" - + System.getProperty("line.separator") + " binary 1: " - + binaryDump(originalArray1, numBytes1) - + System.getProperty("line.separator") + " binary 2: " - + binaryDump(original2, numBytes2) - + System.getProperty("line.separator") + "encoded 1: " - + charArrayDump(encoded1, encodedLen1) - + System.getProperty("line.separator") + "encoded 2: " - + charArrayDump(encoded2, encodedLen2) - + System.getProperty("line.separator"), originalComparison, - encodedComparison); - } - } - - public void testEmptyInput() { - byte[] binary = new byte[0]; - - int encodedLen = IndexableBinaryStringTools.getEncodedLength(binary, 0, - binary.length); - char[] encoded = new char[encodedLen]; - IndexableBinaryStringTools.encode(binary, 0, binary.length, encoded, 0, - encoded.length); - - int decodedLen = IndexableBinaryStringTools.getDecodedLength(encoded, 0, - encoded.length); - byte[] decoded = new byte[decodedLen]; - IndexableBinaryStringTools.decode(encoded, 0, encoded.length, decoded, 0, - decoded.length); - - assertEquals("decoded empty input was not empty", decoded.length, 0); - } - - public void testAllNullInput() { - byte[] binary = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - - int encodedLen = IndexableBinaryStringTools.getEncodedLength(binary, 0, - binary.length); - char encoded[] = new char[encodedLen]; - IndexableBinaryStringTools.encode(binary, 0, binary.length, encoded, 0, - encoded.length); - - int decodedLen = IndexableBinaryStringTools.getDecodedLength(encoded, 0, - encoded.length); - byte[] decoded = new byte[decodedLen]; - IndexableBinaryStringTools.decode(encoded, 0, encoded.length, decoded, 0, - decoded.length); - - assertEquals("Round trip decode/decode returned different results:" - + System.getProperty("line.separator") + " original: " - + binaryDump(binary, binary.length) - + System.getProperty("line.separator") + "decodedBuf: " - + binaryDump(decoded, decoded.length), - binaryDump(binary, binary.length), binaryDump(decoded, decoded.length)); - } - - public void testRandomBinaryRoundTrip() { - byte[] binary = new byte[MAX_RANDOM_BINARY_LENGTH]; - char[] encoded = new char[MAX_RANDOM_BINARY_LENGTH * 10]; - byte[] decoded = new byte[MAX_RANDOM_BINARY_LENGTH]; - for (int testNum = 0; testNum < NUM_RANDOM_TESTS; ++testNum) { - int numBytes = random().nextInt(MAX_RANDOM_BINARY_LENGTH - 1) + 1; // Min == 1 - - for (int byteNum = 0; byteNum < numBytes; ++byteNum) { - binary[byteNum] = (byte) random().nextInt(0x100); - } - - int encodedLen = IndexableBinaryStringTools.getEncodedLength(binary, 0, - numBytes); - if (encoded.length < encodedLen) - encoded = new char[ArrayUtil.oversize(encodedLen, RamUsageEstimator.NUM_BYTES_CHAR)]; - IndexableBinaryStringTools.encode(binary, 0, numBytes, encoded, 0, - encodedLen); - - int decodedLen = IndexableBinaryStringTools.getDecodedLength(encoded, 0, - encodedLen); - IndexableBinaryStringTools.decode(encoded, 0, encodedLen, decoded, 0, - decodedLen); - - assertEquals("Test #" + (testNum + 1) - + ": Round trip decode/decode returned different results:" - + System.getProperty("line.separator") + " original: " - + binaryDump(binary, numBytes) + System.getProperty("line.separator") - + "encodedBuf: " + charArrayDump(encoded, encodedLen) - + System.getProperty("line.separator") + "decodedBuf: " - + binaryDump(decoded, decodedLen), binaryDump(binary, numBytes), - binaryDump(decoded, decodedLen)); - } - } - - public String binaryDump(byte[] binary, int numBytes) { - StringBuilder buf = new StringBuilder(); - for (int byteNum = 0 ; byteNum < numBytes ; ++byteNum) { - String hex = Integer.toHexString(binary[byteNum] & 0xFF); - if (hex.length() == 1) { - buf.append('0'); - } - buf.append(hex.toUpperCase()); - if (byteNum < numBytes - 1) { - buf.append(' '); - } - } - return buf.toString(); - } - - public String charArrayDump(char[] charArray, int numBytes) { - StringBuilder buf = new StringBuilder(); - for (int charNum = 0 ; charNum < numBytes ; ++charNum) { - String hex = Integer.toHexString(charArray[charNum]); - for (int digit = 0 ; digit < 4 - hex.length() ; ++digit) { - buf.append('0'); - } - buf.append(hex.toUpperCase()); - if (charNum < numBytes - 1) { - buf.append(' '); - } - } - return buf.toString(); - } -} diff --git a/lucene/core/src/test/org/apache/lucene/util/TestIntsRef.java b/lucene/core/src/test/org/apache/lucene/util/TestIntsRef.java new file mode 100644 index 0000000..1b37873 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/util/TestIntsRef.java @@ -0,0 +1,40 @@ +package org.apache.lucene.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class TestIntsRef extends LuceneTestCase { + public void testEmpty() { + IntsRef i = new IntsRef(); + assertEquals(IntsRef.EMPTY_INTS, i.ints); + assertEquals(0, i.offset); + assertEquals(0, i.length); + } + + public void testFromInts() { + int ints[] = new int[] { 1, 2, 3, 4 }; + IntsRef i = new IntsRef(ints, 0, 4); + assertEquals(ints, i.ints); + assertEquals(0, i.offset); + assertEquals(4, i.length); + + IntsRef i2 = new IntsRef(ints, 1, 3); + assertEquals(new IntsRef(new int[] { 2, 3, 4 }, 0, 3), i2); + + assertFalse(i.equals(i2)); + } +} diff --git a/lucene/core/src/test/org/apache/lucene/util/TestMaxFailuresRule.java b/lucene/core/src/test/org/apache/lucene/util/TestMaxFailuresRule.java new file mode 100644 index 0000000..bd92eae --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/util/TestMaxFailuresRule.java @@ -0,0 +1,85 @@ +package org.apache.lucene.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.util.junitcompat.WithNestedTests; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.JUnitCore; +import org.junit.runner.Result; +import org.junit.runner.notification.Failure; +import org.junit.runner.notification.RunListener; + +import com.carrotsearch.randomizedtesting.SysGlobals; +import com.carrotsearch.randomizedtesting.annotations.Repeat; +import com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule; +import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule; + +/** + * @see TestRuleIgnoreAfterMaxFailures + * @see SystemPropertiesInvariantRule + */ +public class TestMaxFailuresRule extends WithNestedTests { + @Rule + public SystemPropertiesRestoreRule restoreSysProps = new SystemPropertiesRestoreRule(); + + public TestMaxFailuresRule() { + super(true); + } + + public static class Nested extends WithNestedTests.AbstractNestedTest { + @Repeat(iterations = 100) + public void testFailSometimes() { + assertFalse(random().nextInt(5) == 0); + } + } + + @Test + public void testMaxFailures() { + int maxFailures = LuceneTestCase.ignoreAfterMaxFailures.maxFailures; + int failuresSoFar = LuceneTestCase.ignoreAfterMaxFailures.failuresSoFar; + System.clearProperty(SysGlobals.SYSPROP_ITERATIONS()); + try { + LuceneTestCase.ignoreAfterMaxFailures.maxFailures = 2; + LuceneTestCase.ignoreAfterMaxFailures.failuresSoFar = 0; + + JUnitCore core = new JUnitCore(); + final int [] assumptions = new int [1]; + core.addListener(new RunListener() { + @Override + public void testAssumptionFailure(Failure failure) { + assumptions[0]++; + } + }); + + Result result = core.run(Nested.class); + Assert.assertEquals(100, result.getRunCount()); + Assert.assertEquals(0, result.getIgnoreCount()); + Assert.assertEquals(2, result.getFailureCount()); + + // JUnit doesn't pass back the number of successful tests, just make sure + // we did have enough assumption-failures. + Assert.assertTrue(assumptions[0] > 50); + } finally { + LuceneTestCase.ignoreAfterMaxFailures.maxFailures = maxFailures; + LuceneTestCase.ignoreAfterMaxFailures.failuresSoFar = failuresSoFar; + } + } +} diff --git a/lucene/core/src/test/org/apache/lucene/util/TestNamedSPILoader.java b/lucene/core/src/test/org/apache/lucene/util/TestNamedSPILoader.java index aea0e82..ef7d047 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestNamedSPILoader.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestNamedSPILoader.java @@ -4,7 +4,7 @@ import java.util.Set; import org.apache.lucene.codecs.Codec; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/TestNumericUtils.java b/lucene/core/src/test/org/apache/lucene/util/TestNumericUtils.java index a34855a..eda91b0 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestNumericUtils.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestNumericUtils.java @@ -219,7 +219,7 @@ public class TestNumericUtils extends LuceneTestCase { /** Note: The neededBounds Iterable must be unsigned (easier understanding what's happening) */ private void assertLongRangeSplit(final long lower, final long upper, int precisionStep, final boolean useBitSet, final Iterable expectedBounds, final Iterable expectedShifts - ) throws Exception { + ) { // Cannot use FixedBitSet since the range could be long: final OpenBitSet bits=useBitSet ? new OpenBitSet(upper-lower+1) : null; final Iterator neededBounds = (expectedBounds == null) ? null : expectedBounds.iterator(); @@ -460,7 +460,7 @@ public class TestNumericUtils extends LuceneTestCase { /** Note: The neededBounds Iterable must be unsigned (easier understanding what's happening) */ private void assertIntRangeSplit(final int lower, final int upper, int precisionStep, final boolean useBitSet, final Iterable expectedBounds, final Iterable expectedShifts - ) throws Exception { + ) { final FixedBitSet bits=useBitSet ? new FixedBitSet(upper-lower+1) : null; final Iterator neededBounds = (expectedBounds == null) ? null : expectedBounds.iterator(); final Iterator neededShifts = (expectedShifts == null) ? null : expectedShifts.iterator(); diff --git a/lucene/core/src/test/org/apache/lucene/util/TestOpenBitSet.java b/lucene/core/src/test/org/apache/lucene/util/TestOpenBitSet.java index c1a2024..010c857 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestOpenBitSet.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestOpenBitSet.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java b/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java index d308fac..4becc30 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/TestPriorityQueue.java b/lucene/core/src/test/org/apache/lucene/util/TestPriorityQueue.java index 7ff3da1..0b9a697 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestPriorityQueue.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestPriorityQueue.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java b/lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java index aff1ea0..caab70b 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimatorOnWildAnimals.java b/lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimatorOnWildAnimals.java index 63f53f5..d85303b 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimatorOnWildAnimals.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimatorOnWildAnimals.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/TestRecyclingByteBlockAllocator.java b/lucene/core/src/test/org/apache/lucene/util/TestRecyclingByteBlockAllocator.java index 21672c5..504cba6 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestRecyclingByteBlockAllocator.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestRecyclingByteBlockAllocator.java @@ -7,7 +7,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.junit.Before; import org.junit.Test; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/TestRollingBuffer.java b/lucene/core/src/test/org/apache/lucene/util/TestRollingBuffer.java index 9c4d1b9..146c490 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestRollingBuffer.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestRollingBuffer.java @@ -2,7 +2,7 @@ package org.apache.lucene.util; import java.util.Random; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/TestRollingCharBuffer.java b/lucene/core/src/test/org/apache/lucene/util/TestRollingCharBuffer.java index 5fb8756..abe57eb 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestRollingCharBuffer.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestRollingCharBuffer.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/TestSetOnce.java b/lucene/core/src/test/org/apache/lucene/util/TestSetOnce.java index bba6e81..bfaad09 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestSetOnce.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestSetOnce.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/TestSortedVIntList.java b/lucene/core/src/test/org/apache/lucene/util/TestSortedVIntList.java index 07ecb24..b07e973 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestSortedVIntList.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestSortedVIntList.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/TestTwoPhaseCommitTool.java b/lucene/core/src/test/org/apache/lucene/util/TestTwoPhaseCommitTool.java deleted file mode 100644 index 95aaeb5..0000000 --- a/lucene/core/src/test/org/apache/lucene/util/TestTwoPhaseCommitTool.java +++ /dev/null @@ -1,164 +0,0 @@ -package org.apache.lucene.util; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import org.apache.lucene.util.TwoPhaseCommitTool.TwoPhaseCommitWrapper; - -public class TestTwoPhaseCommitTool extends LuceneTestCase { - - private static class TwoPhaseCommitImpl implements TwoPhaseCommit { - static boolean commitCalled = false; - final boolean failOnPrepare; - final boolean failOnCommit; - final boolean failOnRollback; - boolean rollbackCalled = false; - Map prepareCommitData = null; - Map commitData = null; - - public TwoPhaseCommitImpl(boolean failOnPrepare, boolean failOnCommit, boolean failOnRollback) { - this.failOnPrepare = failOnPrepare; - this.failOnCommit = failOnCommit; - this.failOnRollback = failOnRollback; - } - - public void prepareCommit() throws IOException { - prepareCommit(null); - } - - public void prepareCommit(Map commitData) throws IOException { - this.prepareCommitData = commitData; - assertFalse("commit should not have been called before all prepareCommit were", commitCalled); - if (failOnPrepare) { - throw new IOException("failOnPrepare"); - } - } - - public void commit() throws IOException { - commit(null); - } - - public void commit(Map commitData) throws IOException { - this.commitData = commitData; - commitCalled = true; - if (failOnCommit) { - throw new RuntimeException("failOnCommit"); - } - } - - public void rollback() throws IOException { - rollbackCalled = true; - if (failOnRollback) { - throw new Error("failOnRollback"); - } - } - } - - @Override - public void setUp() throws Exception { - super.setUp(); - TwoPhaseCommitImpl.commitCalled = false; // reset count before every test - } - - public void testPrepareThenCommit() throws Exception { - // tests that prepareCommit() is called on all objects before commit() - TwoPhaseCommitImpl[] objects = new TwoPhaseCommitImpl[2]; - for (int i = 0; i < objects.length; i++) { - objects[i] = new TwoPhaseCommitImpl(false, false, false); - } - - // following call will fail if commit() is called before all prepare() were - TwoPhaseCommitTool.execute(objects); - } - - public void testRollback() throws Exception { - // tests that rollback is called if failure occurs at any stage - int numObjects = random().nextInt(8) + 3; // between [3, 10] - TwoPhaseCommitImpl[] objects = new TwoPhaseCommitImpl[numObjects]; - for (int i = 0; i < objects.length; i++) { - boolean failOnPrepare = random().nextBoolean(); - // we should not hit failures on commit usually - boolean failOnCommit = random().nextDouble() < 0.05; - boolean railOnRollback = random().nextBoolean(); - objects[i] = new TwoPhaseCommitImpl(failOnPrepare, failOnCommit, railOnRollback); - } - - boolean anyFailure = false; - try { - TwoPhaseCommitTool.execute(objects); - } catch (Throwable t) { - anyFailure = true; - } - - if (anyFailure) { - // if any failure happened, ensure that rollback was called on all. - for (TwoPhaseCommitImpl tpc : objects) { - assertTrue("rollback was not called while a failure occurred during the 2-phase commit", tpc.rollbackCalled); - } - } - } - - public void testWrapper() throws Exception { - // tests that TwoPhaseCommitWrapper delegates prepare/commit w/ commitData - TwoPhaseCommitImpl impl = new TwoPhaseCommitImpl(false, false, false); - HashMap commitData = new HashMap(); - TwoPhaseCommitWrapper wrapper = new TwoPhaseCommitWrapper(impl, commitData); - - wrapper.prepareCommit(); - assertSame(commitData, impl.prepareCommitData); - - // wrapper should ignore passed commitData - wrapper.prepareCommit(new HashMap()); - assertSame(commitData, impl.prepareCommitData); - - wrapper.commit(); - assertSame(commitData, impl.commitData); - - // wrapper should ignore passed commitData - wrapper.commit(new HashMap()); - assertSame(commitData, impl.commitData); - } - - public void testNullTPCs() throws Exception { - int numObjects = random().nextInt(4) + 3; // between [3, 6] - TwoPhaseCommit[] tpcs = new TwoPhaseCommit[numObjects]; - boolean setNull = false; - for (int i = 0; i < tpcs.length; i++) { - boolean isNull = random().nextDouble() < 0.3; - if (isNull) { - setNull = true; - tpcs[i] = null; - } else { - tpcs[i] = new TwoPhaseCommitImpl(false, false, false); - } - } - - if (!setNull) { - // none of the TPCs were picked to be null, pick one at random - int idx = random().nextInt(numObjects); - tpcs[idx] = null; - } - - // following call would fail if TPCTool won't handle null TPCs properly - TwoPhaseCommitTool.execute(tpcs); - } - -} diff --git a/lucene/core/src/test/org/apache/lucene/util/TestUnicodeUtil.java b/lucene/core/src/test/org/apache/lucene/util/TestUnicodeUtil.java index e438956..c1aaac2 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestUnicodeUtil.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestUnicodeUtil.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/TestVersion.java b/lucene/core/src/test/org/apache/lucene/util/TestVersion.java index 5497787..6afc52d 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestVersion.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestVersion.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,9 +23,8 @@ public class TestVersion extends LuceneTestCase { for (Version v : Version.values()) { assertTrue("LUCENE_CURRENT must be always onOrAfter("+v+")", Version.LUCENE_CURRENT.onOrAfter(v)); } - assertTrue(Version.LUCENE_40.onOrAfter(Version.LUCENE_31)); - assertTrue(Version.LUCENE_40.onOrAfter(Version.LUCENE_40)); - assertFalse(Version.LUCENE_30.onOrAfter(Version.LUCENE_31)); + assertTrue(Version.LUCENE_50.onOrAfter(Version.LUCENE_40)); + assertFalse(Version.LUCENE_40.onOrAfter(Version.LUCENE_50)); } public void testParseLeniently() { diff --git a/lucene/core/src/test/org/apache/lucene/util/TestVersionComparator.java b/lucene/core/src/test/org/apache/lucene/util/TestVersionComparator.java index d9646d2..8c649d7 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestVersionComparator.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestVersionComparator.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/TestVirtualMethod.java b/lucene/core/src/test/org/apache/lucene/util/TestVirtualMethod.java index b0a80fd..fffa0e8 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestVirtualMethod.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestVirtualMethod.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/TestWeakIdentityMap.java b/lucene/core/src/test/org/apache/lucene/util/TestWeakIdentityMap.java index 2faf1c7..37cc86c 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestWeakIdentityMap.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestWeakIdentityMap.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,7 +17,9 @@ package org.apache.lucene.util; +import java.util.Iterator; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Random; import java.util.concurrent.atomic.AtomicReferenceArray; import java.util.concurrent.Executors; @@ -42,9 +44,18 @@ public class TestWeakIdentityMap extends LuceneTestCase { assertNotSame(key2, key3); assertEquals(key2, key3); + // try null key & check its iterator also return null: + map.put(null, "null"); + { + Iterator it = map.keyIterator(); + assertTrue(it.hasNext()); + assertNull(it.next()); + assertFalse(it.hasNext()); + assertFalse(it.hasNext()); + } + // 2 more keys: map.put(key1, "bar1"); map.put(key2, "bar2"); - map.put(null, "null"); assertEquals(3, map.size()); @@ -84,6 +95,25 @@ public class TestWeakIdentityMap extends LuceneTestCase { map.put(key3, "bar3"); assertEquals(3, map.size()); + int c = 0, keysAssigned = 0; + for (Iterator it = map.keyIterator(); it.hasNext();) { + assertTrue(it.hasNext()); // try again, should return same result! + final String k = it.next(); + assertTrue(k == key1 || k == key2 | k == key3); + keysAssigned += (k == key1) ? 1 : ((k == key2) ? 2 : 4); + c++; + } + assertEquals(3, c); + assertEquals("all keys must have been seen", 1+2+4, keysAssigned); + + c = 0; + for (Iterator it = map.valueIterator(); it.hasNext();) { + final String v = it.next(); + assertTrue(v.startsWith("bar")); + c++; + } + assertEquals(3, c); + // clear strong refs key1 = key2 = key3 = null; @@ -93,7 +123,13 @@ public class TestWeakIdentityMap extends LuceneTestCase { System.runFinalization(); System.gc(); Thread.sleep(100L); - assertTrue(size >= map.size()); + c = 0; + for (Iterator it = map.keyIterator(); it.hasNext();) { + assertNotNull(it.next()); + c++; + } + assertTrue(size >= c); + assertTrue(c >= map.size()); size = map.size(); } catch (InterruptedException ie) {} @@ -101,6 +137,14 @@ public class TestWeakIdentityMap extends LuceneTestCase { assertEquals(0, map.size()); assertTrue(map.isEmpty()); + Iterator it = map.keyIterator(); + assertFalse(it.hasNext()); + try { + it.next(); + fail("Should throw NoSuchElementException"); + } catch (NoSuchElementException nse) { + } + key1 = new String("foo"); key2 = new String("foo"); map.put(key1, "bar1"); @@ -133,7 +177,7 @@ public class TestWeakIdentityMap extends LuceneTestCase { final int count = atLeast(rnd, 10000); for (int i = 0; i < count; i++) { final int j = rnd.nextInt(keyCount); - switch (rnd.nextInt(4)) { + switch (rnd.nextInt(5)) { case 0: map.put(keys.get(j), Integer.valueOf(j)); break; @@ -150,6 +194,12 @@ public class TestWeakIdentityMap extends LuceneTestCase { // renew key, the old one will be GCed at some time: keys.set(j, new Object()); break; + case 4: + // check iterator still working + for (Iterator it = map.keyIterator(); it.hasNext();) { + assertNotNull(it.next()); + } + break; default: fail("Should not get here."); } @@ -173,7 +223,13 @@ public class TestWeakIdentityMap extends LuceneTestCase { System.runFinalization(); System.gc(); Thread.sleep(100L); - assertTrue(size >= map.size()); + int c = 0; + for (Iterator it = map.keyIterator(); it.hasNext();) { + assertNotNull(it.next()); + c++; + } + assertTrue(size >= c); + assertTrue(c >= map.size()); size = map.size(); } catch (InterruptedException ie) {} } diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestBasicOperations.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestBasicOperations.java index 482f895..b209201 100644 --- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestBasicOperations.java +++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestBasicOperations.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.automaton; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,10 +17,35 @@ package org.apache.lucene.util.automaton; * limitations under the License. */ -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.UnicodeUtil; +import java.util.*; + +import org.apache.lucene.util.*; + +import com.carrotsearch.randomizedtesting.generators.RandomInts; + +public class TestBasicOperations extends LuceneTestCase { + /** Test string union. */ + public void testStringUnion() { + List strings = new ArrayList(); + for (int i = RandomInts.randomIntBetween(random(), 0, 1000); --i >= 0;) { + strings.add(new BytesRef(_TestUtil.randomUnicodeString(random()))); + } + + Collections.sort(strings); + Automaton union = BasicAutomata.makeStringUnion(strings); + assertTrue(union.isDeterministic()); + assertTrue(BasicOperations.sameLanguage(union, naiveUnion(strings))); + } + + private static Automaton naiveUnion(List strings) { + Automaton [] eachIndividual = new Automaton [strings.size()]; + int i = 0; + for (BytesRef bref : strings) { + eachIndividual[i++] = BasicAutomata.makeString(bref.utf8ToString()); + } + return BasicOperations.union(Arrays.asList(eachIndividual)); + } -public class TestBasicOperations extends LuceneTestCase { /** Test optimization to concatenate() */ public void testSingletonConcatenate() { Automaton singleton = BasicAutomata.makeString("prefix"); diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java index 89369ae..ae27033 100644 --- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java +++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.automaton; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestDeterminism.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestDeterminism.java index f0d3e2e..2191b37 100644 --- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestDeterminism.java +++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestDeterminism.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.automaton; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java index f4837d3..69e0946 100644 --- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java +++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.automaton; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestLevenshteinAutomata.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestLevenshteinAutomata.java index 20a5c87..d0c01b0 100644 --- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestLevenshteinAutomata.java +++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestLevenshteinAutomata.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.automaton; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestMinimize.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestMinimize.java index 8eb8c98..e306253 100644 --- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestMinimize.java +++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestMinimize.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.automaton; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestSpecialOperations.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestSpecialOperations.java index be48339..1071632 100644 --- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestSpecialOperations.java +++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestSpecialOperations.java @@ -2,7 +2,7 @@ package org.apache.lucene.util.automaton; import org.apache.lucene.util.LuceneTestCase; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestUTF32ToUTF8.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestUTF32ToUTF8.java index 9010cbc..eeff261 100644 --- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestUTF32ToUTF8.java +++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestUTF32ToUTF8.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.automaton; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java index 085a631..27ff14d 100644 --- a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java +++ b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -33,7 +33,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -53,16 +53,22 @@ import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IntsRef; import org.apache.lucene.util.LineFileDocs; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; +import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.UnicodeUtil; import org.apache.lucene.util._TestUtil; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.CompiledAutomaton; +import org.apache.lucene.util.automaton.RegExp; import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput; import org.apache.lucene.util.fst.FST.Arc; import org.apache.lucene.util.fst.FST.BytesReader; import org.apache.lucene.util.fst.PairOutputs.Pair; +import org.apache.lucene.util.packed.PackedInts; @SuppressCodecs({ "SimpleText", "Memory" }) +@Slow public class TestFSTs extends LuceneTestCase { private MockDirectoryWrapper dir; @@ -257,7 +263,7 @@ public class TestFSTs extends LuceneTestCase { final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(random().nextBoolean()); final List> pairs = new ArrayList>(terms.length); for(int idx=0;idx(terms[idx], random().nextLong() & Long.MAX_VALUE)); + pairs.add(new FSTTester.InputOutput(terms[idx], _TestUtil.nextLong(random(), 0, Long.MAX_VALUE))); } new FSTTester(random(), dir, inputMode, pairs, outputs, false).doTest(); } @@ -432,13 +438,14 @@ public class TestFSTs extends LuceneTestCase { in.offset = 0; final T NO_OUTPUT = fst.outputs.getNoOutput(); T output = NO_OUTPUT; + final FST.BytesReader fstReader = fst.getBytesReader(0); while(true) { // read all arcs: - fst.readFirstTargetArc(arc, arc); + fst.readFirstTargetArc(arc, arc, fstReader); arcs.add(new FST.Arc().copyFrom(arc)); while(!arc.isLast()) { - fst.readNextArc(arc); + fst.readNextArc(arc, fstReader); arcs.add(new FST.Arc().copyFrom(arc)); } @@ -494,7 +501,6 @@ public class TestFSTs extends LuceneTestCase { FST fst = builder.finish(); if (random.nextBoolean() && fst != null && !willRewrite) { - TestFSTs t = new TestFSTs(); IOContext context = LuceneTestCase.newIOContext(random); IndexOutput out = dir.createOutput("fst.bin", context); fst.save(out); @@ -533,7 +539,7 @@ public class TestFSTs extends LuceneTestCase { if (VERBOSE) { System.out.println("TEST: now rewrite"); } - final FST packed = fst.pack(_TestUtil.nextInt(random, 1, 10), _TestUtil.nextInt(random, 0, 10000000)); + final FST packed = fst.pack(_TestUtil.nextInt(random, 1, 10), _TestUtil.nextInt(random, 0, 10000000), random.nextFloat()); if (VERBOSE) { System.out.println("TEST: now verify packed FST"); } @@ -626,7 +632,7 @@ public class TestFSTs extends LuceneTestCase { final int num = atLeast(100); for(int iter=0;iter().copyFrom(arc), depth + 1); assertEquals( @@ -1913,7 +1930,7 @@ public class TestFSTs extends LuceneTestCase { final Long nothing = outputs.getNoOutput(); final Builder b = new Builder(FST.INPUT_TYPE.BYTE1, outputs); - final FST fst = new FST(FST.INPUT_TYPE.BYTE1, outputs, false); + final FST fst = new FST(FST.INPUT_TYPE.BYTE1, outputs, false, PackedInts.COMPACT); final Builder.UnCompiledNode rootNode = new Builder.UnCompiledNode(b, 0); @@ -1969,12 +1986,13 @@ public class TestFSTs extends LuceneTestCase { assertEquals(nothing, startArc.output); assertEquals(nothing, startArc.nextFinalOutput); - FST.Arc arc = fst.readFirstTargetArc(startArc, new FST.Arc()); + FST.Arc arc = fst.readFirstTargetArc(startArc, new FST.Arc(), + fst.getBytesReader(0)); assertEquals('a', arc.label); assertEquals(17, arc.nextFinalOutput.longValue()); assertTrue(arc.isFinal()); - arc = fst.readNextArc(arc); + arc = fst.readNextArc(arc, fst.getBytesReader(0)); assertEquals('b', arc.label); assertFalse(arc.isFinal()); assertEquals(42, arc.output.longValue()); diff --git a/lucene/core/src/test/org/apache/lucene/util/junitcompat/SorePoint.java b/lucene/core/src/test/org/apache/lucene/util/junitcompat/SorePoint.java index 48e8884..5b86632 100644 --- a/lucene/core/src/test/org/apache/lucene/util/junitcompat/SorePoint.java +++ b/lucene/core/src/test/org/apache/lucene/util/junitcompat/SorePoint.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.junitcompat; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/junitcompat/SoreType.java b/lucene/core/src/test/org/apache/lucene/util/junitcompat/SoreType.java index a246c99..27d9ca4 100644 --- a/lucene/core/src/test/org/apache/lucene/util/junitcompat/SoreType.java +++ b/lucene/core/src/test/org/apache/lucene/util/junitcompat/SoreType.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.junitcompat; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestBeforeAfterOverrides.java b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestBeforeAfterOverrides.java index 5b58dd7..17eea60 100644 --- a/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestBeforeAfterOverrides.java +++ b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestBeforeAfterOverrides.java @@ -7,7 +7,7 @@ import org.junit.Test; import org.junit.runner.JUnitCore; import org.junit.runner.Result; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestCodecReported.java b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestCodecReported.java new file mode 100644 index 0000000..4d2b904 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestCodecReported.java @@ -0,0 +1,47 @@ +package org.apache.lucene.util.junitcompat; + +import org.apache.lucene.codecs.Codec; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.JUnitCore; +import org.junit.runner.Result; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class TestCodecReported extends WithNestedTests { + public TestCodecReported() { + super(true); + } + + public static class Nested1 extends WithNestedTests.AbstractNestedTest { + public static String codecName; + + public void testDummy() { + codecName = Codec.getDefault().getName(); + fail(); + } + } + + @Test + public void testCorrectCodecReported() { + Result r = JUnitCore.runClasses(Nested1.class); + Assert.assertEquals(1, r.getFailureCount()); + Assert.assertTrue(super.getSysErr(), + super.getSysErr().contains("codec=" + Nested1.codecName)); + } +} diff --git a/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestExceptionInBeforeClassHooks.java b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestExceptionInBeforeClassHooks.java index 6a0b0d5..26719c3 100644 --- a/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestExceptionInBeforeClassHooks.java +++ b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestExceptionInBeforeClassHooks.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.junitcompat; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestJUnitRuleOrder.java b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestJUnitRuleOrder.java index a04fb0e..8c91a57 100644 --- a/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestJUnitRuleOrder.java +++ b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestJUnitRuleOrder.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.junitcompat; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestReproduceMessage.java b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestReproduceMessage.java index 7df711f..f635ce5 100644 --- a/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestReproduceMessage.java +++ b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestReproduceMessage.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.junitcompat; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -293,7 +293,7 @@ public class TestReproduceMessage extends WithNestedTests { Assert.assertTrue(runAndReturnSyserr().contains("NOTE: reproduce with:")); } - private String runAndReturnSyserr() throws Exception { + private String runAndReturnSyserr() { JUnitCore.runClasses(Nested.class); String err = getSysErr(); diff --git a/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSameRandomnessLocalePassedOrNot.java b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSameRandomnessLocalePassedOrNot.java index 4575b5a..191348d 100644 --- a/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSameRandomnessLocalePassedOrNot.java +++ b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSameRandomnessLocalePassedOrNot.java @@ -12,7 +12,7 @@ import org.junit.runner.Result; import com.carrotsearch.randomizedtesting.RandomizedContext; import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -39,7 +39,7 @@ public class TestSameRandomnessLocalePassedOrNot extends WithNestedTests { RuleChain.outerRule(new SystemPropertiesRestoreRule()); public TestSameRandomnessLocalePassedOrNot() { - super(false); + super(true); } public static class Nested extends WithNestedTests.AbstractNestedTest { diff --git a/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSeedFromUncaught.java b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSeedFromUncaught.java index a49cff7..9f4a08f 100644 --- a/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSeedFromUncaught.java +++ b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSeedFromUncaught.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.junitcompat; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSetupTeardownChaining.java b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSetupTeardownChaining.java index 6daa1c2..807ec33 100644 --- a/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSetupTeardownChaining.java +++ b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSetupTeardownChaining.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.junitcompat; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSystemPropertiesInvariantRule.java b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSystemPropertiesInvariantRule.java index 0f0a348..1bd971b 100644 --- a/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSystemPropertiesInvariantRule.java +++ b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSystemPropertiesInvariantRule.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.junitcompat; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/core/src/test/org/apache/lucene/util/junitcompat/WithNestedTests.java b/lucene/core/src/test/org/apache/lucene/util/junitcompat/WithNestedTests.java index adf0f94..ace8174 100644 --- a/lucene/core/src/test/org/apache/lucene/util/junitcompat/WithNestedTests.java +++ b/lucene/core/src/test/org/apache/lucene/util/junitcompat/WithNestedTests.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.junitcompat; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -26,13 +26,8 @@ import org.apache.lucene.util.TestRuleIgnoreTestSuites; import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.ClassRule; import org.junit.Rule; -import org.junit.rules.TestRule; -import org.junit.runner.Description; -import org.junit.runners.model.Statement; -import com.carrotsearch.randomizedtesting.RandomizedRunner; import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule; /** diff --git a/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java b/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java index 35fad05..8140d13 100644 --- a/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java +++ b/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.packed; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,15 +17,20 @@ package org.apache.lucene.util.packed; * limitations under the License. */ -import org.apache.lucene.store.*; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.packed.PackedInts.Reader; - +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Random; -import java.io.IOException; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.store.*; +import org.apache.lucene.util.LongsRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; +import org.apache.lucene.util.LuceneTestCase.Slow; +import org.apache.lucene.util.packed.PackedInts.Reader; + +@Slow public class TestPackedInts extends LuceneTestCase { public void testBitsRequired() throws Exception { assertEquals(61, PackedInts.bitsRequired((long)Math.pow(2, 61)-1)); @@ -50,69 +55,96 @@ public class TestPackedInts extends LuceneTestCase { public void testPackedInts() throws IOException { int num = atLeast(5); for (int iter = 0; iter < num; iter++) { - long ceil = 2; - for(int nbits=1;nbits<63;nbits++) { + for(int nbits=1;nbits<=64;nbits++) { + final long maxValue = PackedInts.maxValue(nbits); final int valueCount = 100+random().nextInt(500); + final int bufferSize = random().nextBoolean() + ? _TestUtil.nextInt(random(), 0, 48) + : _TestUtil.nextInt(random(), 0, 4096); final Directory d = newDirectory(); IndexOutput out = d.createOutput("out.bin", newIOContext(random())); PackedInts.Writer w = PackedInts.getWriter( - out, valueCount, nbits, random().nextFloat()*PackedInts.FASTEST); + out, valueCount, nbits, random().nextFloat()); + final long startFp = out.getFilePointer(); + final int actualValueCount = random().nextBoolean() ? valueCount : _TestUtil.nextInt(random(), 0, valueCount); final long[] values = new long[valueCount]; - for(int i=0;i packedInts = createPackedInts(valueCount, bitsPerValue); for (PackedInts.Mutable packedInt: packedInts) { try { - fill(packedInt, (long)(Math.pow(2, bitsPerValue)-1), randomSeed); + fill(packedInt, PackedInts.maxValue(bitsPerValue), randomSeed); } catch (Exception e) { e.printStackTrace(System.err); fail(String.format( @@ -202,7 +298,7 @@ public class TestPackedInts extends LuceneTestCase { packedInts.add(new Packed64(valueCount, bitsPerValue)); } packedInts.add(new Direct64(valueCount)); - for (int bpv = bitsPerValue; bpv <= 64; ++bpv) { + for (int bpv = bitsPerValue; bpv <= Packed64SingleBlock.MAX_SUPPORTED_BITS_PER_VALUE; ++bpv) { if (Packed64SingleBlock.isSupported(bpv)) { packedInts.add(Packed64SingleBlock.create(valueCount, bpv)); } @@ -212,9 +308,8 @@ public class TestPackedInts extends LuceneTestCase { private void fill(PackedInts.Mutable packedInt, long maxValue, long randomSeed) { Random rnd2 = new Random(randomSeed); - maxValue++; for (int i = 0 ; i < packedInt.size() ; i++) { - long value = Math.abs(rnd2.nextLong() % maxValue); + long value = _TestUtil.nextLong(rnd2, 0, maxValue); packedInt.set(i, value); assertEquals(String.format( "The set/get of the value at index %d should match for %s", @@ -291,16 +386,33 @@ public class TestPackedInts extends LuceneTestCase { int INDEX = (int)Math.pow(2, 30)+1; int BITS = 2; - Packed64 p64 = new Packed64(INDEX, BITS); - p64.set(INDEX-1, 1); - assertEquals("The value at position " + (INDEX-1) - + " should be correct for Packed64", 1, p64.get(INDEX-1)); - p64 = null; + Packed64 p64 = null; + try { + p64 = new Packed64(INDEX, BITS); + } catch (OutOfMemoryError oome) { + // This can easily happen: we're allocating a + // long[] that needs 256-273 MB. Heap is 512 MB, + // but not all of that is available for large + // objects ... empirical testing shows we only + // have ~ 67 MB free. + } + if (p64 != null) { + p64.set(INDEX-1, 1); + assertEquals("The value at position " + (INDEX-1) + + " should be correct for Packed64", 1, p64.get(INDEX-1)); + p64 = null; + } for (int bits = 1; bits <=64; ++bits) { if (Packed64SingleBlock.isSupported(bits)) { int index = Integer.MAX_VALUE / bits + (bits == 1 ? 0 : 1); - Packed64SingleBlock p64sb = Packed64SingleBlock.create(index, bits); + Packed64SingleBlock p64sb = null; + try { + p64sb = Packed64SingleBlock.create(index, bits); + } catch (OutOfMemoryError oome) { + // Ignore: see comment above + continue; + } p64sb.set(index - 1, 1); assertEquals("The value at position " + (index-1) + " should be correct for " + p64sb.getClass().getSimpleName(), @@ -309,18 +421,209 @@ public class TestPackedInts extends LuceneTestCase { } int index = Integer.MAX_VALUE / 24 + 1; - Packed8ThreeBlocks p8 = new Packed8ThreeBlocks(index); - p8.set(index - 1, 1); - assertEquals("The value at position " + (index-1) - + " should be correct for Packed8ThreeBlocks", 1, p8.get(index-1)); - p8 = null; + Packed8ThreeBlocks p8 = null; + try { + p8 = new Packed8ThreeBlocks(index); + } catch (OutOfMemoryError oome) { + // Ignore: see comment above + } + if (p8 != null) { + p8.set(index - 1, 1); + assertEquals("The value at position " + (index-1) + + " should be correct for Packed8ThreeBlocks", 1, p8.get(index-1)); + p8 = null; + } index = Integer.MAX_VALUE / 48 + 1; - Packed16ThreeBlocks p16 = new Packed16ThreeBlocks(index); - p16.set(index - 1, 1); - assertEquals("The value at position " + (index-1) - + " should be correct for Packed16ThreeBlocks", 1, p16.get(index-1)); - p16 = null; + Packed16ThreeBlocks p16 = null; + try { + p16 = new Packed16ThreeBlocks(index); + } catch (OutOfMemoryError oome) { + // Ignore: see comment above + } + if (p16 != null) { + p16.set(index - 1, 1); + assertEquals("The value at position " + (index-1) + + " should be correct for Packed16ThreeBlocks", 1, p16.get(index-1)); + p16 = null; + } + } + + public void testFill() { + final int valueCount = 1111; + final int from = random().nextInt(valueCount + 1); + final int to = from + random().nextInt(valueCount + 1 - from); + for (int bpv = 1; bpv <= 64; ++bpv) { + final long val = _TestUtil.nextLong(random(), 0, PackedInts.maxValue(bpv)); + List packedInts = createPackedInts(valueCount, bpv); + for (PackedInts.Mutable ints : packedInts) { + String msg = ints.getClass().getSimpleName() + " bpv=" + bpv + ", from=" + from + ", to=" + to + ", val=" + val; + ints.fill(0, ints.size(), 1); + ints.fill(from, to, val); + for (int i = 0; i < ints.size(); ++i) { + if (i >= from && i < to) { + assertEquals(msg + ", i=" + i, val, ints.get(i)); + } else { + assertEquals(msg + ", i=" + i, 1, ints.get(i)); + } + } + } + } + } + + public void testBulkGet() { + final int valueCount = 1111; + final int index = random().nextInt(valueCount); + final int len = _TestUtil.nextInt(random(), 1, valueCount * 2); + final int off = random().nextInt(77); + + for (int bpv = 1; bpv <= 64; ++bpv) { + long mask = PackedInts.maxValue(bpv); + List packedInts = createPackedInts(valueCount, bpv); + + for (PackedInts.Mutable ints : packedInts) { + for (int i = 0; i < ints.size(); ++i) { + ints.set(i, (31L * i - 1099) & mask); + } + long[] arr = new long[off+len]; + + String msg = ints.getClass().getSimpleName() + " valueCount=" + valueCount + + ", index=" + index + ", len=" + len + ", off=" + off; + final int gets = ints.get(index, arr, off, len); + assertTrue(msg, gets > 0); + assertTrue(msg, gets <= len); + assertTrue(msg, gets <= ints.size() - index); + + for (int i = 0; i < arr.length; ++i) { + String m = msg + ", i=" + i; + if (i >= off && i < off + gets) { + assertEquals(m, ints.get(i - off + index), arr[i]); + } else { + assertEquals(m, 0, arr[i]); + } + } + } + } + } + + public void testBulkSet() { + final int valueCount = 1111; + final int index = random().nextInt(valueCount); + final int len = _TestUtil.nextInt(random(), 1, valueCount * 2); + final int off = random().nextInt(77); + long[] arr = new long[off+len]; + + for (int bpv = 1; bpv <= 64; ++bpv) { + long mask = PackedInts.maxValue(bpv); + List packedInts = createPackedInts(valueCount, bpv); + for (int i = 0; i < arr.length; ++i) { + arr[i] = (31L * i + 19) & mask; + } + + for (PackedInts.Mutable ints : packedInts) { + String msg = ints.getClass().getSimpleName() + " valueCount=" + valueCount + + ", index=" + index + ", len=" + len + ", off=" + off; + final int sets = ints.set(index, arr, off, len); + assertTrue(msg, sets > 0); + assertTrue(msg, sets <= len); + + for (int i = 0; i < ints.size(); ++i) { + String m = msg + ", i=" + i; + if (i >= index && i < index + sets) { + assertEquals(m, arr[off - index + i], ints.get(i)); + } else { + assertEquals(m, 0, ints.get(i)); + } + } + } + } + } + + public void testCopy() { + final int valueCount = 689; + final int off1 = random().nextInt(valueCount); + final int off2 = random().nextInt(valueCount); + final int len = random().nextInt(Math.min(valueCount - off1, valueCount - off2)); + final int mem = random().nextInt(1024); + + for (int bpv = 1; bpv <= 64; ++bpv) { + long mask = PackedInts.maxValue(bpv); + for (PackedInts.Mutable r1 : createPackedInts(valueCount, bpv)) { + for (int i = 0; i < r1.size(); ++i) { + r1.set(i, (31L * i - 1023) & mask); + } + for (PackedInts.Mutable r2 : createPackedInts(valueCount, bpv)) { + String msg = "src=" + r1 + ", dest=" + r2 + ", srcPos=" + off1 + + ", destPos=" + off2 + ", len=" + len + ", mem=" + mem; + PackedInts.copy(r1, off1, r2, off2, len, mem); + for (int i = 0; i < r2.size(); ++i) { + String m = msg + ", i=" + i; + if (i >= off2 && i < off2 + len) { + assertEquals(m, r1.get(i - off2 + off1), r2.get(i)); + } else { + assertEquals(m, 0, r2.get(i)); + } + } + } + } + } + } + + public void testGrowableWriter() { + final int valueCount = 113 + random().nextInt(1111); + GrowableWriter wrt = new GrowableWriter(1, valueCount, PackedInts.DEFAULT); + wrt.set(4, 2); + wrt.set(7, 10); + wrt.set(valueCount - 10, 99); + wrt.set(99, 999); + wrt.set(valueCount - 1, 1 << 10); + assertEquals(1 << 10, wrt.get(valueCount - 1)); + wrt.set(99, (1 << 23) - 1); + assertEquals(1 << 10, wrt.get(valueCount - 1)); + wrt.set(1, Long.MAX_VALUE); + assertEquals(1 << 10, wrt.get(valueCount - 1)); + assertEquals(Long.MAX_VALUE, wrt.get(1)); + assertEquals(2, wrt.get(4)); + assertEquals((1 << 23) - 1, wrt.get(99)); + assertEquals(10, wrt.get(7)); + assertEquals(99, wrt.get(valueCount - 10)); + assertEquals(1 << 10, wrt.get(valueCount - 1)); + } + + public void testSave() throws IOException { + final int valueCount = _TestUtil.nextInt(random(), 1, 2048); + for (int bpv = 1; bpv <= 64; ++bpv) { + final int maxValue = (int) Math.min(PackedInts.maxValue(31), PackedInts.maxValue(bpv)); + final RAMDirectory directory = new RAMDirectory(); + List packedInts = createPackedInts(valueCount, bpv); + for (PackedInts.Mutable mutable : packedInts) { + for (int i = 0; i < mutable.size(); ++i) { + mutable.set(i, random().nextInt(maxValue)); + } + + IndexOutput out = directory.createOutput("packed-ints.bin", IOContext.DEFAULT); + mutable.save(out); + out.close(); + + IndexInput in = directory.openInput("packed-ints.bin", IOContext.DEFAULT); + PackedInts.Reader reader = PackedInts.getReader(in); + assertEquals(mutable.getBitsPerValue(), reader.getBitsPerValue()); + assertEquals(valueCount, reader.size()); + if (mutable instanceof Packed64SingleBlock) { + // make sure that we used the right format so that the reader has + // the same performance characteristics as the mutable that has been + // serialized + assertTrue(reader instanceof Packed64SingleBlock); + } else { + assertFalse(reader instanceof Packed64SingleBlock); + } + for (int i = 0; i < valueCount; ++i) { + assertEquals(mutable.get(i), reader.get(i)); + } + in.close(); + directory.deleteFile("packed-ints.bin"); + } + } } } diff --git a/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java b/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java index e0b7edb..a8e338e 100644 --- a/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java +++ b/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java @@ -1,6 +1,6 @@ package org.apache.lucene.demo; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -174,7 +174,7 @@ public class IndexFiles { // field that is indexed (i.e. searchable), but don't tokenize // the field into separate words and don't index term frequency // or positional information: - Field pathField = new Field("path", file.getPath(), StringField.TYPE_STORED); + Field pathField = new StringField("path", file.getPath(), Field.Store.YES); doc.add(pathField); // Add the last modified date of the file a field named "modified". @@ -184,13 +184,13 @@ public class IndexFiles { // year/month/day/hour/minutes/seconds, down the resolution you require. // For example the long value 2011021714 would mean // February 17, 2011, 2-3 PM. - doc.add(new LongField("modified", file.lastModified())); + doc.add(new LongField("modified", file.lastModified(), Field.Store.NO)); // Add the contents of the file to a field named "contents". Specify a Reader, // so that the text of the file is tokenized and indexed, but not stored. // Note that FileReader expects the file to be in UTF-8 encoding. // If that's not the case searching for special characters will fail. - doc.add(new TextField("contents", new BufferedReader(new InputStreamReader(fis, "UTF-8")))); + doc.add(new TextField("contents", new BufferedReader(new InputStreamReader(fis, "UTF-8")), Field.Store.NO)); if (writer.getConfig().getOpenMode() == OpenMode.CREATE) { // New index, so we just add the document (no old document can be there): diff --git a/lucene/demo/src/java/org/apache/lucene/demo/SearchFiles.java b/lucene/demo/src/java/org/apache/lucene/demo/SearchFiles.java index 7f5aa2f..b837b65 100644 --- a/lucene/demo/src/java/org/apache/lucene/demo/SearchFiles.java +++ b/lucene/demo/src/java/org/apache/lucene/demo/SearchFiles.java @@ -1,6 +1,6 @@ package org.apache.lucene.demo; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -27,6 +27,7 @@ import java.util.Date; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.IndexSearcher; @@ -86,7 +87,7 @@ public class SearchFiles { } } - IndexReader reader = IndexReader.open(FSDirectory.open(new File(index))); + IndexReader reader = DirectoryReader.open(FSDirectory.open(new File(index))); IndexSearcher searcher = new IndexSearcher(reader); Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_40); diff --git a/lucene/demo/src/java/org/apache/lucene/demo/xmlparser/FormBasedXmlQueryDemo.java b/lucene/demo/src/java/org/apache/lucene/demo/xmlparser/FormBasedXmlQueryDemo.java index 6e2564b..d92f772 100644 --- a/lucene/demo/src/java/org/apache/lucene/demo/xmlparser/FormBasedXmlQueryDemo.java +++ b/lucene/demo/src/java/org/apache/lucene/demo/xmlparser/FormBasedXmlQueryDemo.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -37,7 +37,6 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; @@ -126,7 +125,7 @@ public class FormBasedXmlQueryDemo extends HttpServlet { } } - private void openExampleIndex() throws CorruptIndexException, IOException { + private void openExampleIndex() throws IOException { //Create a RAM-based index from our test data file RAMDirectory rd = new RAMDirectory(); IndexWriterConfig iwConfig = new IndexWriterConfig(Version.LUCENE_40, analyzer); diff --git a/lucene/demo/src/test/org/apache/lucene/demo/TestDemo.java b/lucene/demo/src/test/org/apache/lucene/demo/TestDemo.java index d2bd59d..74ba354 100644 --- a/lucene/demo/src/test/org/apache/lucene/demo/TestDemo.java +++ b/lucene/demo/src/test/org/apache/lucene/demo/TestDemo.java @@ -1,6 +1,6 @@ package org.apache.lucene.demo; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/examples/org/apache/lucene/facet/example/ExampleResult.java b/lucene/facet/src/examples/org/apache/lucene/facet/example/ExampleResult.java index 25c8d6b..fe0f053 100644 --- a/lucene/facet/src/examples/org/apache/lucene/facet/example/ExampleResult.java +++ b/lucene/facet/src/examples/org/apache/lucene/facet/example/ExampleResult.java @@ -4,7 +4,7 @@ import java.util.List; import org.apache.lucene.facet.search.results.FacetResult; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/examples/org/apache/lucene/facet/example/ExampleUtils.java b/lucene/facet/src/examples/org/apache/lucene/facet/example/ExampleUtils.java index f5e211b..a58f80c 100644 --- a/lucene/facet/src/examples/org/apache/lucene/facet/example/ExampleUtils.java +++ b/lucene/facet/src/examples/org/apache/lucene/facet/example/ExampleUtils.java @@ -2,7 +2,7 @@ package org.apache.lucene.facet.example; import org.apache.lucene.util.Version; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/examples/org/apache/lucene/facet/example/adaptive/AdaptiveMain.java b/lucene/facet/src/examples/org/apache/lucene/facet/example/adaptive/AdaptiveMain.java index 260333a..c0abd6f 100644 --- a/lucene/facet/src/examples/org/apache/lucene/facet/example/adaptive/AdaptiveMain.java +++ b/lucene/facet/src/examples/org/apache/lucene/facet/example/adaptive/AdaptiveMain.java @@ -12,7 +12,7 @@ import org.apache.lucene.facet.example.simple.SimpleSearcher; import org.apache.lucene.facet.search.AdaptiveFacetsAccumulator; import org.apache.lucene.facet.search.results.FacetResult; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/examples/org/apache/lucene/facet/example/adaptive/AdaptiveSearcher.java b/lucene/facet/src/examples/org/apache/lucene/facet/example/adaptive/AdaptiveSearcher.java index 6cf8083..6497eb8 100644 --- a/lucene/facet/src/examples/org/apache/lucene/facet/example/adaptive/AdaptiveSearcher.java +++ b/lucene/facet/src/examples/org/apache/lucene/facet/example/adaptive/AdaptiveSearcher.java @@ -2,6 +2,7 @@ package org.apache.lucene.facet.example.adaptive; import java.util.List; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; @@ -22,7 +23,7 @@ import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -56,7 +57,7 @@ public class AdaptiveSearcher { public static List searchWithFacets (Directory indexDir, Directory taxoDir) throws Exception { // prepare index reader and taxonomy. TaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir); - IndexReader indexReader = IndexReader.open(indexDir); + IndexReader indexReader = DirectoryReader.open(indexDir); // prepare searcher to search against IndexSearcher searcher = new IndexSearcher(indexReader); diff --git a/lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationIndexer.java b/lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationIndexer.java index 6c91040..ecd9d14 100644 --- a/lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationIndexer.java +++ b/lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationIndexer.java @@ -18,7 +18,7 @@ import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -93,8 +93,8 @@ public class AssociationIndexer { // create a plain Lucene document and add some regular Lucene fields // to it Document doc = new Document(); - doc.add(new Field(SimpleUtils.TITLE, SimpleUtils.docTitles[docNum], TextField.TYPE_STORED)); - doc.add(new TextField(SimpleUtils.TEXT, SimpleUtils.docTexts[docNum])); + doc.add(new TextField(SimpleUtils.TITLE, SimpleUtils.docTitles[docNum], Field.Store.YES)); + doc.add(new TextField(SimpleUtils.TEXT, SimpleUtils.docTexts[docNum], Field.Store.NO)); // invoke the category document builder for adding categories to the // document and, diff --git a/lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationMain.java b/lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationMain.java index f49dc3d..619377b 100644 --- a/lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationMain.java +++ b/lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationMain.java @@ -9,7 +9,7 @@ import org.apache.lucene.facet.example.ExampleResult; import org.apache.lucene.facet.example.ExampleUtils; import org.apache.lucene.facet.search.results.FacetResult; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationSearcher.java b/lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationSearcher.java index f429c43..195c6aa 100644 --- a/lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationSearcher.java +++ b/lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationSearcher.java @@ -2,6 +2,7 @@ package org.apache.lucene.facet.example.association; import java.util.List; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.store.Directory; @@ -13,7 +14,7 @@ import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -42,7 +43,7 @@ public class AssociationSearcher { public static List searchSumIntAssociation(Directory indexDir, Directory taxoDir) throws Exception { // prepare index reader - IndexReader indexReader = IndexReader.open(indexDir); + IndexReader indexReader = DirectoryReader.open(indexDir); TaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir); AssociationIntSumFacetRequest facetRequest = new AssociationIntSumFacetRequest( @@ -62,7 +63,7 @@ public class AssociationSearcher { public static List searchSumFloatAssociation(Directory indexDir, Directory taxoDir) throws Exception { // prepare index reader - IndexReader indexReader = IndexReader.open(indexDir); + IndexReader indexReader = DirectoryReader.open(indexDir); TaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir); AssociationFloatSumFacetRequest facetRequest = new AssociationFloatSumFacetRequest( diff --git a/lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationUtils.java b/lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationUtils.java index 3bc749d..f13bf4d 100644 --- a/lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationUtils.java +++ b/lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationUtils.java @@ -7,7 +7,7 @@ import org.apache.lucene.facet.enhancements.association.AssociationProperty; import org.apache.lucene.facet.enhancements.params.DefaultEnhancementsIndexingParams; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java b/lucene/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java index 835c8e8..028c1a5 100644 --- a/lucene/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java +++ b/lucene/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java @@ -2,6 +2,7 @@ package org.apache.lucene.facet.example.merge; import java.io.IOException; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -16,7 +17,7 @@ import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter.DiskOr import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter.MemoryOrdinalMap; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter.OrdinalMap; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -87,7 +88,7 @@ public class TaxonomyMergeUtils { srcIndexDir, map.getMap(), new DefaultFacetIndexingParams()); destIndexWriter.setPayloadProcessorProvider(payloadProcessor); - IndexReader reader = IndexReader.open(srcIndexDir); + IndexReader reader = DirectoryReader.open(srcIndexDir); try { destIndexWriter.addIndexes(reader); diff --git a/lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLIndexer.java b/lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLIndexer.java index 9778386..b16c596 100644 --- a/lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLIndexer.java +++ b/lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLIndexer.java @@ -23,7 +23,7 @@ import org.apache.lucene.facet.index.params.PerDimensionIndexingParams; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -172,8 +172,8 @@ public class MultiCLIndexer { // create a plain Lucene document and add some regular Lucene fields // to it Document doc = new Document(); - doc.add(new Field(SimpleUtils.TITLE, docTitles[docNum], TextField.TYPE_STORED)); - doc.add(new TextField(SimpleUtils.TEXT, docTexts[docNum])); + doc.add(new TextField(SimpleUtils.TITLE, docTitles[docNum], Field.Store.YES)); + doc.add(new TextField(SimpleUtils.TEXT, docTexts[docNum], Field.Store.NO)); // finally add the document to the index categoryDocBuilder.build(doc); diff --git a/lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLMain.java b/lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLMain.java index 6562c3a..2be2113 100644 --- a/lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLMain.java +++ b/lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLMain.java @@ -9,7 +9,7 @@ import org.apache.lucene.facet.example.ExampleResult; import org.apache.lucene.facet.example.ExampleUtils; import org.apache.lucene.facet.search.results.FacetResult; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLSearcher.java b/lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLSearcher.java index 650b420..933b426 100644 --- a/lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLSearcher.java +++ b/lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLSearcher.java @@ -2,6 +2,7 @@ package org.apache.lucene.facet.example.multiCL; import java.util.List; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; @@ -22,7 +23,7 @@ import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -63,7 +64,7 @@ public class MultiCLSearcher { Directory taxoDir, FacetIndexingParams iParams) throws Exception { // prepare index reader and taxonomy. - IndexReader indexReader = IndexReader.open(indexDir); + IndexReader indexReader = DirectoryReader.open(indexDir); TaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir); // Get results diff --git a/lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleIndexer.java b/lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleIndexer.java index cef9ff7..6f02db4 100644 --- a/lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleIndexer.java +++ b/lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleIndexer.java @@ -17,7 +17,7 @@ import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -69,8 +69,8 @@ public class SimpleIndexer { // create a plain Lucene document and add some regular Lucene fields to it Document doc = new Document(); - doc.add(new Field(SimpleUtils.TITLE, SimpleUtils.docTitles[docNum], TextField.TYPE_STORED)); - doc.add(new TextField(SimpleUtils.TEXT, SimpleUtils.docTexts[docNum])); + doc.add(new TextField(SimpleUtils.TITLE, SimpleUtils.docTitles[docNum], Field.Store.YES)); + doc.add(new TextField(SimpleUtils.TEXT, SimpleUtils.docTexts[docNum], Field.Store.NO)); // invoke the category document builder for adding categories to the document and, // as required, to the taxonomy index diff --git a/lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleMain.java b/lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleMain.java index e81c23c..78c74d1 100644 --- a/lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleMain.java +++ b/lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleMain.java @@ -2,6 +2,7 @@ package org.apache.lucene.facet.example.simple; import java.util.List; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; @@ -12,7 +13,7 @@ import org.apache.lucene.facet.search.results.FacetResult; import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -57,7 +58,7 @@ public class SimpleMain { // open readers TaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir); - IndexReader indexReader = IndexReader.open(indexDir); + IndexReader indexReader = DirectoryReader.open(indexDir); ExampleUtils.log("search the sample documents..."); List facetRes = SimpleSearcher.searchWithFacets(indexReader, taxo); @@ -82,7 +83,7 @@ public class SimpleMain { // open readers TaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir); - IndexReader indexReader = IndexReader.open(indexDir); + IndexReader indexReader = DirectoryReader.open(indexDir); ExampleUtils.log("search the sample documents..."); List facetRes = SimpleSearcher.searchWithDrillDown(indexReader, taxo); diff --git a/lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleSearcher.java b/lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleSearcher.java index 2aa9daf..7295888 100644 --- a/lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleSearcher.java +++ b/lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleSearcher.java @@ -24,7 +24,7 @@ import org.apache.lucene.facet.search.results.FacetResultNode; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleUtils.java b/lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleUtils.java index fda6ab0..087f739 100644 --- a/lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleUtils.java +++ b/lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleUtils.java @@ -9,7 +9,7 @@ import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.facet.example.ExampleUtils; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/FacetException.java b/lucene/facet/src/java/org/apache/lucene/facet/FacetException.java index a03797c..d028192 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/FacetException.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/FacetException.java @@ -2,7 +2,7 @@ package org.apache.lucene.facet; import java.io.IOException; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/CategoryEnhancement.java b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/CategoryEnhancement.java index 90c4726..5c74869 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/CategoryEnhancement.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/CategoryEnhancement.java @@ -9,7 +9,7 @@ import org.apache.lucene.facet.index.streaming.CategoryListTokenizer; import org.apache.lucene.facet.index.streaming.CategoryParentsStream; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/EnhancementsCategoryTokenizer.java b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/EnhancementsCategoryTokenizer.java index 9d401bc..b5e0053 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/EnhancementsCategoryTokenizer.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/EnhancementsCategoryTokenizer.java @@ -1,6 +1,5 @@ package org.apache.lucene.facet.enhancements; -import java.io.IOException; import java.util.List; import org.apache.lucene.analysis.TokenStream; @@ -9,7 +8,7 @@ import org.apache.lucene.facet.enhancements.params.EnhancementsIndexingParams; import org.apache.lucene.facet.index.streaming.CategoryTokenizer; import org.apache.lucene.util.Vint8; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -59,10 +58,9 @@ public class EnhancementsCategoryTokenizer extends CategoryTokenizer { * The stream of category tokens. * @param indexingParams * The indexing params to use. - * @throws IOException */ public EnhancementsCategoryTokenizer(TokenStream input, - EnhancementsIndexingParams indexingParams) throws IOException { + EnhancementsIndexingParams indexingParams) { super(input, indexingParams); payloadBytes = new byte[Vint8.MAXIMUM_BYTES_NEEDED * (indexingParams.getCategoryEnhancements().size() + 1)]; @@ -114,7 +112,9 @@ public class EnhancementsCategoryTokenizer extends CategoryTokenizer { nBytes += enhancementBytes[i].length; } } - payload.setData(payloadBytes, 0, nBytes); + payload.bytes = payloadBytes; + payload.offset = 0; + payload.length = nBytes; payloadAttribute.setPayload(payload); } } diff --git a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/EnhancementsDocumentBuilder.java b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/EnhancementsDocumentBuilder.java index 0ca364a..c8ca028 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/EnhancementsDocumentBuilder.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/EnhancementsDocumentBuilder.java @@ -14,7 +14,7 @@ import org.apache.lucene.facet.index.streaming.CategoryParentsStream; import org.apache.lucene.facet.index.streaming.CategoryTokenizer; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -46,10 +46,9 @@ public class EnhancementsDocumentBuilder extends CategoryDocumentBuilder { * @param taxonomyWriter * @param params * Indexing params which include {@link CategoryEnhancement}s. - * @throws IOException */ public EnhancementsDocumentBuilder(TaxonomyWriter taxonomyWriter, - EnhancementsIndexingParams params) throws IOException { + EnhancementsIndexingParams params) { super(taxonomyWriter, params); } diff --git a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/EnhancementsPayloadIterator.java b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/EnhancementsPayloadIterator.java index 7eb69b9..cb4c129 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/EnhancementsPayloadIterator.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/EnhancementsPayloadIterator.java @@ -10,7 +10,7 @@ import org.apache.lucene.facet.search.PayloadIterator; import org.apache.lucene.util.Vint8; import org.apache.lucene.util.Vint8.Position; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationEnhancement.java b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationEnhancement.java index 8fc0d1b..81222c3 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationEnhancement.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationEnhancement.java @@ -14,7 +14,7 @@ import org.apache.lucene.facet.taxonomy.TaxonomyWriter; import org.apache.lucene.util.Vint8; import org.apache.lucene.util.Vint8.Position; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationFloatProperty.java b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationFloatProperty.java index 0af8fa1..ed58c45 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationFloatProperty.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationFloatProperty.java @@ -2,7 +2,7 @@ package org.apache.lucene.facet.enhancements.association; import org.apache.lucene.facet.index.attributes.CategoryProperty; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationIntProperty.java b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationIntProperty.java index 675cb34..c604870 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationIntProperty.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationIntProperty.java @@ -2,7 +2,7 @@ package org.apache.lucene.facet.enhancements.association; import org.apache.lucene.facet.index.attributes.CategoryProperty; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationListTokenizer.java b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationListTokenizer.java index c0a30a8..7376a28 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationListTokenizer.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationListTokenizer.java @@ -11,7 +11,7 @@ import org.apache.lucene.facet.index.attributes.OrdinalProperty; import org.apache.lucene.facet.index.streaming.CategoryListTokenizer; import org.apache.lucene.util.encoding.SimpleIntEncoder; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -79,7 +79,9 @@ public class AssociationListTokenizer extends CategoryListTokenizer { } if (payloadStream != null) { termAttribute.setEmpty().append(categoryListTermText); - payload.setData(payloadStream.convertStreamToByteArray()); + payload.bytes = payloadStream.convertStreamToByteArray(); + payload.offset = 0; + payload.length = payload.bytes.length; payloadAttribute.setPayload(payload); payloadStream = null; return true; diff --git a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationProperty.java b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationProperty.java index bdda839..6521816 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationProperty.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationProperty.java @@ -3,7 +3,7 @@ package org.apache.lucene.facet.enhancements.association; import org.apache.lucene.facet.index.attributes.CategoryAttribute; import org.apache.lucene.facet.index.attributes.CategoryProperty; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationsPayloadIterator.java b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationsPayloadIterator.java index bae9a41..c49807f 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationsPayloadIterator.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationsPayloadIterator.java @@ -11,7 +11,7 @@ import org.apache.lucene.util.collections.IntIterator; import org.apache.lucene.util.collections.IntToIntMap; import org.apache.lucene.util.encoding.SimpleIntDecoder; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/params/DefaultEnhancementsIndexingParams.java b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/params/DefaultEnhancementsIndexingParams.java index 57d4580..1c74a66 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/params/DefaultEnhancementsIndexingParams.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/params/DefaultEnhancementsIndexingParams.java @@ -8,7 +8,7 @@ import org.apache.lucene.facet.index.attributes.CategoryProperty; import org.apache.lucene.facet.index.params.CategoryListParams; import org.apache.lucene.facet.index.params.PerDimensionIndexingParams; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/params/EnhancementsIndexingParams.java b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/params/EnhancementsIndexingParams.java index 2862e5b..8e0fd93 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/enhancements/params/EnhancementsIndexingParams.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/enhancements/params/EnhancementsIndexingParams.java @@ -8,7 +8,7 @@ import org.apache.lucene.facet.index.attributes.CategoryProperty; import org.apache.lucene.facet.index.params.FacetIndexingParams; import org.apache.lucene.facet.index.streaming.CategoryParentsStream; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/CategoryContainer.java b/lucene/facet/src/java/org/apache/lucene/facet/index/CategoryContainer.java index 3199bea..803411b 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/CategoryContainer.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/CategoryContainer.java @@ -16,7 +16,7 @@ import org.apache.lucene.facet.index.attributes.CategoryAttributeImpl; import org.apache.lucene.facet.index.attributes.CategoryProperty; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/CategoryDocumentBuilder.java b/lucene/facet/src/java/org/apache/lucene/facet/index/CategoryDocumentBuilder.java index e6622ac..f67cd72 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/CategoryDocumentBuilder.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/CategoryDocumentBuilder.java @@ -27,7 +27,7 @@ import org.apache.lucene.facet.index.streaming.CountingListTokenizer; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -93,11 +93,9 @@ public class CategoryDocumentBuilder { * @param taxonomyWriter * to which new categories will be added, as well as translating * known categories to ordinals - * @throws IOException - * + * */ - public CategoryDocumentBuilder(TaxonomyWriter taxonomyWriter) - throws IOException { + public CategoryDocumentBuilder(TaxonomyWriter taxonomyWriter) { this(taxonomyWriter, new DefaultFacetIndexingParams()); } @@ -111,10 +109,9 @@ public class CategoryDocumentBuilder { * @param params * holds all parameters the indexing process should use such as * category-list parameters - * @throws IOException */ public CategoryDocumentBuilder(TaxonomyWriter taxonomyWriter, - FacetIndexingParams params) throws IOException { + FacetIndexingParams params) { this.taxonomyWriter = taxonomyWriter; this.indexingParams = params; this.categoriesMap = new HashMap>(); @@ -184,7 +181,7 @@ public class CategoryDocumentBuilder { // Finally creating a suitable field with stream and adding it to a // master field-list, used during the build process (see // super.build()) - FieldType ft = new FieldType(TextField.TYPE_UNSTORED); + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); ft.setOmitNorms(true); fieldList.add(new Field(e.getKey(), stream, ft)); } diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/CategoryListPayloadStream.java b/lucene/facet/src/java/org/apache/lucene/facet/index/CategoryListPayloadStream.java index eb0026b..261be9f 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/CategoryListPayloadStream.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/CategoryListPayloadStream.java @@ -5,7 +5,7 @@ import java.io.IOException; import org.apache.lucene.util.encoding.IntEncoder; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -24,7 +24,7 @@ import org.apache.lucene.util.encoding.IntEncoder; /** * Accumulates category IDs for a single document, for writing in byte array - * form, for example, to a Lucene Payload. + * form, for example, to a Lucene payload. * * @lucene.experimental */ @@ -33,7 +33,7 @@ public class CategoryListPayloadStream { private ByteArrayOutputStream baos = new ByteArrayOutputStream(50); private IntEncoder encoder; - /** Creates a Payload stream using the specified encoder. */ + /** Creates a payload stream using the specified encoder. */ public CategoryListPayloadStream(IntEncoder encoder) { this.encoder = encoder; this.encoder.reInit(baos); diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/FacetsPayloadProcessorProvider.java b/lucene/facet/src/java/org/apache/lucene/facet/index/FacetsPayloadProcessorProvider.java index 219160f..6160239 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/FacetsPayloadProcessorProvider.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/FacetsPayloadProcessorProvider.java @@ -21,7 +21,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.encoding.IntDecoder; import org.apache.lucene.util.encoding.IntEncoder; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryAttribute.java b/lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryAttribute.java index 55ceef7..0c8c4ee 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryAttribute.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryAttribute.java @@ -7,7 +7,7 @@ import org.apache.lucene.util.Attribute; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryAttributeImpl.java b/lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryAttributeImpl.java index 64566c2..b70907e 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryAttributeImpl.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryAttributeImpl.java @@ -8,7 +8,7 @@ import org.apache.lucene.util.AttributeImpl; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryAttributesIterable.java b/lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryAttributesIterable.java index 74760d7..1ce7208 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryAttributesIterable.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryAttributesIterable.java @@ -5,7 +5,7 @@ import java.util.Iterator; import org.apache.lucene.facet.index.streaming.CategoryAttributesStream; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryProperty.java b/lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryProperty.java index e9e55ba..3d390f5 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryProperty.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryProperty.java @@ -4,7 +4,7 @@ import java.io.Serializable; import org.apache.lucene.facet.index.CategoryContainer; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/attributes/OrdinalProperty.java b/lucene/facet/src/java/org/apache/lucene/facet/index/attributes/OrdinalProperty.java index 2511ce9..babce00 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/attributes/OrdinalProperty.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/attributes/OrdinalProperty.java @@ -1,6 +1,6 @@ package org.apache.lucene.facet.index.attributes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/DefaultOrdinalPolicy.java b/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/DefaultOrdinalPolicy.java index 95de238..8e67125 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/DefaultOrdinalPolicy.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/DefaultOrdinalPolicy.java @@ -3,7 +3,7 @@ package org.apache.lucene.facet.index.categorypolicy; import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/DefaultPathPolicy.java b/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/DefaultPathPolicy.java index 2fb172d..e2ed2c0 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/DefaultPathPolicy.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/DefaultPathPolicy.java @@ -3,7 +3,7 @@ package org.apache.lucene.facet.index.categorypolicy; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/NonTopLevelOrdinalPolicy.java b/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/NonTopLevelOrdinalPolicy.java index ee4c6fb..3548569 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/NonTopLevelOrdinalPolicy.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/NonTopLevelOrdinalPolicy.java @@ -3,7 +3,7 @@ package org.apache.lucene.facet.index.categorypolicy; import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/NonTopLevelPathPolicy.java b/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/NonTopLevelPathPolicy.java index 768c0b2..5b111bf 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/NonTopLevelPathPolicy.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/NonTopLevelPathPolicy.java @@ -3,7 +3,7 @@ package org.apache.lucene.facet.index.categorypolicy; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/OrdinalPolicy.java b/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/OrdinalPolicy.java index b300a28..b764d68 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/OrdinalPolicy.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/OrdinalPolicy.java @@ -5,7 +5,7 @@ import java.io.Serializable; import org.apache.lucene.facet.index.streaming.CategoryParentsStream; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/PathPolicy.java b/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/PathPolicy.java index 9f49f50..e0fb867 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/PathPolicy.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/PathPolicy.java @@ -5,7 +5,7 @@ import java.io.Serializable; import org.apache.lucene.facet.index.streaming.CategoryParentsStream; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java b/lucene/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java index 4ec1a14..349da82 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java @@ -17,7 +17,7 @@ import org.apache.lucene.util.encoding.SortingIntEncoder; import org.apache.lucene.util.encoding.UniqueValuesIntEncoder; import org.apache.lucene.util.encoding.VInt8IntEncoder; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/params/DefaultFacetIndexingParams.java b/lucene/facet/src/java/org/apache/lucene/facet/index/params/DefaultFacetIndexingParams.java index 557b9180..10b56a7 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/params/DefaultFacetIndexingParams.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/params/DefaultFacetIndexingParams.java @@ -9,7 +9,7 @@ import org.apache.lucene.facet.index.categorypolicy.OrdinalPolicy; import org.apache.lucene.facet.index.categorypolicy.PathPolicy; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/params/FacetIndexingParams.java b/lucene/facet/src/java/org/apache/lucene/facet/index/params/FacetIndexingParams.java index 8a91b14..a5d1d42 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/params/FacetIndexingParams.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/params/FacetIndexingParams.java @@ -6,7 +6,7 @@ import org.apache.lucene.facet.index.categorypolicy.OrdinalPolicy; import org.apache.lucene.facet.index.categorypolicy.PathPolicy; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/params/FacetParamsMissingPropertyException.java b/lucene/facet/src/java/org/apache/lucene/facet/index/params/FacetParamsMissingPropertyException.java index adb8181..574a9e2 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/params/FacetParamsMissingPropertyException.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/params/FacetParamsMissingPropertyException.java @@ -2,7 +2,7 @@ package org.apache.lucene.facet.index.params; import org.apache.lucene.facet.FacetException; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/params/PerDimensionIndexingParams.java b/lucene/facet/src/java/org/apache/lucene/facet/index/params/PerDimensionIndexingParams.java index 1df4c0e..433bc30 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/params/PerDimensionIndexingParams.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/params/PerDimensionIndexingParams.java @@ -6,7 +6,7 @@ import java.util.Map; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryAttributesStream.java b/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryAttributesStream.java index a869219..7d07655 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryAttributesStream.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryAttributesStream.java @@ -1,13 +1,12 @@ package org.apache.lucene.facet.index.streaming; -import java.io.IOException; import java.util.Iterator; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.facet.index.attributes.CategoryAttribute; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -59,7 +58,7 @@ public class CategoryAttributesStream extends TokenStream { } @Override - public final boolean incrementToken() throws IOException { + public final boolean incrementToken() { if (iterator == null) { if (iterable == null) { return false; diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryListTokenizer.java b/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryListTokenizer.java index 999f379..5d44771 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryListTokenizer.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryListTokenizer.java @@ -6,7 +6,7 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.facet.index.params.FacetIndexingParams; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryParentsStream.java b/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryParentsStream.java index b5a0cb3..2ef1651 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryParentsStream.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryParentsStream.java @@ -17,7 +17,7 @@ import org.apache.lucene.facet.index.params.FacetIndexingParams; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryTokenizer.java b/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryTokenizer.java index a2c4db2..27d97c9 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryTokenizer.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryTokenizer.java @@ -9,7 +9,7 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.facet.index.params.FacetIndexingParams; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryTokenizerBase.java b/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryTokenizerBase.java index 81a242e..2423ef4 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryTokenizerBase.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryTokenizerBase.java @@ -6,13 +6,13 @@ import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.index.Payload; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.facet.index.CategoryDocumentBuilder; import org.apache.lucene.facet.index.attributes.CategoryAttribute; import org.apache.lucene.facet.index.params.FacetIndexingParams; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -49,7 +49,7 @@ public abstract class CategoryTokenizerBase extends TokenFilter { protected CharTermAttribute termAttribute; /** The object used for constructing payloads. */ - protected Payload payload = new Payload(); + protected BytesRef payload = new BytesRef(); /** Indexing params for creating term text **/ protected FacetIndexingParams indexingParams; diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CountingListTokenizer.java b/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CountingListTokenizer.java index 84df4c2..19066a2 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CountingListTokenizer.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CountingListTokenizer.java @@ -15,7 +15,7 @@ import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.util.PartitionsUtils; import org.apache.lucene.util.encoding.IntEncoder; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -90,7 +90,9 @@ public class CountingListTokenizer extends CategoryListTokenizer { countingListName.getChars(0, length, termAttribute.buffer(), 0); this.termAttribute.setLength(length); CategoryListPayloadStream payloadStream = entry.getValue(); - payload.setData(payloadStream.convertStreamToByteArray()); + payload.bytes = payloadStream.convertStreamToByteArray(); + payload.offset = 0; + payload.length = payload.bytes.length; this.payloadAttribute.setPayload(payload); return true; } diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/AdaptiveFacetsAccumulator.java b/lucene/facet/src/java/org/apache/lucene/facet/search/AdaptiveFacetsAccumulator.java index 3c41787..faeae37 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/AdaptiveFacetsAccumulator.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/AdaptiveFacetsAccumulator.java @@ -15,7 +15,7 @@ import org.apache.lucene.facet.search.sampling.Sampler; import org.apache.lucene.facet.search.sampling.SamplingAccumulator; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/CategoryListIterator.java b/lucene/facet/src/java/org/apache/lucene/facet/search/CategoryListIterator.java index 5a134c4..79f8990 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/CategoryListIterator.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/CategoryListIterator.java @@ -2,7 +2,7 @@ package org.apache.lucene.facet.search; import java.io.IOException; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/DrillDown.java b/lucene/facet/src/java/org/apache/lucene/facet/search/DrillDown.java index 823a729..1c55248 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/DrillDown.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/DrillDown.java @@ -11,7 +11,7 @@ import org.apache.lucene.facet.index.params.FacetIndexingParams; import org.apache.lucene.facet.search.params.FacetSearchParams; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/FacetArrays.java b/lucene/facet/src/java/org/apache/lucene/facet/search/FacetArrays.java index cf954ad..9d666cb 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/FacetArrays.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/FacetArrays.java @@ -1,6 +1,6 @@ package org.apache.lucene.facet.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/FacetResultsHandler.java b/lucene/facet/src/java/org/apache/lucene/facet/search/FacetResultsHandler.java index 10ea484..314329c 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/FacetResultsHandler.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/FacetResultsHandler.java @@ -8,7 +8,7 @@ import org.apache.lucene.facet.search.results.FacetResultNode; import org.apache.lucene.facet.search.results.IntermediateFacetResult; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsAccumulator.java b/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsAccumulator.java index 779e34d..9172ae7 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsAccumulator.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsAccumulator.java @@ -10,7 +10,7 @@ import org.apache.lucene.facet.search.params.FacetRequest; import org.apache.lucene.facet.search.results.FacetResult; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java b/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java index 5bb25f7..c5e732f 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java @@ -13,7 +13,7 @@ import org.apache.lucene.facet.search.params.FacetSearchParams; import org.apache.lucene.facet.search.results.FacetResult; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/FloatArrayAllocator.java b/lucene/facet/src/java/org/apache/lucene/facet/search/FloatArrayAllocator.java index 78abe88..7c7e829 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/FloatArrayAllocator.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/FloatArrayAllocator.java @@ -2,7 +2,7 @@ package org.apache.lucene.facet.search; import java.util.Arrays; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/Heap.java b/lucene/facet/src/java/org/apache/lucene/facet/search/Heap.java index 8dc5cce..070bd84 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/Heap.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/Heap.java @@ -1,6 +1,6 @@ package org.apache.lucene.facet.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/IntArrayAllocator.java b/lucene/facet/src/java/org/apache/lucene/facet/search/IntArrayAllocator.java index 6b03d1c..555d644 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/IntArrayAllocator.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/IntArrayAllocator.java @@ -2,7 +2,7 @@ package org.apache.lucene.facet.search; import java.util.Arrays; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIntDecodingIterator.java b/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIntDecodingIterator.java index 87b6e1f..1dea804 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIntDecodingIterator.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIntDecodingIterator.java @@ -8,7 +8,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.util.UnsafeByteArrayInputStream; import org.apache.lucene.util.encoding.IntDecoder; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java b/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java index 090dcd6..6bcc63d 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java @@ -10,7 +10,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/SamplingWrapper.java b/lucene/facet/src/java/org/apache/lucene/facet/search/SamplingWrapper.java index 61a09b4..dc93f38 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/SamplingWrapper.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/SamplingWrapper.java @@ -10,7 +10,7 @@ import org.apache.lucene.facet.search.results.FacetResultNode; import org.apache.lucene.facet.search.sampling.Sampler; import org.apache.lucene.facet.search.sampling.Sampler.SampleResult; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/ScoredDocIDs.java b/lucene/facet/src/java/org/apache/lucene/facet/search/ScoredDocIDs.java index e183f9e..fa4264f 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/ScoredDocIDs.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/ScoredDocIDs.java @@ -4,7 +4,7 @@ import java.io.IOException; import org.apache.lucene.search.DocIdSet; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/ScoredDocIDsIterator.java b/lucene/facet/src/java/org/apache/lucene/facet/search/ScoredDocIDsIterator.java index 70f2398..82363d9 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/ScoredDocIDsIterator.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/ScoredDocIDsIterator.java @@ -1,6 +1,6 @@ package org.apache.lucene.facet.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/ScoredDocIdCollector.java b/lucene/facet/src/java/org/apache/lucene/facet/search/ScoredDocIdCollector.java index 628a897..85e1f2a 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/ScoredDocIdCollector.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/ScoredDocIdCollector.java @@ -11,7 +11,7 @@ import org.apache.lucene.search.Scorer; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.OpenBitSet; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -51,7 +51,7 @@ public abstract class ScoredDocIdCollector extends Collector { public boolean acceptsDocsOutOfOrder() { return true; } @Override - public void collect(int doc) throws IOException { + public void collect(int doc) { docIds.fastSet(docBase + doc); ++numDocIds; } @@ -62,7 +62,7 @@ public abstract class ScoredDocIdCollector extends Collector { } @Override - public ScoredDocIDsIterator scoredDocIdsIterator() throws IOException { + public ScoredDocIDsIterator scoredDocIdsIterator() { return new ScoredDocIDsIterator() { private DocIdSetIterator docIdsIter = docIds.iterator(); @@ -92,7 +92,7 @@ public abstract class ScoredDocIdCollector extends Collector { } @Override - public void setScorer(Scorer scorer) throws IOException {} + public void setScorer(Scorer scorer) {} } private static final class ScoringDocIdCollector extends ScoredDocIdCollector { @@ -124,7 +124,7 @@ public abstract class ScoredDocIdCollector extends Collector { } @Override - public ScoredDocIDsIterator scoredDocIdsIterator() throws IOException { + public ScoredDocIDsIterator scoredDocIdsIterator() { return new ScoredDocIDsIterator() { private DocIdSetIterator docIdsIter = docIds.iterator(); @@ -160,7 +160,7 @@ public abstract class ScoredDocIdCollector extends Collector { public void setDefaultScore(float defaultScore) {} @Override - public void setScorer(Scorer scorer) throws IOException { + public void setScorer(Scorer scorer) { this.scorer = scorer; } } diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsAccumulator.java b/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsAccumulator.java index 742bc8b..66d1200 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsAccumulator.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsAccumulator.java @@ -19,7 +19,7 @@ import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.facet.util.PartitionsUtils; import org.apache.lucene.facet.util.ScoredDocIdsUtils; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/TemporaryObjectAllocator.java b/lucene/facet/src/java/org/apache/lucene/facet/search/TemporaryObjectAllocator.java index 5b219e4..93782d7 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/TemporaryObjectAllocator.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/TemporaryObjectAllocator.java @@ -2,7 +2,7 @@ package org.apache.lucene.facet.search; import java.util.concurrent.ConcurrentLinkedQueue; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/TopKFacetResultsHandler.java b/lucene/facet/src/java/org/apache/lucene/facet/search/TopKFacetResultsHandler.java index 43df368..93914f1 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/TopKFacetResultsHandler.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/TopKFacetResultsHandler.java @@ -12,7 +12,7 @@ import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.facet.taxonomy.TaxonomyReader.ChildrenArrays; import org.apache.lucene.facet.util.ResultSortUtils; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/TopKInEachNodeHandler.java b/lucene/facet/src/java/org/apache/lucene/facet/search/TopKInEachNodeHandler.java index 8f773fa..6512a01 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/TopKInEachNodeHandler.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/TopKInEachNodeHandler.java @@ -17,7 +17,7 @@ import org.apache.lucene.facet.taxonomy.TaxonomyReader.ChildrenArrays; import org.apache.lucene.util.collections.IntIterator; import org.apache.lucene.util.collections.IntToObjectMap; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/TotalFacetCounts.java b/lucene/facet/src/java/org/apache/lucene/facet/search/TotalFacetCounts.java index 5aea862..3c29404 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/TotalFacetCounts.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/TotalFacetCounts.java @@ -12,7 +12,6 @@ import java.util.HashMap; import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.facet.index.params.CategoryListParams; import org.apache.lucene.facet.index.params.FacetIndexingParams; @@ -25,7 +24,7 @@ import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.facet.util.PartitionsUtils; import org.apache.lucene.facet.util.ScoredDocIdsUtils; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -73,7 +72,7 @@ public class TotalFacetCounts { * Construct by key - from index Directory or by recomputing. */ private TotalFacetCounts (TaxonomyReader taxonomy, FacetIndexingParams facetIndexingParams, - int[][] counts, CreationType createType4Test) throws IOException, LockObtainFailedException { + int[][] counts, CreationType createType4Test) { this.taxonomy = taxonomy; this.facetIndexingParams = facetIndexingParams; this.totalCounts = counts; diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/TotalFacetCountsCache.java b/lucene/facet/src/java/org/apache/lucene/facet/search/TotalFacetCountsCache.java index 23b9562..edb4b50 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/TotalFacetCountsCache.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/TotalFacetCountsCache.java @@ -14,7 +14,7 @@ import org.apache.lucene.facet.index.params.FacetIndexingParams; import org.apache.lucene.facet.search.cache.CategoryListCache; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/Aggregator.java b/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/Aggregator.java index 45f15ca..447ddd9 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/Aggregator.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/Aggregator.java @@ -2,7 +2,7 @@ package org.apache.lucene.facet.search.aggregator; import java.io.IOException; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/ComplementCountingAggregator.java b/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/ComplementCountingAggregator.java index eab1eb3..3bf62c4 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/ComplementCountingAggregator.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/ComplementCountingAggregator.java @@ -1,6 +1,6 @@ package org.apache.lucene.facet.search.aggregator; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/CountingAggregator.java b/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/CountingAggregator.java index d3569a4..b63b758 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/CountingAggregator.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/CountingAggregator.java @@ -1,6 +1,6 @@ package org.apache.lucene.facet.search.aggregator; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/ScoringAggregator.java b/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/ScoringAggregator.java index 6b1843c..e866d96 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/ScoringAggregator.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/ScoringAggregator.java @@ -1,6 +1,6 @@ package org.apache.lucene.facet.search.aggregator; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/association/AssociationFloatSumAggregator.java b/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/association/AssociationFloatSumAggregator.java index ab20ffb..5932226 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/association/AssociationFloatSumAggregator.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/association/AssociationFloatSumAggregator.java @@ -7,7 +7,7 @@ import org.apache.lucene.facet.index.params.CategoryListParams; import org.apache.lucene.facet.search.aggregator.Aggregator; import org.apache.lucene.index.IndexReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/association/AssociationIntSumAggregator.java b/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/association/AssociationIntSumAggregator.java index 7452aab..6c85cdd 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/association/AssociationIntSumAggregator.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/association/AssociationIntSumAggregator.java @@ -7,7 +7,7 @@ import org.apache.lucene.facet.index.params.CategoryListParams; import org.apache.lucene.facet.search.aggregator.Aggregator; import org.apache.lucene.index.IndexReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/cache/CategoryListCache.java b/lucene/facet/src/java/org/apache/lucene/facet/search/cache/CategoryListCache.java index 2acc218..d87356f 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/cache/CategoryListCache.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/cache/CategoryListCache.java @@ -9,7 +9,7 @@ import org.apache.lucene.facet.index.params.CategoryListParams; import org.apache.lucene.facet.index.params.FacetIndexingParams; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/cache/CategoryListData.java b/lucene/facet/src/java/org/apache/lucene/facet/search/cache/CategoryListData.java index a661077..ff4a25b 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/cache/CategoryListData.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/cache/CategoryListData.java @@ -10,7 +10,7 @@ import org.apache.lucene.facet.search.CategoryListIterator; import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.util.collections.IntArray; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/params/CountFacetRequest.java b/lucene/facet/src/java/org/apache/lucene/facet/search/params/CountFacetRequest.java index 099ed02..9a55244 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/params/CountFacetRequest.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/params/CountFacetRequest.java @@ -9,7 +9,7 @@ import org.apache.lucene.facet.search.aggregator.CountingAggregator; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/params/FacetRequest.java b/lucene/facet/src/java/org/apache/lucene/facet/search/params/FacetRequest.java index 6ef723a..2978650 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/params/FacetRequest.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/params/FacetRequest.java @@ -16,7 +16,7 @@ import org.apache.lucene.facet.search.cache.CategoryListCache; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/params/FacetSearchParams.java b/lucene/facet/src/java/org/apache/lucene/facet/search/params/FacetSearchParams.java index 99a6bd4..a7522a9 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/params/FacetSearchParams.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/params/FacetSearchParams.java @@ -8,7 +8,7 @@ import org.apache.lucene.facet.index.params.FacetIndexingParams; import org.apache.lucene.facet.search.cache.CategoryListCache; import org.apache.lucene.facet.search.results.FacetResult; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/params/ScoreFacetRequest.java b/lucene/facet/src/java/org/apache/lucene/facet/search/params/ScoreFacetRequest.java index dcb723a..b3ce102 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/params/ScoreFacetRequest.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/params/ScoreFacetRequest.java @@ -8,7 +8,7 @@ import org.apache.lucene.facet.search.aggregator.ScoringAggregator; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/params/association/AssociationFloatSumFacetRequest.java b/lucene/facet/src/java/org/apache/lucene/facet/search/params/association/AssociationFloatSumFacetRequest.java index ce7a1c9..3a842d8 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/params/association/AssociationFloatSumFacetRequest.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/params/association/AssociationFloatSumFacetRequest.java @@ -11,7 +11,7 @@ import org.apache.lucene.facet.search.params.FacetRequest; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/params/association/AssociationIntSumFacetRequest.java b/lucene/facet/src/java/org/apache/lucene/facet/search/params/association/AssociationIntSumFacetRequest.java index 32ee788..8c443c8 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/params/association/AssociationIntSumFacetRequest.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/params/association/AssociationIntSumFacetRequest.java @@ -11,7 +11,7 @@ import org.apache.lucene.facet.search.params.FacetRequest; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/results/FacetResult.java b/lucene/facet/src/java/org/apache/lucene/facet/search/results/FacetResult.java index af0d32c..14078f9 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/results/FacetResult.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/results/FacetResult.java @@ -2,7 +2,7 @@ package org.apache.lucene.facet.search.results; import org.apache.lucene.facet.search.params.FacetRequest; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/results/FacetResultNode.java b/lucene/facet/src/java/org/apache/lucene/facet/search/results/FacetResultNode.java index eff9f2b..c5713e5 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/results/FacetResultNode.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/results/FacetResultNode.java @@ -8,7 +8,7 @@ import org.apache.lucene.facet.search.sampling.SampleFixer; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/results/IntermediateFacetResult.java b/lucene/facet/src/java/org/apache/lucene/facet/search/results/IntermediateFacetResult.java index 100256b..5bc4f5b 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/results/IntermediateFacetResult.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/results/IntermediateFacetResult.java @@ -3,7 +3,7 @@ package org.apache.lucene.facet.search.results; import org.apache.lucene.facet.search.FacetResultsHandler; import org.apache.lucene.facet.search.params.FacetRequest; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/results/MutableFacetResultNode.java b/lucene/facet/src/java/org/apache/lucene/facet/search/results/MutableFacetResultNode.java index 92dcecb..3f508bb 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/results/MutableFacetResultNode.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/results/MutableFacetResultNode.java @@ -7,7 +7,7 @@ import java.util.List; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/RandomSampler.java b/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/RandomSampler.java index da9aa25..f90a8d9 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/RandomSampler.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/RandomSampler.java @@ -7,7 +7,7 @@ import org.apache.lucene.facet.search.ScoredDocIDs; import org.apache.lucene.facet.search.ScoredDocIDsIterator; import org.apache.lucene.facet.util.ScoredDocIdsUtils; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/RepeatableSampler.java b/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/RepeatableSampler.java index 0d04b62..30bfe26 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/RepeatableSampler.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/RepeatableSampler.java @@ -11,7 +11,7 @@ import org.apache.lucene.facet.search.ScoredDocIDs; import org.apache.lucene.facet.search.ScoredDocIDsIterator; import org.apache.lucene.facet.util.ScoredDocIdsUtils; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/SampleFixer.java b/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/SampleFixer.java index dc6a3a2..24735ad 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/SampleFixer.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/SampleFixer.java @@ -5,7 +5,7 @@ import java.io.IOException; import org.apache.lucene.facet.search.ScoredDocIDs; import org.apache.lucene.facet.search.results.FacetResult; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/Sampler.java b/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/Sampler.java index 0f660eb..2618e62 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/Sampler.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/Sampler.java @@ -14,7 +14,7 @@ import org.apache.lucene.facet.search.results.FacetResultNode; import org.apache.lucene.facet.search.results.MutableFacetResultNode; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/SamplingAccumulator.java b/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/SamplingAccumulator.java index fa48c68..e0fcefc 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/SamplingAccumulator.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/SamplingAccumulator.java @@ -19,7 +19,7 @@ import org.apache.lucene.facet.search.results.FacetResultNode; import org.apache.lucene.facet.search.sampling.Sampler.SampleResult; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/SamplingParams.java b/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/SamplingParams.java index 014fb7c..a50b7ac 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/SamplingParams.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/SamplingParams.java @@ -1,6 +1,6 @@ package org.apache.lucene.facet.search.sampling; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/TakmiSampleFixer.java b/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/TakmiSampleFixer.java index 1c7e099..fe34ba3 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/TakmiSampleFixer.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/TakmiSampleFixer.java @@ -18,7 +18,7 @@ import org.apache.lucene.facet.search.results.FacetResultNode; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/CategoryPath.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/CategoryPath.java index 8220666..30de4d9 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/CategoryPath.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/CategoryPath.java @@ -5,7 +5,7 @@ import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.io.Serializable; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -1013,7 +1013,7 @@ public class CategoryPath implements Serializable, Cloneable, Comparablenot complete, and @@ -111,13 +120,13 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { * categories, or if a put() to the cache ever returned true (meaning * that some of the cached data was cleared). */ - private boolean cacheIsComplete; - private DirectoryReader reader; - private int cacheMisses; + private volatile boolean cacheIsComplete; + private volatile ReaderManager readerManager; + private volatile boolean shouldRefreshReaderManager; + private volatile boolean isClosed = false; + private volatile ParentArray parentArray; + private volatile int nextID; - /** Records the taxonomy index creation time. */ - private final String createTime; - /** Reads the commit data from a Directory. */ private static Map readCommitData(Directory dir) throws IOException { SegmentInfos infos = new SegmentInfos(); @@ -126,16 +135,16 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { } /** - * setDelimiter changes the character that the taxonomy uses in its internal - * storage as a delimiter between category components. Do not use this - * method unless you really know what you are doing. It has nothing to do - * with whatever character the application may be using to represent - * categories for its own use. - *

      + * Changes the character that the taxonomy uses in its internal storage as a + * delimiter between category components. Do not use this method unless you + * really know what you are doing. It has nothing to do with whatever + * character the application may be using to represent categories for its own + * use. + *

      * If you do use this method, make sure you call it before any other methods - * that actually queries the taxonomy. Moreover, make sure you always pass - * the same delimiter for all LuceneTaxonomyWriter and LuceneTaxonomyReader - * objects you create for the same directory. + * that actually queries the taxonomy. Moreover, make sure you always pass the + * same delimiter for all taxonomy writer and reader instances you create for + * the same directory. */ public void setDelimiter(char delimiter) { ensureOpen(); @@ -204,21 +213,20 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { } } + dir = directory; IndexWriterConfig config = createIndexWriterConfig(openMode); - indexWriter = openIndexWriter(directory, config); + indexWriter = openIndexWriter(dir, config); // verify (to some extent) that merge policy in effect would preserve category docids assert !(indexWriter.getConfig().getMergePolicy() instanceof TieredMergePolicy) : "for preserving category docids, merging none-adjacent segments is not allowed"; - reader = null; - - FieldType ft = new FieldType(TextField.TYPE_UNSTORED); + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); ft.setOmitNorms(true); parentStreamField = new Field(Consts.FIELD_PAYLOADS, parentStream, ft); - fullPathField = new Field(Consts.FULL, "", StringField.TYPE_STORED); + fullPathField = new StringField(Consts.FULL, "", Field.Store.YES); - this.nextID = indexWriter.maxDoc(); + nextID = indexWriter.maxDoc(); if (cache == null) { cache = defaultTaxonomyWriterCache(); @@ -230,17 +238,15 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { // Make sure that the taxonomy always contain the root category // with category id 0. addCategory(new CategoryPath()); - refreshInternalReader(); } else { // There are some categories on the disk, which we have not yet // read into the cache, and therefore the cache is incomplete. - // We chose not to read all the categories into the cache now, + // We choose not to read all the categories into the cache now, // to avoid terrible performance when a taxonomy index is opened // to add just a single category. We will do it later, after we // notice a few cache misses. cacheIsComplete = false; } - cacheMisses = 0; } /** @@ -282,27 +288,31 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { protected IndexWriterConfig createIndexWriterConfig(OpenMode openMode) { // Make sure we use a MergePolicy which always merges adjacent segments and thus // keeps the doc IDs ordered as well (this is crucial for the taxonomy index). - return new IndexWriterConfig(Version.LUCENE_40, + return new IndexWriterConfig(Version.LUCENE_50, new KeywordAnalyzer()).setOpenMode(openMode).setMergePolicy( new LogByteSizeMergePolicy()); } - /** Opens a {@link DirectoryReader} from the internal {@link IndexWriter}. */ - private synchronized void openInternalReader() throws IOException { - // verify that the taxo-writer hasn't been closed on us. the method is - // synchronized since it may be called from a non sync'ed block, and it - // needs to protect against close() happening concurrently. - ensureOpen(); - assert reader == null : "a reader is already open !"; - reader = DirectoryReader.open(indexWriter, false); + /** Opens a {@link ReaderManager} from the internal {@link IndexWriter}. */ + private void initReaderManager() throws IOException { + if (readerManager == null) { + synchronized (this) { + // verify that the taxo-writer hasn't been closed on us. + ensureOpen(); + if (readerManager == null) { + readerManager = new ReaderManager(indexWriter, false); + shouldRefreshReaderManager = false; + } + } + } } /** - * Creates a new instance with a default cached as defined by + * Creates a new instance with a default cache as defined by * {@link #defaultTaxonomyWriterCache()}. */ public DirectoryTaxonomyWriter(Directory directory, OpenMode openMode) - throws CorruptIndexException, LockObtainFailedException, IOException { + throws IOException { this(directory, openMode, defaultTaxonomyWriterCache()); } @@ -320,9 +330,7 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { // convenience constructors: - public DirectoryTaxonomyWriter(Directory d) - throws CorruptIndexException, LockObtainFailedException, - IOException { + public DirectoryTaxonomyWriter(Directory d) throws IOException { this(d, OpenMode.CREATE_OR_APPEND); } @@ -332,16 +340,16 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { * {@link Directory}. */ @Override - public synchronized void close() throws CorruptIndexException, IOException { - if (indexWriter != null) { + public synchronized void close() throws IOException { + if (!isClosed) { indexWriter.commit(combinedCommitData(null)); doClose(); } } - private void doClose() throws CorruptIndexException, IOException { + private void doClose() throws IOException { indexWriter.close(); - indexWriter = null; + isClosed = true; closeResources(); } @@ -353,13 +361,12 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { * super.closeResources() call in your implementation. */ protected synchronized void closeResources() throws IOException { - if (reader != null) { - reader.close(); - reader = null; + if (readerManager != null) { + readerManager.close(); + readerManager = null; } if (cache != null) { cache.close(); - cache = null; } } @@ -369,52 +376,51 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { * category does not yet exist in the taxonomy. */ protected int findCategory(CategoryPath categoryPath) throws IOException { - // If we can find the category in our cache, we can return the - // response directly from it: + // If we can find the category in the cache, or we know the cache is + // complete, we can return the response directly from it int res = cache.get(categoryPath); - if (res >= 0) { + if (res >= 0 || cacheIsComplete) { return res; } - // If we know that the cache is complete, i.e., contains every category - // which exists, we can return -1 immediately. However, if the cache is - // not complete, we need to check the disk. - if (cacheIsComplete) { - return -1; - } - cacheMisses++; + + cacheMisses.incrementAndGet(); // After a few cache misses, it makes sense to read all the categories // from disk and into the cache. The reason not to do this on the first // cache miss (or even when opening the writer) is that it will // significantly slow down the case when a taxonomy is opened just to // add one category. The idea only spending a long time on reading - // after enough time was spent on cache misses is known as a "online + // after enough time was spent on cache misses is known as an "online // algorithm". - if (perhapsFillCache()) { - return cache.get(categoryPath); + perhapsFillCache(); + res = cache.get(categoryPath); + if (res >= 0 || cacheIsComplete) { + // if after filling the cache from the info on disk, the category is in it + // or the cache is complete, return whatever cache.get returned. + return res; } - // We need to get an answer from the on-disk index. If a reader - // is not yet open, do it now: - if (reader == null) { - openInternalReader(); - } + // if we get here, it means the category is not in the cache, and it is not + // complete, and therefore we must look for the category on disk. + + // We need to get an answer from the on-disk index. + initReaderManager(); - int base = 0; int doc = -1; - for (AtomicReader r : reader.getSequentialSubReaders()) { - DocsEnum docs = r.termDocsEnum(null, Consts.FULL, - new BytesRef(categoryPath.toString(delimiter)), false); - if (docs != null) { - doc = docs.nextDoc() + base; - break; + DirectoryReader reader = readerManager.acquire(); + try { + final BytesRef catTerm = new BytesRef(categoryPath.toString(delimiter)); + int base = 0; + for (AtomicReader r : reader.getSequentialSubReaders()) { + DocsEnum docs = r.termDocsEnum(null, Consts.FULL, catTerm, false); + if (docs != null) { + doc = docs.nextDoc() + base; + break; + } + base += r.maxDoc(); // we don't have deletions, so it's ok to call maxDoc } - base += r.maxDoc(); // we don't have deletions, so it's ok to call maxDoc + } finally { + readerManager.release(reader); } - // Note: we do NOT add to the cache the fact that the category - // does not exist. The reason is that our only use for this - // method is just before we actually add this category. If - // in the future this usage changes, we should consider caching - // the fact that the category is not in the taxonomy. if (doc > 0) { addToCache(categoryPath, doc); } @@ -429,30 +435,34 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { private int findCategory(CategoryPath categoryPath, int prefixLen) throws IOException { int res = cache.get(categoryPath, prefixLen); - if (res >= 0) { + if (res >= 0 || cacheIsComplete) { return res; } - if (cacheIsComplete) { - return -1; - } - cacheMisses++; - if (perhapsFillCache()) { - return cache.get(categoryPath, prefixLen); - } - if (reader == null) { - openInternalReader(); + + cacheMisses.incrementAndGet(); + perhapsFillCache(); + res = cache.get(categoryPath, prefixLen); + if (res >= 0 || cacheIsComplete) { + return res; } + + initReaderManager(); - int base = 0; int doc = -1; - for (AtomicReader r : reader.getSequentialSubReaders()) { - DocsEnum docs = r.termDocsEnum(null, Consts.FULL, - new BytesRef(categoryPath.toString(delimiter, prefixLen)), false); - if (docs != null) { - doc = docs.nextDoc() + base; - break; + DirectoryReader reader = readerManager.acquire(); + try { + final BytesRef catTerm = new BytesRef(categoryPath.toString(delimiter, prefixLen)); + int base = 0; + for (AtomicReader r : reader.getSequentialSubReaders()) { + DocsEnum docs = r.termDocsEnum(null, Consts.FULL, catTerm, false); + if (docs != null) { + doc = docs.nextDoc() + base; + break; + } + base += r.maxDoc(); // we don't have deletions, so it's ok to call maxDoc } - base += r.maxDoc(); // we don't have deletions, so it's ok to call maxDoc + } finally { + readerManager.release(reader); } if (doc > 0) { @@ -490,7 +500,7 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { /** * Add a new category into the index (and the cache), and return its new * ordinal. - *

      + *

      * Actually, we might also need to add some of the category's ancestors * before we can add the category itself (while keeping the invariant that a * parent is always added to the taxonomy before its child). We do this by @@ -524,7 +534,7 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { * {@link AlreadyClosedException} if it is. */ protected final void ensureOpen() { - if (indexWriter == null) { + if (isClosed) { throw new AlreadyClosedException("The taxonomy writer has already been closed"); } } @@ -544,7 +554,7 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { // we write here (e.g., to write parent+2), and need to do a workaround // in the reader (which knows that anyway only category 0 has a parent // -1). - parentStream.set(parent + 1); + parentStream.set(Math.max(parent+1, 1)); Document d = new Document(); d.add(parentStreamField); @@ -557,8 +567,11 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { indexWriter.addDocument(d); int id = nextID++; + // added a category document, mark that ReaderManager is not up-to-date + shouldRefreshReaderManager = true; + addToCache(categoryPath, length, id); - + // also add to the parent array getParentArray().add(id, parent); @@ -596,24 +609,18 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { if (returned) { return false; } - returned = true; - return true; + return returned = true; } } private void addToCache(CategoryPath categoryPath, int id) throws IOException { if (cache.put(categoryPath, id)) { // If cache.put() returned true, it means the cache was limited in - // size, became full, so parts of it had to be cleared. - // Unfortunately we don't know which part was cleared - it is - // possible that a relatively-new category that hasn't yet been - // committed to disk (and therefore isn't yet visible in our - // "reader") was deleted from the cache, and therefore we must - // now refresh the reader. - // Because this is a slow operation, cache implementations are - // expected not to delete entries one-by-one but rather in bulk - // (LruTaxonomyWriterCache removes the 2/3rd oldest entries). - refreshInternalReader(); + // size, became full, and parts of it had to be evicted. It is + // possible that a relatively-new category that isn't yet visible + // to our 'reader' was evicted, and therefore we must now refresh + // the reader. + refreshReaderManager(); cacheIsComplete = false; } } @@ -621,18 +628,22 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { private void addToCache(CategoryPath categoryPath, int prefixLen, int id) throws IOException { if (cache.put(categoryPath, prefixLen, id)) { - refreshInternalReader(); + refreshReaderManager(); cacheIsComplete = false; } } - private synchronized void refreshInternalReader() throws IOException { - if (reader != null) { - DirectoryReader r2 = DirectoryReader.openIfChanged(reader); - if (r2 != null) { - reader.close(); - reader = r2; - } + private synchronized void refreshReaderManager() throws IOException { + // this method is synchronized since it cannot happen concurrently with + // addCategoryDocument -- when this method returns, we must know that the + // reader manager's state is current. also, it sets shouldRefresh to false, + // and this cannot overlap with addCatDoc too. + // NOTE: since this method is sync'ed, it can call maybeRefresh, instead of + // maybeRefreshBlocking. If ever this is changed, make sure to change the + // call too. + if (shouldRefreshReaderManager && readerManager != null) { + readerManager.maybeRefresh(); + shouldRefreshReaderManager = false; } } @@ -643,10 +654,9 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { * See {@link TaxonomyWriter#commit()} */ @Override - public synchronized void commit() throws CorruptIndexException, IOException { + public synchronized void commit() throws IOException { ensureOpen(); indexWriter.commit(combinedCommitData(null)); - refreshInternalReader(); } /** @@ -669,10 +679,9 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { * See {@link TaxonomyWriter#commit(Map)}. */ @Override - public synchronized void commit(Map commitUserData) throws CorruptIndexException, IOException { + public synchronized void commit(Map commitUserData) throws IOException { ensureOpen(); indexWriter.commit(combinedCommitData(commitUserData)); - refreshInternalReader(); } /** @@ -680,7 +689,7 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { * See {@link IndexWriter#prepareCommit}. */ @Override - public synchronized void prepareCommit() throws CorruptIndexException, IOException { + public synchronized void prepareCommit() throws IOException { ensureOpen(); indexWriter.prepareCommit(combinedCommitData(null)); } @@ -690,144 +699,121 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { * See {@link IndexWriter#prepareCommit(Map)} */ @Override - public synchronized void prepareCommit(Map commitUserData) throws CorruptIndexException, IOException { + public synchronized void prepareCommit(Map commitUserData) throws IOException { ensureOpen(); indexWriter.prepareCommit(combinedCommitData(commitUserData)); } - /** - * getSize() returns the number of categories in the taxonomy. - *

      - * Because categories are numbered consecutively starting with 0, it means - * the taxonomy contains ordinals 0 through getSize()-1. - *

      - * Note that the number returned by getSize() is often slightly higher than - * the number of categories inserted into the taxonomy; This is because when - * a category is added to the taxonomy, its ancestors are also added - * automatically (including the root, which always get ordinal 0). - */ @Override - synchronized public int getSize() { + public int getSize() { ensureOpen(); - return indexWriter.maxDoc(); + return nextID; } - - private boolean alreadyCalledFillCache = false; - + /** - * Set the number of cache misses before an attempt is made to read the - * entire taxonomy into the in-memory cache. - *

      - * LuceneTaxonomyWriter holds an in-memory cache of recently seen - * categories to speed up operation. On each cache-miss, the on-disk index - * needs to be consulted. When an existing taxonomy is opened, a lot of - * slow disk reads like that are needed until the cache is filled, so it - * is more efficient to read the entire taxonomy into memory at once. - * We do this complete read after a certain number (defined by this method) - * of cache misses. - *

      - * If the number is set to 0, the entire taxonomy is read - * into the cache on first use, without fetching individual categories - * first. - *

      - * Note that if the memory cache of choice is limited in size, and cannot - * hold the entire content of the on-disk taxonomy, then it is never - * read in its entirety into the cache, regardless of the setting of this - * method. + * Set the number of cache misses before an attempt is made to read the entire + * taxonomy into the in-memory cache. + *

      + * This taxonomy writer holds an in-memory cache of recently seen categories + * to speed up operation. On each cache-miss, the on-disk index needs to be + * consulted. When an existing taxonomy is opened, a lot of slow disk reads + * like that are needed until the cache is filled, so it is more efficient to + * read the entire taxonomy into memory at once. We do this complete read + * after a certain number (defined by this method) of cache misses. + *

      + * If the number is set to {@code 0}, the entire taxonomy is read into the + * cache on first use, without fetching individual categories first. + *

      + * NOTE: it is assumed that this method is called immediately after the + * taxonomy writer has been created. */ public void setCacheMissesUntilFill(int i) { ensureOpen(); cacheMissesUntilFill = i; } - private int cacheMissesUntilFill = 11; - - private boolean perhapsFillCache() throws IOException { - // Note: we assume that we're only called when cacheIsComplete==false. - // TODO (Facet): parametrize this criterion: - if (cacheMisses < cacheMissesUntilFill) { - return false; - } - // If the cache was already filled (or we decided not to fill it because - // there was no room), there is no sense in trying it again. - if (alreadyCalledFillCache) { - return false; + // we need to guarantee that if several threads call this concurrently, only + // one executes it, and after it returns, the cache is updated and is either + // complete or not. + private synchronized void perhapsFillCache() throws IOException { + if (cacheMisses.get() < cacheMissesUntilFill) { + return; } - alreadyCalledFillCache = true; - // TODO (Facet): we should probably completely clear the cache before starting - // to read it? - if (reader == null) { - openInternalReader(); - } - - if (!cache.hasRoom(reader.numDocs())) { - return false; + + if (!shouldFillCache) { + // we already filled the cache once, there's no need to re-fill it + return; } + shouldFillCache = false; + + initReaderManager(); - CategoryPath cp = new CategoryPath(); - TermsEnum termsEnum = null; - DocsEnum docsEnum = null; - int base = 0; - for (AtomicReader r : reader.getSequentialSubReaders()) { - Terms terms = r.terms(Consts.FULL); - if (terms != null) { // cannot really happen, but be on the safe side - termsEnum = terms.iterator(termsEnum); - while (termsEnum.next() != null) { - BytesRef t = termsEnum.term(); - // Since we guarantee uniqueness of categories, each term has exactly - // one document. Also, since we do not allow removing categories (and - // hence documents), there are no deletions in the index. Therefore, it - // is sufficient to call next(), and then doc(), exactly once with no - // 'validation' checks. - cp.clear(); - cp.add(t.utf8ToString(), delimiter); - docsEnum = termsEnum.docs(null, docsEnum, false); - cache.put(cp, docsEnum.nextDoc() + base); + boolean aborted = false; + DirectoryReader reader = readerManager.acquire(); + try { + CategoryPath cp = new CategoryPath(); + TermsEnum termsEnum = null; + DocsEnum docsEnum = null; + int base = 0; + for (AtomicReader r : reader.getSequentialSubReaders()) { + Terms terms = r.terms(Consts.FULL); + if (terms != null) { // cannot really happen, but be on the safe side + termsEnum = terms.iterator(termsEnum); + while (termsEnum.next() != null) { + if (!cache.isFull()) { + BytesRef t = termsEnum.term(); + // Since we guarantee uniqueness of categories, each term has exactly + // one document. Also, since we do not allow removing categories (and + // hence documents), there are no deletions in the index. Therefore, it + // is sufficient to call next(), and then doc(), exactly once with no + // 'validation' checks. + cp.clear(); + cp.add(t.utf8ToString(), delimiter); + docsEnum = termsEnum.docs(null, docsEnum, false); + boolean res = cache.put(cp, docsEnum.nextDoc() + base); + assert !res : "entries should not have been evicted from the cache"; + } else { + // the cache is full and the next put() will evict entries from it, therefore abort the iteration. + aborted = true; + break; + } + } + } + if (aborted) { + break; } + base += r.maxDoc(); // we don't have any deletions, so we're ok } - base += r.maxDoc(); // we don't have any deletions, so we're ok + } finally { + readerManager.release(reader); } - /*Terms terms = MultiFields.getTerms(reader, Consts.FULL); - // The check is done here to avoid checking it on every iteration of the - // below loop. A null term wlil be returned if there are no terms in the - // lexicon, or after the Consts.FULL term. However while the loop is - // executed we're safe, because we only iterate as long as there are next() - // terms. - if (terms != null) { - TermsEnum termsEnum = terms.iterator(null); - Bits liveDocs = MultiFields.getLiveDocs(reader); - DocsEnum docsEnum = null; - while (termsEnum.next() != null) { - BytesRef t = termsEnum.term(); - // Since we guarantee uniqueness of categories, each term has exactly - // one document. Also, since we do not allow removing categories (and - // hence documents), there are no deletions in the index. Therefore, it - // is sufficient to call next(), and then doc(), exactly once with no - // 'validation' checks. - docsEnum = termsEnum.docs(liveDocs, docsEnum, false); - docsEnum.nextDoc(); - cp.clear(); - cp.add(t.utf8ToString(), delimiter); - cache.put(cp, docsEnum.docID()); - } - }*/ - cacheIsComplete = true; - // No sense to keep the reader open - we will not need to read from it - // if everything is in the cache. - reader.close(); - reader = null; - return true; + cacheIsComplete = !aborted; + if (cacheIsComplete) { + synchronized (this) { + // everything is in the cache, so no need to keep readerManager open. + // this block is executed in a sync block so that it works well with + // initReaderManager called in parallel. + readerManager.close(); + readerManager = null; + } + } } - private ParentArray parentArray; - private synchronized ParentArray getParentArray() throws IOException { - if (parentArray==null) { - if (reader == null) { - openInternalReader(); + private ParentArray getParentArray() throws IOException { + if (parentArray == null) { + synchronized (this) { + if (parentArray == null) { + initReaderManager(); + parentArray = new ParentArray(); + DirectoryReader reader = readerManager.acquire(); + try { + parentArray.refresh(reader); + } finally { + readerManager.release(reader); + } + } } - parentArray = new ParentArray(); - parentArray.refresh(reader); } return parentArray; } @@ -838,8 +824,8 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { // Note: the following if() just enforces that a user can never ask // for the parent of a nonexistant category - even if the parent array // was allocated bigger than it really needs to be. - if (ordinal >= getSize()) { - throw new ArrayIndexOutOfBoundsException(); + if (ordinal >= nextID) { + throw new ArrayIndexOutOfBoundsException("requested ordinal is bigger than the largest ordinal in the taxonomy"); } return getParentArray().getArray()[ordinal]; } @@ -1021,4 +1007,32 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { doClose(); } + /** + * Replaces the current taxonomy with the given one. This method should + * generally be called in conjunction with + * {@link IndexWriter#addIndexes(Directory...)} to replace both the taxonomy + * as well as the search index content. + */ + public synchronized void replaceTaxonomy(Directory taxoDir) throws IOException { + // replace the taxonomy by doing IW optimized operations + indexWriter.deleteAll(); + indexWriter.addIndexes(taxoDir); + shouldRefreshReaderManager = true; + nextID = indexWriter.maxDoc(); + + // need to clear the cache, so that addCategory won't accidentally return + // old categories that are in the cache. + cache.clear(); + cacheIsComplete = false; + shouldFillCache = true; + + // update createTime as a taxonomy replace is just like it has be recreated + createTime = Long.toString(System.nanoTime()); + } + + /** Returns the {@link Directory} of this taxonomy writer. */ + public Directory getDirectory() { + return dir; + } + } diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/ParentArray.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/ParentArray.java index 1e68db9..88897cc 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/ParentArray.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/ParentArray.java @@ -12,7 +12,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -147,7 +147,7 @@ class ParentArray { * NOTE: add() and refresh() CANNOT be used together. If you call add(), * this changes the arrays and refresh() can no longer be used. */ - void add(int ordinal, int parentOrdinal) throws IOException { + void add(int ordinal, int parentOrdinal) { if (ordinal >= prefetchParentOrdinal.length) { // grow the array, if necessary. // In Java 6, we could just do Arrays.copyOf()... diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/TaxonomyWriterCache.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/TaxonomyWriterCache.java index 646bdc0..ab2f36f 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/TaxonomyWriterCache.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/TaxonomyWriterCache.java @@ -3,7 +3,7 @@ package org.apache.lucene.facet.taxonomy.writercache; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -97,19 +97,18 @@ public interface TaxonomyWriterCache { * If the given length is negative or bigger than the path's actual * length, the full path is taken. */ - public boolean put(CategoryPath categoryPath, int prefixLen, int ordinal); - + public boolean put(CategoryPath categoryPath, int prefixLen, int ordinal); + /** - * Sometimes the cache is either unlimited in size, or limited by a very - * big size, and in that case when we add a lot of categories it might - * make sense to pre-load the cache with all the existing categories. - * However, this pre-load does not make sense when the allowed cache - * size is small. The hasRoom() method allows to differentiate between - * these cases. - *

      - * After hasRoom(n) returned true, the following n put() - * should return false (meaning that the cache was not cleared). + * Returns true if the cache is full, such that the next {@link #put} will + * evict entries from it, false otherwise. */ - public boolean hasRoom(int numberOfEntries); + public boolean isFull(); + /** + * Clears the content of the cache. Unlike {@link #close()}, the caller can + * assume that the cache is still operable after this method returns. + */ + public void clear(); + } diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/cl2o/CharBlockArray.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/cl2o/CharBlockArray.java index 13e0112..f482c79 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/cl2o/CharBlockArray.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/cl2o/CharBlockArray.java @@ -9,7 +9,7 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.List; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/cl2o/Cl2oTaxonomyWriterCache.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/cl2o/Cl2oTaxonomyWriterCache.java index f69a33a..34a0897 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/cl2o/Cl2oTaxonomyWriterCache.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/cl2o/Cl2oTaxonomyWriterCache.java @@ -6,7 +6,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.writercache.TaxonomyWriterCache; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -34,21 +34,37 @@ import org.apache.lucene.facet.taxonomy.writercache.TaxonomyWriterCache; public class Cl2oTaxonomyWriterCache implements TaxonomyWriterCache { private final ReadWriteLock lock = new ReentrantReadWriteLock(); - private CompactLabelToOrdinal cache; + private final int initialCapcity, numHashArrays; + private final float loadFactor; + + private volatile CompactLabelToOrdinal cache; public Cl2oTaxonomyWriterCache(int initialCapcity, float loadFactor, int numHashArrays) { this.cache = new CompactLabelToOrdinal(initialCapcity, loadFactor, numHashArrays); + this.initialCapcity = initialCapcity; + this.numHashArrays = numHashArrays; + this.loadFactor = loadFactor; } @Override + public void clear() { + lock.writeLock().lock(); + try { + cache = new CompactLabelToOrdinal(initialCapcity, loadFactor, numHashArrays); + } finally { + lock.writeLock().unlock(); + } + } + + @Override public synchronized void close() { cache = null; } @Override - public boolean hasRoom(int n) { - // This cache is unlimited, so we always have room for remembering more: - return true; + public boolean isFull() { + // This cache is never full + return false; } @Override diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/cl2o/CollisionMap.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/cl2o/CollisionMap.java index bd6f98d..7d40cf0 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/cl2o/CollisionMap.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/cl2o/CollisionMap.java @@ -6,7 +6,7 @@ import java.util.NoSuchElementException; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/cl2o/CompactLabelToOrdinal.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/cl2o/CompactLabelToOrdinal.java index f7a1ca6..c252a0d 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/cl2o/CompactLabelToOrdinal.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/cl2o/CompactLabelToOrdinal.java @@ -1,6 +1,6 @@ package org.apache.lucene.facet.taxonomy.writercache.cl2o; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/cl2o/LabelToOrdinal.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/cl2o/LabelToOrdinal.java index fc7fc79..797ec11 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/cl2o/LabelToOrdinal.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/cl2o/LabelToOrdinal.java @@ -2,7 +2,7 @@ package org.apache.lucene.facet.taxonomy.writercache.cl2o; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/lru/LruTaxonomyWriterCache.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/lru/LruTaxonomyWriterCache.java index aeb8507..9f50fa8 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/lru/LruTaxonomyWriterCache.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/lru/LruTaxonomyWriterCache.java @@ -3,7 +3,7 @@ package org.apache.lucene.facet.taxonomy.writercache.lru; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.writercache.TaxonomyWriterCache; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -61,11 +61,16 @@ public class LruTaxonomyWriterCache implements TaxonomyWriterCache { } @Override - public synchronized boolean hasRoom(int n) { - return n <= (cache.getMaxSize() - cache.getSize()); + public synchronized boolean isFull() { + return cache.getSize() == cache.getMaxSize(); } @Override + public synchronized void clear() { + cache.clear(); + } + + @Override public synchronized void close() { cache.clear(); cache = null; diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/lru/NameHashIntCacheLRU.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/lru/NameHashIntCacheLRU.java index f076435..33f4033 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/lru/NameHashIntCacheLRU.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/lru/NameHashIntCacheLRU.java @@ -2,7 +2,7 @@ package org.apache.lucene.facet.taxonomy.writercache.lru; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/lru/NameIntCacheLRU.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/lru/NameIntCacheLRU.java index 24998cb..24d010a 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/lru/NameIntCacheLRU.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/lru/NameIntCacheLRU.java @@ -4,9 +4,8 @@ import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import org.apache.lucene.facet.taxonomy.CategoryPath; -import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; // javadocs -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -105,7 +104,7 @@ class NameIntCacheLRU { } private boolean isCacheFull() { - return (cache.size()>maxCacheSize); + return cache.size() > maxCacheSize; } void clear() { @@ -115,14 +114,13 @@ class NameIntCacheLRU { String stats() { return "#miss="+nMisses+" #hit="+nHits; } - + /** - * If cache is full remove least recently used entries from cache. - * Return true if anything was removed, false otherwise. + * If cache is full remove least recently used entries from cache. Return true + * if anything was removed, false otherwise. * - * See comment in {@link DirectoryTaxonomyWriter#addToCache(CategoryPath, int)} - * for an explanation why we clean 2/3rds of the cache, and not just one - * entry. + * See comment in DirectoryTaxonomyWriter.addToCache(CategoryPath, int) for an + * explanation why we clean 2/3rds of the cache, and not just one entry. */ boolean makeRoomLRU() { if (!isCacheFull()) { diff --git a/lucene/facet/src/java/org/apache/lucene/facet/util/MultiCategoryListIterator.java b/lucene/facet/src/java/org/apache/lucene/facet/util/MultiCategoryListIterator.java index 3a6e509..cdea679 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/util/MultiCategoryListIterator.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/util/MultiCategoryListIterator.java @@ -6,7 +6,7 @@ import java.util.List; import org.apache.lucene.facet.search.CategoryListIterator; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/util/PartitionsUtils.java b/lucene/facet/src/java/org/apache/lucene/facet/util/PartitionsUtils.java index b287de6..4c6673c 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/util/PartitionsUtils.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/util/PartitionsUtils.java @@ -5,7 +5,7 @@ import org.apache.lucene.facet.index.params.FacetIndexingParams; import org.apache.lucene.facet.search.params.FacetSearchParams; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/util/ResultSortUtils.java b/lucene/facet/src/java/org/apache/lucene/facet/util/ResultSortUtils.java index 3eb42d6..49330e6 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/util/ResultSortUtils.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/util/ResultSortUtils.java @@ -11,7 +11,7 @@ import org.apache.lucene.facet.search.params.FacetRequest; import org.apache.lucene.facet.search.params.FacetRequest.SortOrder; import org.apache.lucene.facet.search.results.FacetResultNode; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java b/lucene/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java index 63a0b33..cd83769 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java @@ -14,7 +14,7 @@ import org.apache.lucene.util.OpenBitSetDISI; import org.apache.lucene.facet.search.ScoredDocIDs; import org.apache.lucene.facet.search.ScoredDocIDsIterator; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -125,13 +125,13 @@ public class ScoredDocIdsUtils { public boolean isCacheable() { return true; } @Override - public DocIdSetIterator iterator() throws IOException { + public DocIdSetIterator iterator() { return new DocIdSetIterator() { private int next = -1; @Override - public int advance(int target) throws IOException { + public int advance(int target) { while (next < size && docids[next++] < target) { } return next == size ? NO_MORE_DOCS : docids[next]; @@ -143,7 +143,7 @@ public class ScoredDocIdsUtils { } @Override - public int nextDoc() throws IOException { + public int nextDoc() { if (++next >= size) { return NO_MORE_DOCS; } @@ -155,7 +155,7 @@ public class ScoredDocIdsUtils { }; } - public ScoredDocIDsIterator iterator() throws IOException { + public ScoredDocIDsIterator iterator() { return new ScoredDocIDsIterator() { int next = -1; @@ -251,12 +251,12 @@ public class ScoredDocIdsUtils { } @Override - public DocIdSetIterator iterator() throws IOException { + public DocIdSetIterator iterator() { return new DocIdSetIterator() { private int next = -1; @Override - public int advance(int target) throws IOException { + public int advance(int target) { if (target <= next) { target = next + 1; } @@ -270,7 +270,7 @@ public class ScoredDocIdsUtils { } @Override - public int nextDoc() throws IOException { + public int nextDoc() { return ++next < maxDoc ? next : NO_MORE_DOCS; } @@ -337,13 +337,13 @@ public class ScoredDocIdsUtils { } @Override - public DocIdSetIterator iterator() throws IOException { + public DocIdSetIterator iterator() { return new DocIdSetIterator() { final Bits liveDocs = MultiFields.getLiveDocs(reader); private int next = -1; @Override - public int advance(int target) throws IOException { + public int advance(int target) { if (target > next) { next = target - 1; } @@ -356,7 +356,7 @@ public class ScoredDocIdsUtils { } @Override - public int nextDoc() throws IOException { + public int nextDoc() { do { ++next; } while (next < maxDoc && liveDocs != null && !liveDocs.get(next)); diff --git a/lucene/facet/src/java/org/apache/lucene/util/UnsafeByteArrayInputStream.java b/lucene/facet/src/java/org/apache/lucene/util/UnsafeByteArrayInputStream.java index 1642b62..10b3ccb 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/UnsafeByteArrayInputStream.java +++ b/lucene/facet/src/java/org/apache/lucene/util/UnsafeByteArrayInputStream.java @@ -4,7 +4,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/UnsafeByteArrayOutputStream.java b/lucene/facet/src/java/org/apache/lucene/util/UnsafeByteArrayOutputStream.java index cc62405..80e5c8c 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/UnsafeByteArrayOutputStream.java +++ b/lucene/facet/src/java/org/apache/lucene/util/UnsafeByteArrayOutputStream.java @@ -3,7 +3,7 @@ package org.apache.lucene.util; import java.io.IOException; import java.io.OutputStream; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/Vint8.java b/lucene/facet/src/java/org/apache/lucene/util/Vint8.java index 5311d53..40dd96d 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/Vint8.java +++ b/lucene/facet/src/java/org/apache/lucene/util/Vint8.java @@ -5,7 +5,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/collections/ArrayHashMap.java b/lucene/facet/src/java/org/apache/lucene/util/collections/ArrayHashMap.java index 858d3bf..b3df7d7 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/collections/ArrayHashMap.java +++ b/lucene/facet/src/java/org/apache/lucene/util/collections/ArrayHashMap.java @@ -3,7 +3,7 @@ package org.apache.lucene.util.collections; import java.util.Arrays; import java.util.Iterator; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/collections/DoubleIterator.java b/lucene/facet/src/java/org/apache/lucene/util/collections/DoubleIterator.java index e301308..c3c9261 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/collections/DoubleIterator.java +++ b/lucene/facet/src/java/org/apache/lucene/util/collections/DoubleIterator.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.collections; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/collections/FloatIterator.java b/lucene/facet/src/java/org/apache/lucene/util/collections/FloatIterator.java index 9d8894b..7e6a728 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/collections/FloatIterator.java +++ b/lucene/facet/src/java/org/apache/lucene/util/collections/FloatIterator.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.collections; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/collections/FloatToObjectMap.java b/lucene/facet/src/java/org/apache/lucene/util/collections/FloatToObjectMap.java index 77e99da..a3ed9ad 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/collections/FloatToObjectMap.java +++ b/lucene/facet/src/java/org/apache/lucene/util/collections/FloatToObjectMap.java @@ -3,7 +3,7 @@ package org.apache.lucene.util.collections; import java.util.Arrays; import java.util.Iterator; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/collections/IntArray.java b/lucene/facet/src/java/org/apache/lucene/util/collections/IntArray.java index 136955c..d1d59ef 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/collections/IntArray.java +++ b/lucene/facet/src/java/org/apache/lucene/util/collections/IntArray.java @@ -2,7 +2,7 @@ package org.apache.lucene.util.collections; import java.util.Arrays; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/collections/IntHashSet.java b/lucene/facet/src/java/org/apache/lucene/util/collections/IntHashSet.java index 3d3d164..288bad3 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/collections/IntHashSet.java +++ b/lucene/facet/src/java/org/apache/lucene/util/collections/IntHashSet.java @@ -2,7 +2,7 @@ package org.apache.lucene.util.collections; import java.util.Arrays; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/collections/IntIterator.java b/lucene/facet/src/java/org/apache/lucene/util/collections/IntIterator.java index 740b133..9bd9ea3 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/collections/IntIterator.java +++ b/lucene/facet/src/java/org/apache/lucene/util/collections/IntIterator.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.collections; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/collections/IntToDoubleMap.java b/lucene/facet/src/java/org/apache/lucene/util/collections/IntToDoubleMap.java index 8698866..f0da36c 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/collections/IntToDoubleMap.java +++ b/lucene/facet/src/java/org/apache/lucene/util/collections/IntToDoubleMap.java @@ -2,7 +2,7 @@ package org.apache.lucene.util.collections; import java.util.Arrays; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/collections/IntToIntMap.java b/lucene/facet/src/java/org/apache/lucene/util/collections/IntToIntMap.java index 2db467d..c9d8716 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/collections/IntToIntMap.java +++ b/lucene/facet/src/java/org/apache/lucene/util/collections/IntToIntMap.java @@ -2,7 +2,7 @@ package org.apache.lucene.util.collections; import java.util.Arrays; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/collections/IntToObjectMap.java b/lucene/facet/src/java/org/apache/lucene/util/collections/IntToObjectMap.java index 211c73c..bf0ce07 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/collections/IntToObjectMap.java +++ b/lucene/facet/src/java/org/apache/lucene/util/collections/IntToObjectMap.java @@ -3,7 +3,7 @@ package org.apache.lucene.util.collections; import java.util.Arrays; import java.util.Iterator; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/collections/LRUHashMap.java b/lucene/facet/src/java/org/apache/lucene/util/collections/LRUHashMap.java index 28a2258..d022c81 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/collections/LRUHashMap.java +++ b/lucene/facet/src/java/org/apache/lucene/util/collections/LRUHashMap.java @@ -3,7 +3,7 @@ package org.apache.lucene.util.collections; import java.util.LinkedHashMap; import java.util.Map; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/collections/ObjectToFloatMap.java b/lucene/facet/src/java/org/apache/lucene/util/collections/ObjectToFloatMap.java index e059e54..143c24c 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/collections/ObjectToFloatMap.java +++ b/lucene/facet/src/java/org/apache/lucene/util/collections/ObjectToFloatMap.java @@ -3,7 +3,7 @@ package org.apache.lucene.util.collections; import java.util.Arrays; import java.util.Iterator; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/collections/ObjectToIntMap.java b/lucene/facet/src/java/org/apache/lucene/util/collections/ObjectToIntMap.java index f5c6e3a..5d55668 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/collections/ObjectToIntMap.java +++ b/lucene/facet/src/java/org/apache/lucene/util/collections/ObjectToIntMap.java @@ -3,7 +3,7 @@ package org.apache.lucene.util.collections; import java.util.Arrays; import java.util.Iterator; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/ChunksIntEncoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/ChunksIntEncoder.java index 8e25500..2038dc7 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/encoding/ChunksIntEncoder.java +++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/ChunksIntEncoder.java @@ -3,7 +3,7 @@ package org.apache.lucene.util.encoding; import java.io.IOException; import java.io.OutputStream; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntDecoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntDecoder.java index 75c6f75..a37a4b9 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntDecoder.java +++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntDecoder.java @@ -3,7 +3,7 @@ package org.apache.lucene.util.encoding; import java.io.IOException; import java.io.InputStream; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntEncoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntEncoder.java index 43bcb4e..8b1a0de 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntEncoder.java +++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntEncoder.java @@ -3,7 +3,7 @@ package org.apache.lucene.util.encoding; import java.io.IOException; import java.io.OutputStream; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntDecoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntDecoder.java index 5f8e22b..fe2b9c6 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntDecoder.java +++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntDecoder.java @@ -3,7 +3,7 @@ package org.apache.lucene.util.encoding; import java.io.IOException; import java.io.InputStream; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntEncoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntEncoder.java index e3e7d75..ac42772 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntEncoder.java +++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntEncoder.java @@ -2,7 +2,7 @@ package org.apache.lucene.util.encoding; import java.io.IOException; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntDecoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntDecoder.java index efafaca..a597f9e 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntDecoder.java +++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntDecoder.java @@ -3,7 +3,7 @@ package org.apache.lucene.util.encoding; import java.io.IOException; import java.io.InputStream; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntEncoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntEncoder.java index b2945ba..a1e227d 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntEncoder.java +++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntEncoder.java @@ -2,7 +2,7 @@ package org.apache.lucene.util.encoding; import java.io.IOException; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/IntDecoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/IntDecoder.java index ebc516f..faabad4 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/encoding/IntDecoder.java +++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/IntDecoder.java @@ -3,7 +3,7 @@ package org.apache.lucene.util.encoding; import java.io.IOException; import java.io.InputStream; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java index bef6a34..7d421d2 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java +++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java @@ -3,7 +3,7 @@ package org.apache.lucene.util.encoding; import java.io.IOException; import java.io.OutputStream; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoderFilter.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoderFilter.java index ac49bbd..67879bd 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoderFilter.java +++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoderFilter.java @@ -3,7 +3,7 @@ package org.apache.lucene.util.encoding; import java.io.IOException; import java.io.OutputStream; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntDecoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntDecoder.java index 2b93d0e..6d00e04 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntDecoder.java +++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntDecoder.java @@ -3,7 +3,7 @@ package org.apache.lucene.util.encoding; import java.io.IOException; import java.io.InputStream; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java index f8e5015..badfe1c 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java +++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java @@ -3,7 +3,7 @@ package org.apache.lucene.util.encoding; import java.io.IOException; import java.io.OutputStream; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java index fb7ea44..414af6e 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java +++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java @@ -3,7 +3,7 @@ package org.apache.lucene.util.encoding; import java.io.IOException; import java.io.StreamCorruptedException; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java index 1124bd7..2e17ef2 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java +++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java @@ -2,7 +2,7 @@ package org.apache.lucene.util.encoding; import java.io.IOException; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java index 107d67c..9f6cee6 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java +++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java @@ -4,7 +4,7 @@ import java.io.IOException; import java.io.OutputStream; import java.util.Arrays; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java index 197962d..3583402 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java +++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java @@ -3,7 +3,7 @@ package org.apache.lucene.util.encoding; import java.io.IOException; import java.io.OutputStream; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java index 04cda37..2beaf77 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java +++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java @@ -2,7 +2,7 @@ package org.apache.lucene.util.encoding; import java.io.IOException; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java b/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java index 228951d..8fe6fbc 100644 --- a/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java +++ b/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java @@ -2,7 +2,7 @@ package org.apache.lucene.util.encoding; import java.io.IOException; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java b/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java index 3350ac5..97e4dfa 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java @@ -15,7 +15,7 @@ import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig; @@ -49,7 +49,7 @@ import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; import org.junit.AfterClass; import org.junit.BeforeClass; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -91,7 +91,7 @@ public abstract class FacetTestBase extends LuceneTestCase { protected IndexSearcher searcher; @BeforeClass - public static void beforeClassFacetTestBase() throws Exception { + public static void beforeClassFacetTestBase() { TEST_DIR = _TestUtil.getTempDir("facets"); dirsPerPartitionSize = new HashMap(); } @@ -173,7 +173,7 @@ public abstract class FacetTestBase extends LuceneTestCase { // prepare for searching taxoReader = new DirectoryTaxonomyReader(pair.taxoDir); - indexReader = IndexReader.open(pair.searchDir); + indexReader = DirectoryReader.open(pair.searchDir); searcher = newSearcher(indexReader); } @@ -214,7 +214,7 @@ public abstract class FacetTestBase extends LuceneTestCase { *

      Subclasses can override this to test different scenarios */ protected void populateIndex(RandomIndexWriter iw, TaxonomyWriter taxo, FacetIndexingParams iParams) - throws IOException, CorruptIndexException { + throws IOException { // add test documents int numDocsToIndex = numDocsToIndex(); for (int doc=0; doc categories) throws IOException, - CorruptIndexException { + TaxonomyWriter tw, String content, List categories) throws IOException { Document d = new Document(); CategoryDocumentBuilder builder = new CategoryDocumentBuilder(tw, iParams); builder.setCategoryPaths(categories); builder.build(d); - d.add(new Field("content", content, TextField.TYPE_STORED)); + d.add(new TextField("content", content, Field.Store.YES)); iw.addDocument(d); } diff --git a/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java b/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java index 9c60f8d..fe2acf2 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java @@ -8,9 +8,7 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.RandomIndexWriter; @@ -35,7 +33,7 @@ import org.apache.lucene.facet.taxonomy.TaxonomyWriter; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -54,7 +52,7 @@ import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; public class FacetTestUtils { - public static Directory[][] createIndexTaxonomyDirs(int number) throws IOException { + public static Directory[][] createIndexTaxonomyDirs(int number) { Directory[][] dirs = new Directory[number][2]; for (int i = 0; i < number; i++) { dirs[i][0] = LuceneTestCase.newDirectory(); @@ -68,7 +66,7 @@ public class FacetTestUtils { IndexTaxonomyReaderPair[] pairs = new IndexTaxonomyReaderPair[dirs.length]; for (int i = 0; i < dirs.length; i++) { IndexTaxonomyReaderPair pair = new IndexTaxonomyReaderPair(); - pair.indexReader = IndexReader.open(dirs[i][0]); + pair.indexReader = DirectoryReader.open(dirs[i][0]); pair.indexSearcher = new IndexSearcher(pair.indexReader); pair.taxReader = new DirectoryTaxonomyReader(dirs[i][1]); pairs[i] = pair; @@ -94,8 +92,7 @@ public class FacetTestUtils { public static Collector[] search(IndexSearcher searcher, TaxonomyReader taxonomyReader, DefaultFacetIndexingParams iParams, - int k, String... facetNames) throws IOException, - IllegalAccessException, InstantiationException { + int k, String... facetNames) throws IOException { Collector[] collectors = new Collector[2]; @@ -122,14 +119,13 @@ public class FacetTestUtils { } public static void add(FacetIndexingParams iParams, RandomIndexWriter iw, - TaxonomyWriter tw, String... strings) throws IOException, - CorruptIndexException { + TaxonomyWriter tw, String... strings) throws IOException { ArrayList cps = new ArrayList(); CategoryPath cp = new CategoryPath(strings); cps.add(cp); Document d = new Document(); new CategoryDocumentBuilder(tw, iParams).setCategoryPaths(cps).build(d); - d.add(new Field("content", "alpha", TextField.TYPE_STORED)); + d.add(new TextField("content", "alpha", Field.Store.YES)); iw.addDocument(d); } diff --git a/lucene/facet/src/test/org/apache/lucene/facet/enhancements/CategoryEnhancementDummy1.java b/lucene/facet/src/test/org/apache/lucene/facet/enhancements/CategoryEnhancementDummy1.java index bd49b96..4261627 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/enhancements/CategoryEnhancementDummy1.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/enhancements/CategoryEnhancementDummy1.java @@ -9,7 +9,7 @@ import org.apache.lucene.facet.index.attributes.CategoryProperty; import org.apache.lucene.facet.index.streaming.CategoryListTokenizer; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/enhancements/CategoryEnhancementDummy2.java b/lucene/facet/src/test/org/apache/lucene/facet/enhancements/CategoryEnhancementDummy2.java index a23fe5e..28349be 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/enhancements/CategoryEnhancementDummy2.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/enhancements/CategoryEnhancementDummy2.java @@ -10,7 +10,7 @@ import org.apache.lucene.facet.index.attributes.CategoryProperty; import org.apache.lucene.facet.index.streaming.CategoryListTokenizer; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/enhancements/CategoryEnhancementDummy3.java b/lucene/facet/src/test/org/apache/lucene/facet/enhancements/CategoryEnhancementDummy3.java index e1cae80..c19a283 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/enhancements/CategoryEnhancementDummy3.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/enhancements/CategoryEnhancementDummy3.java @@ -9,7 +9,7 @@ import org.apache.lucene.facet.index.attributes.CategoryProperty; import org.apache.lucene.facet.index.streaming.CategoryListTokenizer; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/enhancements/EnhancementsPayloadIteratorTest.java b/lucene/facet/src/test/org/apache/lucene/facet/enhancements/EnhancementsPayloadIteratorTest.java index d038cba..65e768c 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/enhancements/EnhancementsPayloadIteratorTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/enhancements/EnhancementsPayloadIteratorTest.java @@ -2,6 +2,7 @@ package org.apache.lucene.facet.enhancements; import java.io.IOException; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; @@ -18,7 +19,7 @@ import org.apache.lucene.facet.example.association.AssociationUtils; import org.apache.lucene.facet.search.DrillDown; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -61,7 +62,7 @@ public class EnhancementsPayloadIteratorTest extends LuceneTestCase { @Test public void testFullIterator() throws IOException { - IndexReader indexReader = IndexReader.open(indexDir); + IndexReader indexReader = DirectoryReader.open(indexDir); Term term = DrillDown.term(indexingParams, new CategoryPath("tags", "lucene")); EnhancementsPayloadIterator iterator = new EnhancementsPayloadIterator( indexingParams.getCategoryEnhancements(), indexReader, term); @@ -77,7 +78,7 @@ public class EnhancementsPayloadIteratorTest extends LuceneTestCase { @Test public void testEmptyIterator() throws IOException { - IndexReader indexReader = IndexReader.open(indexDir); + IndexReader indexReader = DirectoryReader.open(indexDir); Term term = DrillDown.term(indexingParams, new CategoryPath("root","a", "f2")); EnhancementsPayloadIterator iterator = new EnhancementsPayloadIterator( indexingParams.getCategoryEnhancements(), indexReader, term); @@ -89,7 +90,7 @@ public class EnhancementsPayloadIteratorTest extends LuceneTestCase { @Test public void testPartialIterator() throws IOException { - IndexReader indexReader = IndexReader.open(indexDir); + IndexReader indexReader = DirectoryReader.open(indexDir); Term term = DrillDown.term(indexingParams, new CategoryPath("genre","software")); EnhancementsPayloadIterator iterator = new EnhancementsPayloadIterator( indexingParams.getCategoryEnhancements(), indexReader, term); diff --git a/lucene/facet/src/test/org/apache/lucene/facet/enhancements/TwoEnhancementsTest.java b/lucene/facet/src/test/org/apache/lucene/facet/enhancements/TwoEnhancementsTest.java index 9a126bd..ddfb57d 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/enhancements/TwoEnhancementsTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/enhancements/TwoEnhancementsTest.java @@ -23,7 +23,7 @@ import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/enhancements/association/AssociationPropertyTest.java b/lucene/facet/src/test/org/apache/lucene/facet/enhancements/association/AssociationPropertyTest.java index e6f2f5a..c70eadb 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/enhancements/association/AssociationPropertyTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/enhancements/association/AssociationPropertyTest.java @@ -8,7 +8,7 @@ import org.apache.lucene.facet.enhancements.association.AssociationIntProperty; import org.apache.lucene.facet.enhancements.association.AssociationProperty; import org.apache.lucene.util.LuceneTestCase; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/enhancements/association/CustomAssociationPropertyTest.java b/lucene/facet/src/test/org/apache/lucene/facet/enhancements/association/CustomAssociationPropertyTest.java index 7a3d5ab..9a1f637 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/enhancements/association/CustomAssociationPropertyTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/enhancements/association/CustomAssociationPropertyTest.java @@ -19,7 +19,7 @@ import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/enhancements/params/DefaultEnhancementsIndexingParamsTest.java b/lucene/facet/src/test/org/apache/lucene/facet/enhancements/params/DefaultEnhancementsIndexingParamsTest.java index c151b54..9bf0861 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/enhancements/params/DefaultEnhancementsIndexingParamsTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/enhancements/params/DefaultEnhancementsIndexingParamsTest.java @@ -13,7 +13,7 @@ import org.apache.lucene.facet.enhancements.params.EnhancementsIndexingParams; import org.apache.lucene.facet.index.DummyProperty; import org.apache.lucene.facet.index.attributes.CategoryProperty; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/example/TestAdaptiveExample.java b/lucene/facet/src/test/org/apache/lucene/facet/example/TestAdaptiveExample.java index d9144e8..f6e2c03 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/example/TestAdaptiveExample.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/example/TestAdaptiveExample.java @@ -6,7 +6,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.facet.example.ExampleResult; import org.apache.lucene.facet.example.adaptive.AdaptiveMain; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/example/TestAssociationExample.java b/lucene/facet/src/test/org/apache/lucene/facet/example/TestAssociationExample.java index bbc0caa..c630fe9 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/example/TestAssociationExample.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/example/TestAssociationExample.java @@ -7,7 +7,7 @@ import org.apache.lucene.facet.example.ExampleResult; import org.apache.lucene.facet.example.association.AssociationMain; import org.apache.lucene.facet.search.results.FacetResultNode; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/example/TestMultiCLExample.java b/lucene/facet/src/test/org/apache/lucene/facet/example/TestMultiCLExample.java index 481f5c6..eea5e18 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/example/TestMultiCLExample.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/example/TestMultiCLExample.java @@ -6,12 +6,11 @@ import java.util.List; import org.junit.Test; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.facet.example.ExampleResult; import org.apache.lucene.facet.example.multiCL.MultiCLMain; import org.apache.lucene.facet.search.results.FacetResult; import org.apache.lucene.facet.search.results.FacetResultNode; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -40,8 +39,7 @@ public class TestMultiCLExample extends LuceneTestCase { assertCorrectMultiResults(res); } - public static void assertCorrectMultiResults(ExampleResult exampleResults) - throws Exception { + public static void assertCorrectMultiResults(ExampleResult exampleResults) { List results = exampleResults.getFacetResults(); FacetResult result = results.get(0); assertNotNull("Result should not be null", result); diff --git a/lucene/facet/src/test/org/apache/lucene/facet/example/TestSimpleExample.java b/lucene/facet/src/test/org/apache/lucene/facet/example/TestSimpleExample.java index de4fdea..7615fe0 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/example/TestSimpleExample.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/example/TestSimpleExample.java @@ -10,7 +10,7 @@ import org.apache.lucene.facet.example.simple.SimpleMain; import org.apache.lucene.facet.search.results.FacetResult; import org.apache.lucene.facet.search.results.FacetResultNode; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/index/CategoryContainerTest.java b/lucene/facet/src/test/org/apache/lucene/facet/index/CategoryContainerTest.java index 20ef633..dfced81 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/index/CategoryContainerTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/index/CategoryContainerTest.java @@ -18,7 +18,7 @@ import org.apache.lucene.facet.index.attributes.CategoryAttributeImpl; import org.apache.lucene.facet.index.streaming.CategoryAttributesStream; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/index/CategoryContainerTestBase.java b/lucene/facet/src/test/org/apache/lucene/facet/index/CategoryContainerTestBase.java index e85ea2e..03e1f00 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/index/CategoryContainerTestBase.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/index/CategoryContainerTestBase.java @@ -6,7 +6,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.facet.index.CategoryContainer; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/index/CategoryListPayloadStreamTest.java b/lucene/facet/src/test/org/apache/lucene/facet/index/CategoryListPayloadStreamTest.java index 218c1eb..d33d1ea 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/index/CategoryListPayloadStreamTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/index/CategoryListPayloadStreamTest.java @@ -13,7 +13,7 @@ import org.apache.lucene.util.encoding.NOnesIntDecoder; import org.apache.lucene.util.encoding.NOnesIntEncoder; import org.apache.lucene.util.encoding.UniqueValuesIntEncoder; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/index/DummyProperty.java b/lucene/facet/src/test/org/apache/lucene/facet/index/DummyProperty.java index 81e91d5..fd2ab7a 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/index/DummyProperty.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/index/DummyProperty.java @@ -2,7 +2,7 @@ package org.apache.lucene.facet.index; import org.apache.lucene.facet.index.attributes.CategoryProperty; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/index/FacetsPayloadProcessorProviderTest.java b/lucene/facet/src/test/org/apache/lucene/facet/index/FacetsPayloadProcessorProviderTest.java index dd4ea8b..9f5e890 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/index/FacetsPayloadProcessorProviderTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/index/FacetsPayloadProcessorProviderTest.java @@ -26,7 +26,7 @@ import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/index/attributes/CategoryAttributeImplTest.java b/lucene/facet/src/test/org/apache/lucene/facet/index/attributes/CategoryAttributeImplTest.java index fb1b27a..63c4816 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/index/attributes/CategoryAttributeImplTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/index/attributes/CategoryAttributeImplTest.java @@ -12,7 +12,7 @@ import org.apache.lucene.facet.index.attributes.CategoryAttribute; import org.apache.lucene.facet.index.attributes.CategoryAttributeImpl; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/index/attributes/CategoryAttributesIterableTest.java b/lucene/facet/src/test/org/apache/lucene/facet/index/attributes/CategoryAttributesIterableTest.java index 6301b0f..e6e2426 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/index/attributes/CategoryAttributesIterableTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/index/attributes/CategoryAttributesIterableTest.java @@ -12,7 +12,7 @@ import org.apache.lucene.facet.index.attributes.CategoryAttribute; import org.apache.lucene.facet.index.attributes.CategoryAttributesIterable; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/OrdinalPolicyTest.java b/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/OrdinalPolicyTest.java index c6f9d23..724f3b9 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/OrdinalPolicyTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/OrdinalPolicyTest.java @@ -12,7 +12,7 @@ import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/PathPolicyTest.java b/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/PathPolicyTest.java index 0cef03d..ca6fbd9 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/PathPolicyTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/index/categorypolicy/PathPolicyTest.java @@ -11,7 +11,7 @@ import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java b/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java index 37b8f26..4ea2aea 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java @@ -6,7 +6,7 @@ import org.junit.Test; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.facet.index.params.CategoryListParams; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/index/params/DefaultFacetIndexingParamsTest.java b/lucene/facet/src/test/org/apache/lucene/facet/index/params/DefaultFacetIndexingParamsTest.java index 849c2b5..dedbf8b 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/index/params/DefaultFacetIndexingParamsTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/index/params/DefaultFacetIndexingParamsTest.java @@ -16,7 +16,7 @@ import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.facet.util.PartitionsUtils; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/index/params/PerDimensionIndexingParamsTest.java b/lucene/facet/src/test/org/apache/lucene/facet/index/params/PerDimensionIndexingParamsTest.java index 3046286..eb2759d 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/index/params/PerDimensionIndexingParamsTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/index/params/PerDimensionIndexingParamsTest.java @@ -11,7 +11,7 @@ import org.apache.lucene.facet.search.DrillDown; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.util.PartitionsUtils; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/index/streaming/CategoryAttributesStreamTest.java b/lucene/facet/src/test/org/apache/lucene/facet/index/streaming/CategoryAttributesStreamTest.java index ace014b..bfdc631 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/index/streaming/CategoryAttributesStreamTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/index/streaming/CategoryAttributesStreamTest.java @@ -13,7 +13,7 @@ import org.apache.lucene.facet.index.attributes.CategoryAttributeImpl; import org.apache.lucene.facet.index.streaming.CategoryAttributesStream; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/index/streaming/CategoryParentsStreamTest.java b/lucene/facet/src/test/org/apache/lucene/facet/index/streaming/CategoryParentsStreamTest.java index eb57984..cc3aff4 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/index/streaming/CategoryParentsStreamTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/index/streaming/CategoryParentsStreamTest.java @@ -15,13 +15,10 @@ import org.apache.lucene.facet.index.categorypolicy.OrdinalPolicy; import org.apache.lucene.facet.index.categorypolicy.PathPolicy; import org.apache.lucene.facet.index.params.DefaultFacetIndexingParams; import org.apache.lucene.facet.index.params.FacetIndexingParams; -import org.apache.lucene.facet.index.streaming.CategoryAttributesStream; -import org.apache.lucene.facet.index.streaming.CategoryListTokenizer; -import org.apache.lucene.facet.index.streaming.CategoryParentsStream; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -116,7 +113,7 @@ public class CategoryParentsStreamTest extends CategoryContainerTestBase { * @throws FacetException */ @Test - public void testNoRetainableAttributes() throws IOException, FacetException { + public void testNoRetainableAttributes() throws IOException { Directory directory = newDirectory(); TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(directory); @@ -150,7 +147,7 @@ public class CategoryParentsStreamTest extends CategoryContainerTestBase { * @throws FacetException */ @Test - public void testRetainableAttributes() throws IOException, FacetException { + public void testRetainableAttributes() throws IOException { Directory directory = newDirectory(); TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter( directory); diff --git a/lucene/facet/src/test/org/apache/lucene/facet/index/streaming/CategoryTokenizerTest.java b/lucene/facet/src/test/org/apache/lucene/facet/index/streaming/CategoryTokenizerTest.java index 8d52ab1..14e46bc 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/index/streaming/CategoryTokenizerTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/index/streaming/CategoryTokenizerTest.java @@ -19,7 +19,7 @@ import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/AdaptiveAccumulatorTest.java b/lucene/facet/src/test/org/apache/lucene/facet/search/AdaptiveAccumulatorTest.java index 498a4dd..7003fbb 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/search/AdaptiveAccumulatorTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/search/AdaptiveAccumulatorTest.java @@ -1,13 +1,14 @@ package org.apache.lucene.facet.search; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.lucene.facet.search.params.FacetSearchParams; import org.apache.lucene.facet.search.sampling.BaseSampleTestTopK; import org.apache.lucene.facet.search.sampling.Sampler; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -24,6 +25,7 @@ import org.apache.lucene.facet.taxonomy.TaxonomyReader; * limitations under the License. */ +@Slow public class AdaptiveAccumulatorTest extends BaseSampleTestTopK { @Override diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java b/lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java index 3b8336a..c63e0fe 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java @@ -4,7 +4,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.util._TestUtil; @@ -17,7 +16,7 @@ import org.apache.lucene.facet.search.params.FacetSearchParams; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -51,7 +50,7 @@ public abstract class BaseTestTopK extends FacetTestBase { @Override protected void populateIndex(RandomIndexWriter iw, TaxonomyWriter taxo, - FacetIndexingParams iParams) throws IOException, CorruptIndexException { + FacetIndexingParams iParams) throws IOException { currDoc = -1; super.populateIndex(iw, taxo, iParams); } diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/CategoryListIteratorTest.java b/lucene/facet/src/test/org/apache/lucene/facet/search/CategoryListIteratorTest.java index dcfec09..ddb4ac5 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/search/CategoryListIteratorTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/search/CategoryListIteratorTest.java @@ -12,12 +12,13 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Payload; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.UnsafeByteArrayOutputStream; import org.apache.lucene.util.encoding.DGapIntEncoder; @@ -27,7 +28,7 @@ import org.apache.lucene.util.encoding.UniqueValuesIntEncoder; import org.apache.lucene.util.encoding.VInt8IntEncoder; import org.junit.Test; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -56,7 +57,7 @@ public class CategoryListIteratorTest extends LuceneTestCase { private boolean exhausted = false; private CharTermAttribute term = addAttribute(CharTermAttribute.class); - public DataTokenStream(String text, IntEncoder encoder) throws IOException { + public DataTokenStream(String text, IntEncoder encoder) { this.encoder = encoder; term.setEmpty().append(text); } @@ -79,7 +80,7 @@ public class CategoryListIteratorTest extends LuceneTestCase { encoder.encode(val); } encoder.close(); - payload.setPayload(new Payload(buf, 0, ubaos.length())); + payload.setPayload(new BytesRef(buf, 0, ubaos.length())); exhausted = true; return true; @@ -152,7 +153,7 @@ public class CategoryListIteratorTest extends LuceneTestCase { dts.setIdx(i); doc.add(new TextField("f", dts)); // only doc 0 has payloads! } else { - doc.add(new TextField("f", "1")); + doc.add(new TextField("f", "1", Field.Store.NO)); } writer.addDocument(doc); writer.commit(); diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownTest.java b/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownTest.java index 7f3e9b9..22a0f86 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownTest.java @@ -6,8 +6,8 @@ import java.util.ArrayList; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; @@ -16,7 +16,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.LockObtainFailedException; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -31,7 +30,7 @@ import org.apache.lucene.facet.taxonomy.TaxonomyWriter; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -57,7 +56,7 @@ public class DrillDownTest extends LuceneTestCase { private static Directory dir; private static Directory taxoDir; - public DrillDownTest() throws IOException { + public DrillDownTest() { PerDimensionIndexingParams iParams = new PerDimensionIndexingParams(); CategoryListParams aClParams = new CategoryListParams(new Term("testing_facets_a", "a")); CategoryListParams bClParams = new CategoryListParams(new Term("testing_facets_b", "b")); @@ -69,7 +68,7 @@ public class DrillDownTest extends LuceneTestCase { } @BeforeClass - public static void createIndexes() throws CorruptIndexException, LockObtainFailedException, IOException { + public static void createIndexes() throws IOException { dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.KEYWORD, false))); @@ -81,10 +80,10 @@ public class DrillDownTest extends LuceneTestCase { ArrayList paths = new ArrayList(); Document doc = new Document(); if (i % 2 == 0) { // 50 - doc.add(new TextField("content", "foo")); + doc.add(new TextField("content", "foo", Field.Store.NO)); } if (i % 3 == 0) { // 33 - doc.add(new TextField("content", "bar")); + doc.add(new TextField("content", "bar", Field.Store.NO)); } if (i % 4 == 0) { // 25 paths.add(new CategoryPath("a")); diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/SamplingWrapperTest.java b/lucene/facet/src/test/org/apache/lucene/facet/search/SamplingWrapperTest.java index e70930a..eb9af30 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/search/SamplingWrapperTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/search/SamplingWrapperTest.java @@ -1,13 +1,14 @@ package org.apache.lucene.facet.search; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.lucene.facet.search.params.FacetSearchParams; import org.apache.lucene.facet.search.sampling.BaseSampleTestTopK; import org.apache.lucene.facet.search.sampling.Sampler; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -24,6 +25,7 @@ import org.apache.lucene.facet.taxonomy.TaxonomyReader; * limitations under the License. */ +@Slow public class SamplingWrapperTest extends BaseSampleTestTopK { @Override diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/TestCategoryListCache.java b/lucene/facet/src/test/org/apache/lucene/facet/search/TestCategoryListCache.java index f77b709..1ba5473 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/search/TestCategoryListCache.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/search/TestCategoryListCache.java @@ -20,7 +20,7 @@ import org.apache.lucene.facet.search.params.FacetSearchParams; import org.apache.lucene.facet.search.results.FacetResult; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -72,7 +72,7 @@ public class TestCategoryListCache extends FacetTestBase { doTest(true,true); } - private void doTest(boolean withCache, boolean plantWrongData) throws IOException, Exception { + private void doTest(boolean withCache, boolean plantWrongData) throws Exception { Map truth = facetCountsTruth(); CategoryPath cp = (CategoryPath) truth.keySet().toArray()[0]; // any category path will do for this test CountFacetRequest frq = new CountFacetRequest(cp, 10); diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetArrays.java b/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetArrays.java index 557a7f6..5d71cd7 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetArrays.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetArrays.java @@ -7,7 +7,7 @@ import org.apache.lucene.facet.search.FacetArrays; import org.apache.lucene.facet.search.FloatArrayAllocator; import org.apache.lucene.facet.search.IntArrayAllocator; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java b/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java index 68246d1..f3071bc 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java @@ -24,7 +24,7 @@ import org.apache.lucene.facet.search.results.FacetResult; import org.apache.lucene.facet.search.results.FacetResultNode; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java b/lucene/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java index 838b85c..6f0c9e8 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java @@ -19,7 +19,6 @@ import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig.OpenMode; @@ -39,7 +38,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; import org.junit.Test; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -271,12 +270,12 @@ public class TestMultipleCategoryLists extends LuceneTestCase { IOUtils.close(dirs[0]); } - private Directory[][] getDirs() throws IOException { + private Directory[][] getDirs() { return FacetTestUtils.createIndexTaxonomyDirs(1); } private void assertCorrectResults(FacetsCollector facetsCollector) - throws IOException, IllegalAccessException, InstantiationException { + throws IOException { List res = facetsCollector.getFacetResults(); FacetResult results = res.get(0); @@ -354,7 +353,7 @@ public class TestMultipleCategoryLists extends LuceneTestCase { } private void seedIndex(RandomIndexWriter iw, TaxonomyWriter tw, - FacetIndexingParams iParams) throws IOException, CorruptIndexException { + FacetIndexingParams iParams) throws IOException { FacetTestUtils.add(iParams, iw, tw, "Author", "Mark Twain"); FacetTestUtils.add(iParams, iw, tw, "Author", "Stephen King"); FacetTestUtils.add(iParams, iw, tw, "Author", "Kurt Vonnegut"); diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/TestScoredDocIdCollector.java b/lucene/facet/src/test/org/apache/lucene/facet/search/TestScoredDocIdCollector.java index a5c8796..730ccc5 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/search/TestScoredDocIdCollector.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/search/TestScoredDocIdCollector.java @@ -23,7 +23,7 @@ import org.apache.lucene.facet.search.results.FacetResult; import org.apache.lucene.facet.search.results.FacetResultNode; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java b/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java index 6a7eeb9..d4b15b4 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java @@ -8,7 +8,6 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.RandomIndexWriter; @@ -33,7 +32,7 @@ import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; import org.apache.lucene.facet.util.PartitionsUtils; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -321,14 +320,13 @@ public class TestTopKInEachNodeResultHandler extends LuceneTestCase { } private void prvt_add(DefaultFacetIndexingParams iParams, RandomIndexWriter iw, - TaxonomyWriter tw, String... strings) throws IOException, - CorruptIndexException { + TaxonomyWriter tw, String... strings) throws IOException { ArrayList cps = new ArrayList(); CategoryPath cp = new CategoryPath(strings); cps.add(cp); Document d = new Document(); new CategoryDocumentBuilder(tw, iParams).setCategoryPaths(cps).build(d); - d.add(new Field("content", "alpha", TextField.TYPE_STORED)); + d.add(new TextField("content", "alpha", Field.Store.YES)); iw.addDocument(d); } diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java b/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java index 9b9748e..3195735 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java @@ -14,7 +14,7 @@ import org.apache.lucene.facet.search.results.FacetResultNode; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java b/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java index d97e111..1da763b 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java @@ -14,7 +14,7 @@ import org.apache.lucene.facet.search.results.FacetResult; import org.apache.lucene.facet.search.results.FacetResultNode; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -34,7 +34,7 @@ import org.apache.lucene.facet.taxonomy.TaxonomyReader; public class TestTopKResultsHandlerRandom extends BaseTestTopK { private List countFacets(int partitionSize, int numResults, final boolean doComplement) - throws IOException, IllegalAccessException, InstantiationException { + throws IOException { Query q = new MatchAllDocsQuery(); FacetSearchParams facetSearchParams = searchParamsWithRequests(numResults, partitionSize); FacetsCollector fc = new FacetsCollector(facetSearchParams, indexReader, taxoReader) { diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCounts.java b/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCounts.java index 4974945..04f1227 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCounts.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCounts.java @@ -15,7 +15,7 @@ import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyReaderPair; import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyWriterPair; import org.apache.lucene.facet.index.params.DefaultFacetIndexingParams; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java b/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java index 6b2800d..84255d2 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java @@ -8,7 +8,6 @@ import java.util.List; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; @@ -41,7 +40,7 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.SlowRAMDirectory; import org.apache.lucene.util._TestUtil; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -140,8 +139,7 @@ public class TestTotalFacetCountsCache extends LuceneTestCase { } private void doTestGeneralSynchronization(int numThreads, int sleepMillis, - int cacheSize) throws Exception, CorruptIndexException, IOException, - InterruptedException { + int cacheSize) throws Exception { TFC.setCacheSize(cacheSize); SlowRAMDirectory slowIndexDir = new SlowRAMDirectory(-1, random()); MockDirectoryWrapper indexDir = new MockDirectoryWrapper(random(), slowIndexDir); @@ -156,7 +154,7 @@ public class TestTotalFacetCountsCache extends LuceneTestCase { slowTaxoDir.setSleepMillis(sleepMillis); // Open the slow readers - IndexReader slowIndexReader = IndexReader.open(indexDir); + IndexReader slowIndexReader = DirectoryReader.open(indexDir); TaxonomyReader slowTaxoReader = new DirectoryTaxonomyReader(taxoDir); // Class to perform search and return results as threads @@ -421,7 +419,7 @@ public class TestTotalFacetCountsCache extends LuceneTestCase { indexDir.setSleepMillis(1); taxoDir.setSleepMillis(1); - IndexReader r = IndexReader.open(indexDir); + IndexReader r = DirectoryReader.open(indexDir); DirectoryTaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir); // Create and start threads. Thread1 should lock the cache and calculate diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/association/AssociationsFacetRequestTest.java b/lucene/facet/src/test/org/apache/lucene/facet/search/association/AssociationsFacetRequestTest.java index 978236e..0115e74 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/search/association/AssociationsFacetRequestTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/search/association/AssociationsFacetRequestTest.java @@ -32,7 +32,7 @@ import org.apache.lucene.facet.taxonomy.TaxonomyWriter; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/params/FacetRequestTest.java b/lucene/facet/src/test/org/apache/lucene/facet/search/params/FacetRequestTest.java index 5927472..6d22d8b 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/search/params/FacetRequestTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/search/params/FacetRequestTest.java @@ -11,7 +11,7 @@ import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/params/FacetSearchParamsTest.java b/lucene/facet/src/test/org/apache/lucene/facet/search/params/FacetSearchParamsTest.java index 40df7ba..c6e70e4 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/search/params/FacetSearchParamsTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/search/params/FacetSearchParamsTest.java @@ -12,7 +12,7 @@ import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; import org.apache.lucene.facet.util.PartitionsUtils; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/params/MultiIteratorsPerCLParamsTest.java b/lucene/facet/src/test/org/apache/lucene/facet/search/params/MultiIteratorsPerCLParamsTest.java index 679c924..6e256d2 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/search/params/MultiIteratorsPerCLParamsTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/search/params/MultiIteratorsPerCLParamsTest.java @@ -7,7 +7,7 @@ import java.util.List; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; -import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.store.Directory; @@ -36,7 +36,7 @@ import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; import org.apache.lucene.facet.util.ScoredDocIdsUtils; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -83,8 +83,7 @@ public class MultiIteratorsPerCLParamsTest extends LuceneTestCase { doTestCLParamMultiIteratorsByRequest(true); } - private void doTestCLParamMultiIteratorsByRequest(boolean cacheCLI) throws Exception, - CorruptIndexException, IOException { + private void doTestCLParamMultiIteratorsByRequest(boolean cacheCLI) throws Exception { // Create a CLP which generates different CLIs according to the // FacetRequest's dimension CategoryListParams clp = new CategoryListParams(); @@ -94,7 +93,7 @@ public class MultiIteratorsPerCLParamsTest extends LuceneTestCase { populateIndex(iParams, indexDir, taxoDir); TaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir); - IndexReader reader = IndexReader.open(indexDir); + IndexReader reader = DirectoryReader.open(indexDir); CategoryListCache clCache = null; if (cacheCLI) { diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java b/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java index 22defc1..536704f 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java @@ -1,6 +1,5 @@ package org.apache.lucene.facet.search.sampling; -import java.io.IOException; import java.util.List; import java.util.Random; @@ -19,7 +18,7 @@ import org.apache.lucene.facet.search.params.FacetSearchParams; import org.apache.lucene.facet.search.results.FacetResult; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -53,7 +52,7 @@ public abstract class BaseSampleTestTopK extends BaseTestTopK { * Lots of randomly generated data is being indexed, and later on a "90% docs" faceted search * is performed. The results are compared to non-sampled ones. */ - public void testCountUsingSamping() throws Exception, IOException { + public void testCountUsingSamping() throws Exception { boolean useRandomSampler = random().nextBoolean(); for (int partitionSize : partitionSizes) { try { diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/SamplingAccumulatorTest.java b/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/SamplingAccumulatorTest.java index 4bdaf8c..cefe4a1 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/SamplingAccumulatorTest.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/SamplingAccumulatorTest.java @@ -1,12 +1,13 @@ package org.apache.lucene.facet.search.sampling; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.lucene.facet.search.FacetsAccumulator; import org.apache.lucene.facet.search.params.FacetSearchParams; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,6 +24,7 @@ import org.apache.lucene.facet.taxonomy.TaxonomyReader; * limitations under the License. */ +@Slow public class SamplingAccumulatorTest extends BaseSampleTestTopK { @Override diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestCategoryPath.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestCategoryPath.java index 784c07c..b796879 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestCategoryPath.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestCategoryPath.java @@ -11,7 +11,7 @@ import org.junit.Test; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.facet.taxonomy.CategoryPath; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyCombined.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyCombined.java index 52f0f88..d221402 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyCombined.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyCombined.java @@ -18,7 +18,7 @@ import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; import org.apache.lucene.util.SlowRAMDirectory; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomy.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomy.java index bc59f78..a257f80 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomy.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomy.java @@ -14,7 +14,7 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java index 27c3adf..bb405b1 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java @@ -16,7 +16,7 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.junit.Test; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java index cfcd0fb..e708e6c 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java @@ -17,12 +17,13 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import org.junit.Test; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -43,31 +44,31 @@ public class TestDirectoryTaxonomyWriter extends LuceneTestCase { // A No-Op TaxonomyWriterCache which always discards all given categories, and // always returns true in put(), to indicate some cache entries were cleared. - private static class NoOpCache implements TaxonomyWriterCache { - - NoOpCache() { } + private static TaxonomyWriterCache NO_OP_CACHE = new TaxonomyWriterCache() { @Override public void close() {} @Override public int get(CategoryPath categoryPath) { return -1; } @Override - public int get(CategoryPath categoryPath, int length) { return get(categoryPath); } + public int get(CategoryPath categoryPath, int length) { return -1; } @Override public boolean put(CategoryPath categoryPath, int ordinal) { return true; } @Override public boolean put(CategoryPath categoryPath, int prefixLen, int ordinal) { return true; } @Override - public boolean hasRoom(int numberOfEntries) { return false; } + public boolean isFull() { return true; } + @Override + public void clear() {} - } + }; @Test public void testCommit() throws Exception { // Verifies that nothing is committed to the underlying Directory, if // commit() wasn't called. Directory dir = newDirectory(); - DirectoryTaxonomyWriter ltw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, new NoOpCache()); + DirectoryTaxonomyWriter ltw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE); assertFalse(DirectoryReader.indexExists(dir)); ltw.commit(); // first commit, so that an index will be created ltw.addCategory(new CategoryPath("a")); @@ -83,7 +84,7 @@ public class TestDirectoryTaxonomyWriter extends LuceneTestCase { public void testCommitUserData() throws Exception { // Verifies taxonomy commit data Directory dir = newDirectory(); - DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, new NoOpCache()); + DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE); taxoWriter.addCategory(new CategoryPath("a")); taxoWriter.addCategory(new CategoryPath("b")); Map userCommitData = new HashMap(); @@ -101,7 +102,7 @@ public class TestDirectoryTaxonomyWriter extends LuceneTestCase { // open DirTaxoWriter again and commit, INDEX_CREATE_TIME should still exist // in the commit data, otherwise DirTaxoReader.refresh() might not detect // that the taxonomy index has been recreated. - taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, new NoOpCache()); + taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE); taxoWriter.addCategory(new CategoryPath("c")); // add a category so that commit will happen taxoWriter.commit(new HashMap(){{ put("just", "data"); @@ -161,7 +162,7 @@ public class TestDirectoryTaxonomyWriter extends LuceneTestCase { // DirTaxoReader succeeding to refresh(). Directory dir = newDirectory(); - DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, new NoOpCache()); + DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE); touchTaxo(taxoWriter, new CategoryPath("a")); DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir); @@ -173,11 +174,11 @@ public class TestDirectoryTaxonomyWriter extends LuceneTestCase { // now recreate the taxonomy, and check that the timestamp is preserved after opening DirTW again. taxoWriter.close(); - taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, new NoOpCache()); + taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, NO_OP_CACHE); touchTaxo(taxoWriter, new CategoryPath("c")); taxoWriter.close(); - taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, new NoOpCache()); + taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE); touchTaxo(taxoWriter, new CategoryPath("d")); taxoWriter.close(); @@ -202,7 +203,7 @@ public class TestDirectoryTaxonomyWriter extends LuceneTestCase { // create an empty index first, so that DirTaxoWriter initializes createTime to null. new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null)).close(); - DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, new NoOpCache()); + DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE); // we cannot commit null keys/values, this ensures that if DirTW.createTime is null, we can still commit. taxoWriter.close(); @@ -214,14 +215,24 @@ public class TestDirectoryTaxonomyWriter extends LuceneTestCase { } public void testConcurrency() throws Exception { - int ncats = atLeast(100000); // add many categories + final int ncats = atLeast(100000); // add many categories final int range = ncats * 3; // affects the categories selection final AtomicInteger numCats = new AtomicInteger(ncats); - Directory dir = newDirectory(); + final Directory dir = newDirectory(); final ConcurrentHashMap values = new ConcurrentHashMap(); - TaxonomyWriterCache cache = random().nextBoolean() - ? new Cl2oTaxonomyWriterCache(1024, 0.15f, 3) - : new LruTaxonomyWriterCache(ncats / 10); + final double d = random().nextDouble(); + final TaxonomyWriterCache cache; + if (d < 0.7) { + // this is the fastest, yet most memory consuming + cache = new Cl2oTaxonomyWriterCache(1024, 0.15f, 3); + } else if (TEST_NIGHTLY && d > 0.98) { + // this is the slowest, but tests the writer concurrency when no caching is done. + // only pick it during NIGHTLY tests, and even then, with very low chances. + cache = NO_OP_CACHE; + } else { + // this is slower than CL2O, but less memory consuming, and exercises finding categories on disk too. + cache = new LruTaxonomyWriterCache(ncats / 10); + } final DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, cache); Thread[] addThreads = new Thread[atLeast(4)]; for (int z = 0; z < addThreads.length; z++) { @@ -247,7 +258,7 @@ public class TestDirectoryTaxonomyWriter extends LuceneTestCase { tw.close(); DirectoryTaxonomyReader dtr = new DirectoryTaxonomyReader(dir); - assertEquals(values.size() + 2, dtr.getSize()); // +2 for root category + "a" + assertEquals("mismatch number of categories", values.size() + 2, dtr.getSize()); // +2 for root category + "a" for (Integer value : values.keySet()) { assertTrue("category not found a/" + value, dtr.getOrdinal(new CategoryPath("a", value.toString())) > 0); } @@ -255,5 +266,57 @@ public class TestDirectoryTaxonomyWriter extends LuceneTestCase { dir.close(); } + + private String getCreateTime(Directory taxoDir) throws IOException { + SegmentInfos infos = new SegmentInfos(); + infos.read(taxoDir); + return infos.getUserData().get(DirectoryTaxonomyWriter.INDEX_CREATE_TIME); + } + + @Test + public void testReplaceTaxonomy() throws Exception { + Directory input = newDirectory(); + DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(input); + taxoWriter.addCategory(new CategoryPath("a")); + taxoWriter.close(); + + Directory dir = newDirectory(); + taxoWriter = new DirectoryTaxonomyWriter(dir); + int ordinal = taxoWriter.addCategory(new CategoryPath("b")); + taxoWriter.addCategory(new CategoryPath("c")); + taxoWriter.commit(); + + String origCreateTime = getCreateTime(dir); + + // replace the taxonomy with the input one + taxoWriter.replaceTaxonomy(input); + + // add the same category again -- it should not receive the same ordinal ! + int newOrdinal = taxoWriter.addCategory(new CategoryPath("b")); + assertNotSame("new ordinal cannot be the original ordinal", ordinal, newOrdinal); + assertEquals("ordinal should have been 2 since only one category was added by replaceTaxonomy", 2, newOrdinal); + + taxoWriter.close(); + + String newCreateTime = getCreateTime(dir); + assertNotSame("create time should have been changed after replaceTaxonomy", origCreateTime, newCreateTime); + + dir.close(); + input.close(); + } + + @Test + public void testReaderFreshness() throws Exception { + // ensures that the internal index reader is always kept fresh. Previously, + // this simple scenario failed, if the cache just evicted the category that + // is being added. + Directory dir = newDirectory(); + DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, NO_OP_CACHE); + int o1 = taxoWriter.addCategory(new CategoryPath("a")); + int o2 = taxoWriter.addCategory(new CategoryPath("a")); + assertTrue("ordinal for same category that is added twice should be the same !", o1 == o2); + taxoWriter.close(); + dir.close(); + } } diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/cl2o/TestCharBlockArray.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/cl2o/TestCharBlockArray.java index 49b7bb0..fba7a6b 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/cl2o/TestCharBlockArray.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/cl2o/TestCharBlockArray.java @@ -11,7 +11,7 @@ import org.junit.Test; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.facet.taxonomy.writercache.cl2o.CharBlockArray; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/cl2o/TestCompactLabelToOrdinal.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/cl2o/TestCompactLabelToOrdinal.java index 00e7165..dd51856 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/cl2o/TestCompactLabelToOrdinal.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/cl2o/TestCompactLabelToOrdinal.java @@ -11,7 +11,7 @@ import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.writercache.cl2o.CompactLabelToOrdinal; import org.apache.lucene.facet.taxonomy.writercache.cl2o.LabelToOrdinal; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/facet/util/TestScoredDocIDsUtils.java b/lucene/facet/src/test/org/apache/lucene/facet/util/TestScoredDocIDsUtils.java index d5e7949..cebb233 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/util/TestScoredDocIDsUtils.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/util/TestScoredDocIDsUtils.java @@ -9,6 +9,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.RandomIndexWriter; @@ -29,7 +30,7 @@ import org.apache.lucene.facet.search.ScoredDocIDs; import org.apache.lucene.facet.search.ScoredDocIDsIterator; import org.apache.lucene.facet.search.ScoredDocIdCollector; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -188,8 +189,8 @@ public class TestScoredDocIDsUtils extends LuceneTestCase { protected final static String delTxt = "delete"; protected final static String alphaTxt = "alpha"; - private final static Field deletionMark = new StringField(field, delTxt); - private final static Field alphaContent = new StringField(field, alphaTxt); + private final static Field deletionMark = new StringField(field, delTxt, Field.Store.NO); + private final static Field alphaContent = new StringField(field, alphaTxt, Field.Store.NO); protected final int numDocs; @@ -235,6 +236,6 @@ public class TestScoredDocIDsUtils extends LuceneTestCase { writer.close(); // Open a fresh read-only reader with the deletions in place - return IndexReader.open(dir); + return DirectoryReader.open(dir); } } diff --git a/lucene/facet/src/test/org/apache/lucene/util/SlowRAMDirectory.java b/lucene/facet/src/test/org/apache/lucene/util/SlowRAMDirectory.java index 41fa0b9..3b07f2e 100644 --- a/lucene/facet/src/test/org/apache/lucene/util/SlowRAMDirectory.java +++ b/lucene/facet/src/test/org/apache/lucene/util/SlowRAMDirectory.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/util/UnsafeByteArrayInputStreamTest.java b/lucene/facet/src/test/org/apache/lucene/util/UnsafeByteArrayInputStreamTest.java index ff13bd0..2c5dc76 100644 --- a/lucene/facet/src/test/org/apache/lucene/util/UnsafeByteArrayInputStreamTest.java +++ b/lucene/facet/src/test/org/apache/lucene/util/UnsafeByteArrayInputStreamTest.java @@ -8,7 +8,7 @@ import org.junit.Test; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.UnsafeByteArrayInputStream; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/util/UnsafeByteArrayOutputStreamTest.java b/lucene/facet/src/test/org/apache/lucene/util/UnsafeByteArrayOutputStreamTest.java index a472c44..b3bad79 100644 --- a/lucene/facet/src/test/org/apache/lucene/util/UnsafeByteArrayOutputStreamTest.java +++ b/lucene/facet/src/test/org/apache/lucene/util/UnsafeByteArrayOutputStreamTest.java @@ -7,7 +7,7 @@ import org.junit.Test; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.UnsafeByteArrayOutputStream; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/util/Vint8Test.java b/lucene/facet/src/test/org/apache/lucene/util/Vint8Test.java index 905f035..8214186 100644 --- a/lucene/facet/src/test/org/apache/lucene/util/Vint8Test.java +++ b/lucene/facet/src/test/org/apache/lucene/util/Vint8Test.java @@ -9,7 +9,7 @@ import org.junit.Test; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.Vint8; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/util/collections/ArrayHashMapTest.java b/lucene/facet/src/test/org/apache/lucene/util/collections/ArrayHashMapTest.java index d7977fe..2a79064 100644 --- a/lucene/facet/src/test/org/apache/lucene/util/collections/ArrayHashMapTest.java +++ b/lucene/facet/src/test/org/apache/lucene/util/collections/ArrayHashMapTest.java @@ -9,7 +9,7 @@ import org.junit.Test; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.collections.ArrayHashMap; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/util/collections/FloatToObjectMapTest.java b/lucene/facet/src/test/org/apache/lucene/util/collections/FloatToObjectMapTest.java index 980a08b..5b4d1a3 100644 --- a/lucene/facet/src/test/org/apache/lucene/util/collections/FloatToObjectMapTest.java +++ b/lucene/facet/src/test/org/apache/lucene/util/collections/FloatToObjectMapTest.java @@ -9,7 +9,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.collections.FloatIterator; import org.apache.lucene.util.collections.FloatToObjectMap; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/util/collections/IntArrayTest.java b/lucene/facet/src/test/org/apache/lucene/util/collections/IntArrayTest.java index 9111ef8..87843c1 100644 --- a/lucene/facet/src/test/org/apache/lucene/util/collections/IntArrayTest.java +++ b/lucene/facet/src/test/org/apache/lucene/util/collections/IntArrayTest.java @@ -5,7 +5,7 @@ import org.junit.Test; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.collections.IntArray; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/util/collections/IntHashSetTest.java b/lucene/facet/src/test/org/apache/lucene/util/collections/IntHashSetTest.java index ad64572..89b4bbf 100644 --- a/lucene/facet/src/test/org/apache/lucene/util/collections/IntHashSetTest.java +++ b/lucene/facet/src/test/org/apache/lucene/util/collections/IntHashSetTest.java @@ -7,7 +7,7 @@ import org.junit.Test; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.collections.IntHashSet; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/util/collections/IntToDoubleMapTest.java b/lucene/facet/src/test/org/apache/lucene/util/collections/IntToDoubleMapTest.java index 8b9bf5c..46d9d16 100644 --- a/lucene/facet/src/test/org/apache/lucene/util/collections/IntToDoubleMapTest.java +++ b/lucene/facet/src/test/org/apache/lucene/util/collections/IntToDoubleMapTest.java @@ -10,7 +10,7 @@ import org.apache.lucene.util.collections.IntToDoubleMap; import java.util.HashSet; import java.util.Random; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/util/collections/IntToIntMapTest.java b/lucene/facet/src/test/org/apache/lucene/util/collections/IntToIntMapTest.java index 88d66d7..dbd40d2 100644 --- a/lucene/facet/src/test/org/apache/lucene/util/collections/IntToIntMapTest.java +++ b/lucene/facet/src/test/org/apache/lucene/util/collections/IntToIntMapTest.java @@ -8,7 +8,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.collections.IntIterator; import org.apache.lucene.util.collections.IntToIntMap; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/util/collections/IntToObjectMapTest.java b/lucene/facet/src/test/org/apache/lucene/util/collections/IntToObjectMapTest.java index 535275d..7cb8b8b 100644 --- a/lucene/facet/src/test/org/apache/lucene/util/collections/IntToObjectMapTest.java +++ b/lucene/facet/src/test/org/apache/lucene/util/collections/IntToObjectMapTest.java @@ -10,7 +10,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.collections.IntIterator; import org.apache.lucene.util.collections.IntToObjectMap; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/util/collections/ObjectToFloatMapTest.java b/lucene/facet/src/test/org/apache/lucene/util/collections/ObjectToFloatMapTest.java index 684f2d6..d1c4f27 100644 --- a/lucene/facet/src/test/org/apache/lucene/util/collections/ObjectToFloatMapTest.java +++ b/lucene/facet/src/test/org/apache/lucene/util/collections/ObjectToFloatMapTest.java @@ -10,7 +10,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.Random; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/util/collections/ObjectToIntMapTest.java b/lucene/facet/src/test/org/apache/lucene/util/collections/ObjectToIntMapTest.java index 407d085..a5629a3 100644 --- a/lucene/facet/src/test/org/apache/lucene/util/collections/ObjectToIntMapTest.java +++ b/lucene/facet/src/test/org/apache/lucene/util/collections/ObjectToIntMapTest.java @@ -10,7 +10,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.collections.IntIterator; import org.apache.lucene.util.collections.ObjectToIntMap; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/util/collections/TestLRUHashMap.java b/lucene/facet/src/test/org/apache/lucene/util/collections/TestLRUHashMap.java index 48221b3..01f028b 100644 --- a/lucene/facet/src/test/org/apache/lucene/util/collections/TestLRUHashMap.java +++ b/lucene/facet/src/test/org/apache/lucene/util/collections/TestLRUHashMap.java @@ -5,7 +5,7 @@ import org.junit.Test; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.collections.LRUHashMap; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/util/encoding/EncodingSpeed.java b/lucene/facet/src/test/org/apache/lucene/util/encoding/EncodingSpeed.java index a78478a..2aeede1 100644 --- a/lucene/facet/src/test/org/apache/lucene/util/encoding/EncodingSpeed.java +++ b/lucene/facet/src/test/org/apache/lucene/util/encoding/EncodingSpeed.java @@ -16,7 +16,7 @@ import org.apache.lucene.util.encoding.SortingIntEncoder; import org.apache.lucene.util.encoding.UniqueValuesIntEncoder; import org.apache.lucene.util.encoding.VInt8IntEncoder; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/facet/src/test/org/apache/lucene/util/encoding/EncodingTest.java b/lucene/facet/src/test/org/apache/lucene/util/encoding/EncodingTest.java index 9d02a80..73e44fa 100644 --- a/lucene/facet/src/test/org/apache/lucene/util/encoding/EncodingTest.java +++ b/lucene/facet/src/test/org/apache/lucene/util/encoding/EncodingTest.java @@ -9,18 +9,8 @@ import java.util.TreeSet; import org.junit.Test; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.encoding.DGapIntEncoder; -import org.apache.lucene.util.encoding.EightFlagsIntEncoder; -import org.apache.lucene.util.encoding.FourFlagsIntEncoder; -import org.apache.lucene.util.encoding.IntDecoder; -import org.apache.lucene.util.encoding.IntEncoder; -import org.apache.lucene.util.encoding.NOnesIntEncoder; -import org.apache.lucene.util.encoding.SimpleIntEncoder; -import org.apache.lucene.util.encoding.SortingIntEncoder; -import org.apache.lucene.util.encoding.UniqueValuesIntEncoder; -import org.apache.lucene.util.encoding.VInt8IntEncoder; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -142,7 +132,7 @@ public class EncodingTest extends LuceneTestCase { } private static void decoding(ByteArrayOutputStream baos, IntDecoder decoder) - throws IOException, InstantiationException, IllegalAccessException { + throws IOException { ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); decoder.reInit(bais); diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractFirstPassGroupingCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractFirstPassGroupingCollector.java index a5d8164..f4fc51b 100644 --- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractFirstPassGroupingCollector.java +++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractFirstPassGroupingCollector.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.grouping; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractSecondPassGroupingCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractSecondPassGroupingCollector.java index 6dbfcd4..fb6f9bd 100644 --- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractSecondPassGroupingCollector.java +++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractSecondPassGroupingCollector.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.grouping; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -124,19 +124,23 @@ public abstract class AbstractSecondPassGroupingCollector exte final GroupDocs[] groupDocsResult = (GroupDocs[]) new GroupDocs[groups.size()]; int groupIDX = 0; + float maxScore = Float.MIN_VALUE; for(SearchGroup group : groups) { final SearchGroupDocs groupDocs = groupMap.get(group.groupValue); final TopDocs topDocs = groupDocs.collector.topDocs(withinGroupOffset, maxDocsPerGroup); - groupDocsResult[groupIDX++] = new GroupDocs(topDocs.getMaxScore(), + groupDocsResult[groupIDX++] = new GroupDocs(Float.NaN, + topDocs.getMaxScore(), topDocs.totalHits, topDocs.scoreDocs, groupDocs.groupValue, group.sortValues); + maxScore = Math.max(maxScore, topDocs.getMaxScore()); } return new TopGroups(groupSort.getSort(), withinGroupSort == null ? null : withinGroupSort.getSort(), - totalHitCount, totalGroupedHitCount, groupDocsResult); + totalHitCount, totalGroupedHitCount, groupDocsResult, + maxScore); } diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java index 928a444..2c34291 100644 --- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java +++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.grouping; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -324,6 +324,8 @@ public class BlockGroupingCollector extends Collector { final FakeScorer fakeScorer = new FakeScorer(); + float maxScore = Float.MIN_VALUE; + @SuppressWarnings({"unchecked","rawtypes"}) final GroupDocs[] groups = new GroupDocs[groupQueue.size() - groupOffset]; for(int downTo=groupQueue.size()-groupOffset-1;downTo>=0;downTo--) { @@ -368,11 +370,15 @@ public class BlockGroupingCollector extends Collector { final TopDocs topDocs = collector.topDocs(withinGroupOffset, maxDocsPerGroup); - groups[downTo] = new GroupDocs(topDocs.getMaxScore(), - og.count, - topDocs.scoreDocs, - null, - groupSortValues); + // TODO: we could aggregate scores across children + // by Sum/Avg instead of passing NaN: + groups[downTo] = new GroupDocs(Float.NaN, + topDocs.getMaxScore(), + og.count, + topDocs.scoreDocs, + null, + groupSortValues); + maxScore = Math.max(maxScore, topDocs.getMaxScore()); } /* @@ -385,7 +391,7 @@ public class BlockGroupingCollector extends Collector { return new TopGroups(new TopGroups(groupSort.getSort(), withinGroupSort == null ? null : withinGroupSort.getSort(), - totalHitCount, totalGroupedHitCount, groups), + totalHitCount, totalGroupedHitCount, groups, maxScore), totalGroupCount); } diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/CollectedSearchGroup.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/CollectedSearchGroup.java index f4cca99..c793f27 100644 --- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/CollectedSearchGroup.java +++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/CollectedSearchGroup.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupDocs.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupDocs.java index 0b24b5d..2ee23ef 100644 --- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupDocs.java +++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupDocs.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.grouping; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -30,6 +30,10 @@ public class GroupDocs { /** Max score in this group */ public final float maxScore; + /** Overall aggregated score of this group (currently only + * set by join queries). */ + public final float score; + /** Hits; this may be {@link * org.apache.lucene.search.FieldDoc} instances if the * withinGroupSort sorted by fields. */ @@ -42,11 +46,13 @@ public class GroupDocs { * AbstractFirstPassGroupingCollector}. */ public final Object[] groupSortValues; - public GroupDocs(float maxScore, + public GroupDocs(float score, + float maxScore, int totalHits, ScoreDoc[] scoreDocs, GROUP_VALUE_TYPE groupValue, Object[] groupSortValues) { + this.score = score; this.maxScore = maxScore; this.totalHits = totalHits; this.scoreDocs = scoreDocs; diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupingSearch.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupingSearch.java index 222bbf4..ba7e877 100644 --- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupingSearch.java +++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupingSearch.java @@ -246,7 +246,7 @@ public class GroupingSearch { Collection topSearchGroups = firstPassCollector.getTopGroups(groupOffset, fillSortFields); if (topSearchGroups == null) { - return new TopGroups(new SortField[0], new SortField[0], 0, 0, new GroupDocs[0]); + return new TopGroups(new SortField[0], new SortField[0], 0, 0, new GroupDocs[0], Float.NaN); } int topNInsideGroup = groupDocsOffset + groupDocsLimit; diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/SearchGroup.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/SearchGroup.java index a67619b..eb4d372 100644 --- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/SearchGroup.java +++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/SearchGroup.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.grouping; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/TopGroups.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/TopGroups.java index 5739d76..8440cee 100644 --- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/TopGroups.java +++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/TopGroups.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.grouping; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -46,13 +46,18 @@ public class TopGroups { /** How docs are sorted within each group */ public final SortField[] withinGroupSort; - public TopGroups(SortField[] groupSort, SortField[] withinGroupSort, int totalHitCount, int totalGroupedHitCount, GroupDocs[] groups) { + /** Highest score across all hits, or + * Float.NaN if scores were not computed. */ + public final float maxScore; + + public TopGroups(SortField[] groupSort, SortField[] withinGroupSort, int totalHitCount, int totalGroupedHitCount, GroupDocs[] groups, float maxScore) { this.groupSort = groupSort; this.withinGroupSort = withinGroupSort; this.totalHitCount = totalHitCount; this.totalGroupedHitCount = totalGroupedHitCount; this.groups = groups; this.totalGroupCount = null; + this.maxScore = maxScore; } public TopGroups(TopGroups oldTopGroups, Integer totalGroupCount) { @@ -61,9 +66,20 @@ public class TopGroups { this.totalHitCount = oldTopGroups.totalHitCount; this.totalGroupedHitCount = oldTopGroups.totalGroupedHitCount; this.groups = oldTopGroups.groups; + this.maxScore = oldTopGroups.maxScore; this.totalGroupCount = totalGroupCount; } + /** How the GroupDocs score (if any) should be merged. */ + public enum ScoreMergeMode { + /** Set score to Float.NaN */ + None, + /* Sum score across all shards for this group. */ + Total, + /* Avg score across all shards for this group. */ + Avg, + }; + /** Merges an array of TopGroups, for example obtained * from the second-pass collector across multiple * shards. Each TopGroups must have been sorted by the @@ -81,7 +97,7 @@ public class TopGroups { * NOTE: the topDocs in each GroupDocs is actually * an instance of TopDocsAndShards */ - public static TopGroups merge(TopGroups[] shardGroups, Sort groupSort, Sort docSort, int docOffset, int docTopN) + public static TopGroups merge(TopGroups[] shardGroups, Sort groupSort, Sort docSort, int docOffset, int docTopN, ScoreMergeMode scoreMergeMode) throws IOException { //System.out.println("TopGroups.merge"); @@ -115,12 +131,14 @@ public class TopGroups { final GroupDocs[] mergedGroupDocs = new GroupDocs[numGroups]; final TopDocs[] shardTopDocs = new TopDocs[shardGroups.length]; + float totalMaxScore = Float.MIN_VALUE; for(int groupIDX=0;groupIDX> DVAllGroupHeadsCollector create(String groupField, Sort sortWithinGroup, DocValues.Type type, boolean diskResident) throws IOException { + public static > DVAllGroupHeadsCollector create(String groupField, Sort sortWithinGroup, DocValues.Type type, boolean diskResident) { switch (type) { case VAR_INTS: case FIXED_INTS_8: @@ -162,7 +161,7 @@ public abstract class DVAllGroupHeadsCollector, GroupHead> groups; - GeneralAllGroupHeadsCollector(String groupField, DocValues.Type valueType, Sort sortWithinGroup, boolean diskResident) throws IOException { + GeneralAllGroupHeadsCollector(String groupField, DocValues.Type valueType, Sort sortWithinGroup, boolean diskResident) { super(groupField, valueType, sortWithinGroup.getSort().length, diskResident); this.sortWithinGroup = sortWithinGroup; groups = new HashMap, GroupHead>(); @@ -218,7 +217,7 @@ public abstract class DVAllGroupHeadsCollectorAbstractAllGroupHeadsCollector instance based on the supplied arguments - * @throws IOException If I/O related errors occur */ - public static AbstractAllGroupHeadsCollector create(String groupField, Sort sortWithinGroup) throws IOException { + public static AbstractAllGroupHeadsCollector create(String groupField, Sort sortWithinGroup) { return create(groupField, sortWithinGroup, DEFAULT_INITIAL_SIZE); } @@ -73,9 +72,8 @@ public abstract class TermAllGroupHeadsCollectorAbstractAllGroupHeadsCollector instance based on the supplied arguments - * @throws IOException If I/O related errors occur */ - public static AbstractAllGroupHeadsCollector create(String groupField, Sort sortWithinGroup, int initialSize) throws IOException { + public static AbstractAllGroupHeadsCollector create(String groupField, Sort sortWithinGroup, int initialSize) { boolean sortAllScore = true; boolean sortAllFieldValue = true; @@ -113,7 +111,7 @@ public abstract class TermAllGroupHeadsCollector(); @@ -409,7 +407,7 @@ public abstract class TermAllGroupHeadsCollector> groupHeads = new HashMap>(); for (GroupDoc groupDoc : groupDocs) { if (!groupDoc.content.startsWith(searchTerm)) { @@ -519,7 +518,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase { } @SuppressWarnings({"unchecked","rawtypes"}) - private AbstractAllGroupHeadsCollector createRandomCollector(String groupField, Sort sortWithinGroup, boolean canUseIDV, Type valueType) throws IOException { + private AbstractAllGroupHeadsCollector createRandomCollector(String groupField, Sort sortWithinGroup, boolean canUseIDV, Type valueType) { AbstractAllGroupHeadsCollector collector; if (random().nextBoolean()) { ValueSource vs = new BytesRefFieldSource(groupField); @@ -539,7 +538,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase { } private void addGroupField(Document doc, String groupField, String value, boolean canUseIDV, Type valueType) { - doc.add(new Field(groupField, value, TextField.TYPE_STORED)); + doc.add(new TextField(groupField, value, Field.Store.YES)); if (canUseIDV) { Field valuesField = null; switch(valueType) { diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupsCollectorTest.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupsCollectorTest.java index 644b51a..219c60b 100644 --- a/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupsCollectorTest.java +++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupsCollectorTest.java @@ -33,7 +33,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; -import java.io.IOException; import java.util.HashMap; public class AllGroupsCollectorTest extends LuceneTestCase { @@ -50,26 +49,26 @@ public class AllGroupsCollectorTest extends LuceneTestCase { dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); - boolean canUseIDV = !"Lucene3x".equals(w.w.getConfig().getCodec().getName()); + boolean canUseIDV = true; // 0 Document doc = new Document(); addGroupField(doc, groupField, "author1", canUseIDV); - doc.add(new Field("content", "random text", TextField.TYPE_STORED)); + doc.add(new TextField("content", "random text", Field.Store.YES)); doc.add(new Field("id", "1", customType)); w.addDocument(doc); // 1 doc = new Document(); addGroupField(doc, groupField, "author1", canUseIDV); - doc.add(new Field("content", "some more random text blob", TextField.TYPE_STORED)); + doc.add(new TextField("content", "some more random text blob", Field.Store.YES)); doc.add(new Field("id", "2", customType)); w.addDocument(doc); // 2 doc = new Document(); addGroupField(doc, groupField, "author1", canUseIDV); - doc.add(new Field("content", "some more random textual data", TextField.TYPE_STORED)); + doc.add(new TextField("content", "some more random textual data", Field.Store.YES)); doc.add(new Field("id", "3", customType)); w.addDocument(doc); w.commit(); // To ensure a second segment @@ -77,27 +76,27 @@ public class AllGroupsCollectorTest extends LuceneTestCase { // 3 doc = new Document(); addGroupField(doc, groupField, "author2", canUseIDV); - doc.add(new Field("content", "some random text", TextField.TYPE_STORED)); + doc.add(new TextField("content", "some random text", Field.Store.YES)); doc.add(new Field("id", "4", customType)); w.addDocument(doc); // 4 doc = new Document(); addGroupField(doc, groupField, "author3", canUseIDV); - doc.add(new Field("content", "some more random text", TextField.TYPE_STORED)); + doc.add(new TextField("content", "some more random text", Field.Store.YES)); doc.add(new Field("id", "5", customType)); w.addDocument(doc); // 5 doc = new Document(); addGroupField(doc, groupField, "author3", canUseIDV); - doc.add(new Field("content", "random blob", TextField.TYPE_STORED)); + doc.add(new TextField("content", "random blob", Field.Store.YES)); doc.add(new Field("id", "6", customType)); w.addDocument(doc); // 6 -- no author field doc = new Document(); - doc.add(new Field("content", "random word stuck in alot of other text", TextField.TYPE_STORED)); + doc.add(new TextField("content", "random word stuck in alot of other text", Field.Store.YES)); doc.add(new Field("id", "6", customType)); w.addDocument(doc); @@ -121,13 +120,13 @@ public class AllGroupsCollectorTest extends LuceneTestCase { } private void addGroupField(Document doc, String groupField, String value, boolean canUseIDV) { - doc.add(new Field(groupField, value, TextField.TYPE_STORED)); + doc.add(new TextField(groupField, value, Field.Store.YES)); if (canUseIDV) { doc.add(new SortedBytesDocValuesField(groupField, new BytesRef(value))); } } - private AbstractAllGroupsCollector createRandomCollector(String groupField, boolean canUseIDV) throws IOException { + private AbstractAllGroupsCollector createRandomCollector(String groupField, boolean canUseIDV) { AbstractAllGroupsCollector selected; if (random().nextBoolean() && canUseIDV) { boolean diskResident = random().nextBoolean(); diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/DistinctValuesCollectorTest.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/DistinctValuesCollectorTest.java index e121ac1..6f2ec60 100644 --- a/lucene/grouping/src/test/org/apache/lucene/search/grouping/DistinctValuesCollectorTest.java +++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/DistinctValuesCollectorTest.java @@ -61,61 +61,61 @@ public class DistinctValuesCollectorTest extends AbstractGroupingTestCase { dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy())); - boolean canUseDV = !"Lucene3x".equals(w.w.getConfig().getCodec().getName()); + boolean canUseDV = true; DocValues.Type dvType = canUseDV ? dvTypes[random.nextInt(dvTypes.length)] : null; Document doc = new Document(); addField(doc, groupField, "1", dvType); addField(doc, countField, "1", dvType); - doc.add(new Field("content", "random text", TextField.TYPE_UNSTORED)); - doc.add(new Field("id", "1", StringField.TYPE_UNSTORED)); + doc.add(new TextField("content", "random text", Field.Store.NO)); + doc.add(new StringField("id", "1", Field.Store.NO)); w.addDocument(doc); // 1 doc = new Document(); addField(doc, groupField, "1", dvType); addField(doc, countField, "1", dvType); - doc.add(new Field("content", "some more random text blob", TextField.TYPE_UNSTORED)); - doc.add(new Field("id", "2", StringField.TYPE_UNSTORED)); + doc.add(new TextField("content", "some more random text blob", Field.Store.NO)); + doc.add(new StringField("id", "2", Field.Store.NO)); w.addDocument(doc); // 2 doc = new Document(); addField(doc, groupField, "1", dvType); addField(doc, countField, "2", dvType); - doc.add(new Field("content", "some more random textual data", TextField.TYPE_UNSTORED)); - doc.add(new Field("id", "3", StringField.TYPE_UNSTORED)); + doc.add(new TextField("content", "some more random textual data", Field.Store.NO)); + doc.add(new StringField("id", "3", Field.Store.NO)); w.addDocument(doc); w.commit(); // To ensure a second segment // 3 doc = new Document(); addField(doc, groupField, "2", dvType); - doc.add(new Field("content", "some random text", TextField.TYPE_UNSTORED)); - doc.add(new Field("id", "4", StringField.TYPE_UNSTORED)); + doc.add(new TextField("content", "some random text", Field.Store.NO)); + doc.add(new StringField("id", "4", Field.Store.NO)); w.addDocument(doc); // 4 doc = new Document(); addField(doc, groupField, "3", dvType); addField(doc, countField, "1", dvType); - doc.add(new Field("content", "some more random text", TextField.TYPE_UNSTORED)); - doc.add(new Field("id", "5", StringField.TYPE_UNSTORED)); + doc.add(new TextField("content", "some more random text", Field.Store.NO)); + doc.add(new StringField("id", "5", Field.Store.NO)); w.addDocument(doc); // 5 doc = new Document(); addField(doc, groupField, "3", dvType); addField(doc, countField, "1", dvType); - doc.add(new Field("content", "random blob", TextField.TYPE_UNSTORED)); - doc.add(new Field("id", "6", StringField.TYPE_UNSTORED)); + doc.add(new TextField("content", "random blob", Field.Store.NO)); + doc.add(new StringField("id", "6", Field.Store.NO)); w.addDocument(doc); // 6 -- no author field doc = new Document(); - doc.add(new Field("content", "random word stuck in alot of other text", TextField.TYPE_STORED)); + doc.add(new TextField("content", "random word stuck in alot of other text", Field.Store.YES)); addField(doc, countField, "1", dvType); - doc.add(new Field("id", "6", StringField.TYPE_UNSTORED)); + doc.add(new StringField("id", "6", Field.Store.NO)); w.addDocument(doc); IndexSearcher indexSearcher = newSearcher(w.getReader()); @@ -316,7 +316,7 @@ public class DistinctValuesCollectorTest extends AbstractGroupingTestCase { } private void addField(Document doc, String field, String value, DocValues.Type type) { - doc.add(new Field(field, value, StringField.TYPE_UNSTORED)); + doc.add(new StringField(field, value, Field.Store.NO)); if (type == null) { return; } @@ -417,7 +417,7 @@ public class DistinctValuesCollectorTest extends AbstractGroupingTestCase { new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()) ); - boolean canUseDV = !"Lucene3x".equals(w.w.getConfig().getCodec().getName()); + boolean canUseDV = true; DocValues.Type dvType = canUseDV ? dvTypes[random.nextInt(dvTypes.length)] : null; int numDocs = 86 + random.nextInt(1087) * RANDOM_MULTIPLIER; @@ -450,14 +450,14 @@ public class DistinctValuesCollectorTest extends AbstractGroupingTestCase { countsVals.add(countValue); Document doc = new Document(); - doc.add(new Field("id", String.format("%09d", i), StringField.TYPE_UNSTORED)); + doc.add(new StringField("id", String.format("%09d", i), Field.Store.NO)); if (groupValue != null) { addField(doc, groupField, groupValue, dvType); } if (countValue != null) { addField(doc, countField, countValue, dvType); } - doc.add(new Field("content", content, TextField.TYPE_UNSTORED)); + doc.add(new TextField("content", content, Field.Store.NO)); w.addDocument(doc); } diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupFacetCollectorTest.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupFacetCollectorTest.java index 790ec05..97e0d66 100644 --- a/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupFacetCollectorTest.java +++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupFacetCollectorTest.java @@ -48,7 +48,7 @@ public class GroupFacetCollectorTest extends AbstractGroupingTestCase { dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); - boolean canUseDV = !"Lucene3x".equals(w.w.getConfig().getCodec().getName()); + boolean canUseDV = true; boolean useDv = canUseDV && random().nextBoolean(); // 0 @@ -218,7 +218,7 @@ public class GroupFacetCollectorTest extends AbstractGroupingTestCase { } private void addField(Document doc, String field, String value, boolean canUseIDV) { - doc.add(new Field(field, value, StringField.TYPE_UNSTORED)); + doc.add(new StringField(field, value, Field.Store.NO)); if (canUseIDV) { doc.add(new SortedBytesDocValuesField(field, new BytesRef(value))); } @@ -360,14 +360,14 @@ public class GroupFacetCollectorTest extends AbstractGroupingTestCase { new MockAnalyzer(random) ) ); - boolean canUseDV = !"Lucene3x".equals(writer.w.getConfig().getCodec().getName()); + boolean canUseDV = true; boolean useDv = canUseDV && random.nextBoolean(); Document doc = new Document(); Document docNoGroup = new Document(); Document docNoFacet = new Document(); Document docNoGroupNoFacet = new Document(); - Field group = newField("group", "", StringField.TYPE_UNSTORED); + Field group = newStringField("group", "", Field.Store.NO); Field groupDc = new SortedBytesDocValuesField("group", new BytesRef()); if (useDv) { doc.add(groupDc); @@ -378,7 +378,7 @@ public class GroupFacetCollectorTest extends AbstractGroupingTestCase { Field[] facetFields; if (useDv) { facetFields = new Field[2]; - facetFields[0] = newField("facet", "", StringField.TYPE_UNSTORED); + facetFields[0] = newStringField("facet", "", Field.Store.NO); doc.add(facetFields[0]); docNoGroup.add(facetFields[0]); facetFields[1] = new SortedBytesDocValuesField("facet", new BytesRef()); @@ -387,12 +387,12 @@ public class GroupFacetCollectorTest extends AbstractGroupingTestCase { } else { facetFields = multipleFacetValuesPerDocument ? new Field[2 + random.nextInt(6)] : new Field[1]; for (int i = 0; i < facetFields.length; i++) { - facetFields[i] = newField("facet", "", StringField.TYPE_UNSTORED); + facetFields[i] = newStringField("facet", "", Field.Store.NO); doc.add(facetFields[i]); docNoGroup.add(facetFields[i]); } } - Field content = newField("content", "", StringField.TYPE_UNSTORED); + Field content = newStringField("content", "", Field.Store.NO); doc.add(content); docNoGroup.add(content); docNoFacet.add(content); diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupingSearchTest.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupingSearchTest.java index b98aff6..2267fe7 100644 --- a/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupingSearchTest.java +++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupingSearchTest.java @@ -18,19 +18,28 @@ package org.apache.lucene.search.grouping; */ import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.*; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.SortedBytesDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource; -import org.apache.lucene.search.*; +import org.apache.lucene.search.CachingWrapperFilter; +import org.apache.lucene.search.Filter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.QueryWrapperFilter; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.mutable.MutableValueStr; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -51,28 +60,28 @@ public class GroupingSearchTest extends LuceneTestCase { dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); - boolean canUseIDV = !"Lucene3x".equals(w.w.getConfig().getCodec().getName()); + boolean canUseIDV = true; List documents = new ArrayList(); // 0 Document doc = new Document(); addGroupField(doc, groupField, "author1", canUseIDV); - doc.add(new Field("content", "random text", TextField.TYPE_STORED)); + doc.add(new TextField("content", "random text", Field.Store.YES)); doc.add(new Field("id", "1", customType)); documents.add(doc); // 1 doc = new Document(); addGroupField(doc, groupField, "author1", canUseIDV); - doc.add(new Field("content", "some more random text", TextField.TYPE_STORED)); + doc.add(new TextField("content", "some more random text", Field.Store.YES)); doc.add(new Field("id", "2", customType)); documents.add(doc); // 2 doc = new Document(); addGroupField(doc, groupField, "author1", canUseIDV); - doc.add(new Field("content", "some more random textual data", TextField.TYPE_STORED)); + doc.add(new TextField("content", "some more random textual data", Field.Store.YES)); doc.add(new Field("id", "3", customType)); - doc.add(new Field("groupend", "x", StringField.TYPE_UNSTORED)); + doc.add(new StringField("groupend", "x", Field.Store.NO)); documents.add(doc); w.addDocuments(documents); documents.clear(); @@ -80,33 +89,33 @@ public class GroupingSearchTest extends LuceneTestCase { // 3 doc = new Document(); addGroupField(doc, groupField, "author2", canUseIDV); - doc.add(new Field("content", "some random text", TextField.TYPE_STORED)); + doc.add(new TextField("content", "some random text", Field.Store.YES)); doc.add(new Field("id", "4", customType)); - doc.add(new Field("groupend", "x", StringField.TYPE_UNSTORED)); + doc.add(new StringField("groupend", "x", Field.Store.NO)); w.addDocument(doc); // 4 doc = new Document(); addGroupField(doc, groupField, "author3", canUseIDV); - doc.add(new Field("content", "some more random text", TextField.TYPE_STORED)); + doc.add(new TextField("content", "some more random text", Field.Store.YES)); doc.add(new Field("id", "5", customType)); documents.add(doc); // 5 doc = new Document(); addGroupField(doc, groupField, "author3", canUseIDV); - doc.add(new Field("content", "random", TextField.TYPE_STORED)); + doc.add(new TextField("content", "random", Field.Store.YES)); doc.add(new Field("id", "6", customType)); - doc.add(new Field("groupend", "x", StringField.TYPE_UNSTORED)); + doc.add(new StringField("groupend", "x", Field.Store.NO)); documents.add(doc); w.addDocuments(documents); documents.clear(); // 6 -- no author field doc = new Document(); - doc.add(new Field("content", "random word stuck in alot of other text", TextField.TYPE_STORED)); + doc.add(new TextField("content", "random word stuck in alot of other text", Field.Store.YES)); doc.add(new Field("id", "6", customType)); - doc.add(new Field("groupend", "x", StringField.TYPE_UNSTORED)); + doc.add(new StringField("groupend", "x", Field.Store.NO)); w.addDocument(doc); @@ -166,7 +175,7 @@ public class GroupingSearchTest extends LuceneTestCase { } private void addGroupField(Document doc, String groupField, String value, boolean canUseIDV) { - doc.add(new Field(groupField, value, TextField.TYPE_STORED)); + doc.add(new TextField(groupField, value, Field.Store.YES)); if (canUseIDV) { doc.add(new SortedBytesDocValuesField(groupField, new BytesRef(value))); } @@ -195,7 +204,7 @@ public class GroupingSearchTest extends LuceneTestCase { } } - private GroupingSearch createRandomGroupingSearch(String groupField, Sort groupSort, int docsInGroup, boolean canUseIDV) throws IOException { + private GroupingSearch createRandomGroupingSearch(String groupField, Sort groupSort, int docsInGroup, boolean canUseIDV) { GroupingSearch groupingSearch; if (random().nextBoolean()) { ValueSource vs = new BytesRefFieldSource(groupField); diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java index 762b241..9a41ff3 100644 --- a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java +++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,6 +23,7 @@ import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.CompositeReaderContext; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.IndexReaderContext; @@ -44,7 +45,6 @@ import org.apache.lucene.search.grouping.term.TermSecondPassGroupingCollector; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.ReaderUtil; import org.apache.lucene.util._TestUtil; import org.apache.lucene.util.mutable.MutableValue; import org.apache.lucene.util.mutable.MutableValueStr; @@ -73,52 +73,52 @@ public class TestGrouping extends LuceneTestCase { dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); - boolean canUseIDV = !"Lucene3x".equals(w.w.getConfig().getCodec().getName()); + boolean canUseIDV = true; // 0 Document doc = new Document(); addGroupField(doc, groupField, "author1", canUseIDV); - doc.add(new Field("content", "random text", TextField.TYPE_STORED)); + doc.add(new TextField("content", "random text", Field.Store.YES)); doc.add(new Field("id", "1", customType)); w.addDocument(doc); // 1 doc = new Document(); addGroupField(doc, groupField, "author1", canUseIDV); - doc.add(new Field("content", "some more random text", TextField.TYPE_STORED)); + doc.add(new TextField("content", "some more random text", Field.Store.YES)); doc.add(new Field("id", "2", customType)); w.addDocument(doc); // 2 doc = new Document(); addGroupField(doc, groupField, "author1", canUseIDV); - doc.add(new Field("content", "some more random textual data", TextField.TYPE_STORED)); + doc.add(new TextField("content", "some more random textual data", Field.Store.YES)); doc.add(new Field("id", "3", customType)); w.addDocument(doc); // 3 doc = new Document(); addGroupField(doc, groupField, "author2", canUseIDV); - doc.add(new Field("content", "some random text", TextField.TYPE_STORED)); + doc.add(new TextField("content", "some random text", Field.Store.YES)); doc.add(new Field("id", "4", customType)); w.addDocument(doc); // 4 doc = new Document(); addGroupField(doc, groupField, "author3", canUseIDV); - doc.add(new Field("content", "some more random text", TextField.TYPE_STORED)); + doc.add(new TextField("content", "some more random text", Field.Store.YES)); doc.add(new Field("id", "5", customType)); w.addDocument(doc); // 5 doc = new Document(); addGroupField(doc, groupField, "author3", canUseIDV); - doc.add(new Field("content", "random", TextField.TYPE_STORED)); + doc.add(new TextField("content", "random", Field.Store.YES)); doc.add(new Field("id", "6", customType)); w.addDocument(doc); // 6 -- no author field doc = new Document(); - doc.add(new Field("content", "random word stuck in alot of other text", TextField.TYPE_STORED)); + doc.add(new TextField("content", "random word stuck in alot of other text", Field.Store.YES)); doc.add(new Field("id", "6", customType)); w.addDocument(doc); @@ -129,10 +129,11 @@ public class TestGrouping extends LuceneTestCase { final AbstractFirstPassGroupingCollector c1 = createRandomFirstPassCollector(groupField, groupSort, 10, canUseIDV); indexSearcher.search(new TermQuery(new Term("content", "random")), c1); - final AbstractSecondPassGroupingCollector c2 = createSecondPassCollector(c1, groupField, groupSort, null, 0, 5, true, false, true); + final AbstractSecondPassGroupingCollector c2 = createSecondPassCollector(c1, groupField, groupSort, null, 0, 5, true, true, true); indexSearcher.search(new TermQuery(new Term("content", "random")), c2); final TopGroups groups = c2.getTopGroups(0); + assertFalse(Float.isNaN(groups.maxScore)); assertEquals(7, groups.totalHitCount); assertEquals(7, groups.totalGroupedHitCount); @@ -173,7 +174,7 @@ public class TestGrouping extends LuceneTestCase { } private void addGroupField(Document doc, String groupField, String value, boolean canUseIDV) { - doc.add(new Field(groupField, value, TextField.TYPE_STORED)); + doc.add(new TextField(groupField, value, Field.Store.YES)); if (canUseIDV) { doc.add(new SortedBytesDocValuesField(groupField, new BytesRef(value))); } @@ -341,9 +342,9 @@ public class TestGrouping extends LuceneTestCase { List> groups = new ArrayList>(mvalTopGroups.groups.length); for (GroupDocs mvalGd : mvalTopGroups.groups) { BytesRef groupValue = mvalGd.groupValue.exists() ? ((MutableValueStr) mvalGd.groupValue).value : null; - groups.add(new GroupDocs(mvalGd.maxScore, mvalGd.totalHits, mvalGd.scoreDocs, groupValue, mvalGd.groupSortValues)); + groups.add(new GroupDocs(Float.NaN, mvalGd.maxScore, mvalGd.totalHits, mvalGd.scoreDocs, groupValue, mvalGd.groupSortValues)); } - return new TopGroups(mvalTopGroups.groupSort, mvalTopGroups.withinGroupSort, mvalTopGroups.totalHitCount, mvalTopGroups.totalGroupedHitCount, groups.toArray(new GroupDocs[groups.size()])); + return new TopGroups(mvalTopGroups.groupSort, mvalTopGroups.withinGroupSort, mvalTopGroups.totalHitCount, mvalTopGroups.totalGroupedHitCount, groups.toArray(new GroupDocs[groups.size()]), Float.NaN); } else if (DVSecondPassGroupingCollector.class.isAssignableFrom(c.getClass())) { return ((DVSecondPassGroupingCollector) c).getTopGroups(withinGroupOffset); } @@ -541,20 +542,21 @@ public class TestGrouping extends LuceneTestCase { hits = new ScoreDoc[0]; } - result[idx-groupOffset] = new GroupDocs(0.0f, - docs.size(), - hits, - group, - fillFields ? sortedGroupFields.get(idx) : null); + result[idx-groupOffset] = new GroupDocs(Float.NaN, + 0.0f, + docs.size(), + hits, + group, + fillFields ? sortedGroupFields.get(idx) : null); } if (doAllGroups) { return new TopGroups( - new TopGroups(groupSort.getSort(), docSort.getSort(), totalHitCount, totalGroupedHitCount, result), - knownGroups.size() + new TopGroups(groupSort.getSort(), docSort.getSort(), totalHitCount, totalGroupedHitCount, result, Float.NaN), + knownGroups.size() ); } else { - return new TopGroups(groupSort.getSort(), docSort.getSort(), totalHitCount, totalGroupedHitCount, result); + return new TopGroups(groupSort.getSort(), docSort.getSort(), totalHitCount, totalGroupedHitCount, result, Float.NaN); } } @@ -580,7 +582,7 @@ public class TestGrouping extends LuceneTestCase { final List> updateDocs = new ArrayList>(); - FieldType groupEndType = new FieldType(StringField.TYPE_UNSTORED); + FieldType groupEndType = new FieldType(StringField.TYPE_NOT_STORED); groupEndType.setIndexOptions(IndexOptions.DOCS_ONLY); groupEndType.setOmitNorms(true); @@ -592,12 +594,12 @@ public class TestGrouping extends LuceneTestCase { Document doc = new Document(); docs.add(doc); if (groupValue.group != null) { - doc.add(newField("group", groupValue.group.utf8ToString(), StringField.TYPE_UNSTORED)); + doc.add(newStringField("group", groupValue.group.utf8ToString(), Field.Store.NO)); } - doc.add(newField("sort1", groupValue.sort1.utf8ToString(), StringField.TYPE_UNSTORED)); - doc.add(newField("sort2", groupValue.sort2.utf8ToString(), StringField.TYPE_UNSTORED)); - doc.add(new IntField("id", groupValue.id)); - doc.add(newField("content", groupValue.content, TextField.TYPE_UNSTORED)); + doc.add(newStringField("sort1", groupValue.sort1.utf8ToString(), Field.Store.NO)); + doc.add(newStringField("sort2", groupValue.sort2.utf8ToString(), Field.Store.NO)); + doc.add(new IntField("id", groupValue.id, Field.Store.NO)); + doc.add(newTextField("content", groupValue.content, Field.Store.NO)); //System.out.println("TEST: doc content=" + groupValue.content + " group=" + (groupValue.group == null ? "null" : groupValue.group.utf8ToString()) + " sort1=" + groupValue.sort1.utf8ToString() + " id=" + groupValue.id); } // So we can pull filter marking last doc in block: @@ -627,25 +629,16 @@ public class TestGrouping extends LuceneTestCase { public final int[] docStarts; public ShardState(IndexSearcher s) { - List subReaders = new ArrayList(); - ReaderUtil.gatherSubReaders(subReaders, s.getIndexReader()); - subSearchers = new ShardSearcher[subReaders.size()]; final IndexReaderContext ctx = s.getTopReaderContext(); - if (ctx instanceof AtomicReaderContext) { - assert subSearchers.length == 1; - subSearchers[0] = new ShardSearcher((AtomicReaderContext) ctx, ctx); - } else { - final CompositeReaderContext compCTX = (CompositeReaderContext) ctx; - for(int searcherIDX=0;searcherIDX leaves = ctx.leaves(); + subSearchers = new ShardSearcher[leaves.size()]; + for(int searcherIDX=0;searcherIDX idvBasedImplsUsedSharded = new ValueHolder(false); final TopGroups topGroupsShards = searchShards(s, shards.subSearchers, query, groupSort, docSort, - groupOffset, topNGroups, docOffset, docsPerGroup, getScores, getMaxScores, canUseIDV, preFlex, idvBasedImplsUsedSharded); + groupOffset, topNGroups, docOffset, docsPerGroup, getScores, getMaxScores, canUseIDV, false, idvBasedImplsUsedSharded); final AbstractSecondPassGroupingCollector c2; if (topGroups != null) { @@ -1238,7 +1230,7 @@ public class TestGrouping extends LuceneTestCase { } } - TopGroups mergedGroups = TopGroups.merge(shardTopGroups, groupSort, docSort, docOffset, topNDocs); + TopGroups mergedGroups = TopGroups.merge(shardTopGroups, groupSort, docSort, docOffset, topNDocs, TopGroups.ScoreMergeMode.None); if (VERBOSE) { System.out.println(" " + mergedGroups.groups.length + " merged groups:"); for(GroupDocs group : mergedGroups.groups) { @@ -1314,24 +1306,20 @@ public class TestGrouping extends LuceneTestCase { } private static class ShardSearcher extends IndexSearcher { - private final AtomicReaderContext[] ctx; + private final List ctx; public ShardSearcher(AtomicReaderContext ctx, IndexReaderContext parent) { super(parent); - this.ctx = new AtomicReaderContext[] {ctx}; + this.ctx = Collections.singletonList(ctx); } public void search(Weight weight, Collector collector) throws IOException { search(ctx, weight, collector); } - public TopDocs search(Weight weight, int topN) throws IOException { - return search(ctx, weight, null, topN); - } - @Override public String toString() { - return "ShardSearcher(" + ctx[0].reader() + ")"; + return "ShardSearcher(" + ctx.get(0).reader() + ")"; } } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Formatter.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Formatter.java index a41d660..1aceed9 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Formatter.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Formatter.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Fragmenter.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Fragmenter.java index f8f4b27..27f0760 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Fragmenter.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Fragmenter.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/GradientFormatter.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/GradientFormatter.java index a073fc2..58f022a 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/GradientFormatter.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/GradientFormatter.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java index 3957c46..07cebe6 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/InvalidTokenOffsetsException.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/InvalidTokenOffsetsException.java index 3a8a37f..c159ccc 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/InvalidTokenOffsetsException.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/InvalidTokenOffsetsException.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/NullFragmenter.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/NullFragmenter.java index 2dd042c..5b35115 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/NullFragmenter.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/NullFragmenter.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/OffsetLimitTokenFilter.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/OffsetLimitTokenFilter.java index 2102c28..f660d19 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/OffsetLimitTokenFilter.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/OffsetLimitTokenFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/PositionSpan.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/PositionSpan.java index 8124ae8..355ec6c 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/PositionSpan.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/PositionSpan.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java index 59fff98..1c51fee 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermExtractor.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermExtractor.java index 0054ae4..a60422c 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermExtractor.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermExtractor.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java index 167bf3d..c17c0b2 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Scorer.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Scorer.java index 92abb24..fa40688 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Scorer.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Scorer.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleFragmenter.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleFragmenter.java index 7be917e..d30aa76 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleFragmenter.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleFragmenter.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleHTMLFormatter.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleHTMLFormatter.java index 313d861..2afebe8 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleHTMLFormatter.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleHTMLFormatter.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java index c468867..56b4e5f 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java @@ -1,7 +1,7 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SpanGradientFormatter.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SpanGradientFormatter.java index 9e53cf7..49a92a3 100755 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SpanGradientFormatter.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SpanGradientFormatter.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TextFragment.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TextFragment.java index 11d8e53..2b31a42 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TextFragment.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TextFragment.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java index e0f88f2..484671c 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java index 126aa1c..1824741 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java @@ -3,7 +3,7 @@ */ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -191,7 +191,7 @@ public class TokenSources { } @Override - public boolean incrementToken() throws IOException { + public boolean incrementToken() { if (currentToken >= tokens.length) { return false; } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java index 02454b3..bbda4a9 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -105,7 +105,7 @@ public final class TokenStreamFromTermPositionVector extends TokenStream { }; @Override - public boolean incrementToken() throws IOException { + public boolean incrementToken() { if (this.tokensAtCurrentPosition.hasNext()) { final Token next = this.tokensAtCurrentPosition.next(); clearAttributes(); @@ -119,7 +119,7 @@ public final class TokenStreamFromTermPositionVector extends TokenStream { } @Override - public void reset() throws IOException { + public void reset() { this.tokensAtCurrentPosition = this.positionedTokens.iterator(); } } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTerm.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTerm.java index eadff70..15c4c01 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTerm.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTerm.java @@ -1,7 +1,7 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java index 52171b9..161225b 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -32,6 +32,7 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; import org.apache.lucene.index.memory.MemoryIndex; import org.apache.lucene.search.*; import org.apache.lucene.search.spans.FieldMaskingSpanQuery; @@ -43,7 +44,6 @@ import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.search.spans.Spans; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.TermContext; /** * Class used to extract {@link WeightedSpanTerm}s from a {@link Query} based on whether diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedTerm.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedTerm.java index 28185ed..9d25aa7 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedTerm.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedTerm.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragListBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragListBuilder.java new file mode 100644 index 0000000..0c77b70 --- /dev/null +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragListBuilder.java @@ -0,0 +1,96 @@ +package org.apache.lucene.search.vectorhighlight; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo; + +/** + * A abstract implementation of {@link FragListBuilder}. + */ +public abstract class BaseFragListBuilder implements FragListBuilder { + + public static final int MARGIN_DEFAULT = 6; + public static final int MIN_FRAG_CHAR_SIZE_FACTOR = 3; + + final int margin; + final int minFragCharSize; + + public BaseFragListBuilder( int margin ){ + if( margin < 0 ) + throw new IllegalArgumentException( "margin(" + margin + ") is too small. It must be 0 or higher." ); + + this.margin = margin; + this.minFragCharSize = Math.max( 1, margin * MIN_FRAG_CHAR_SIZE_FACTOR ); + } + + public BaseFragListBuilder(){ + this( MARGIN_DEFAULT ); + } + + protected FieldFragList createFieldFragList( FieldPhraseList fieldPhraseList, FieldFragList fieldFragList, int fragCharSize ){ + + if( fragCharSize < minFragCharSize ) + throw new IllegalArgumentException( "fragCharSize(" + fragCharSize + ") is too small. It must be " + minFragCharSize + " or higher." ); + + List wpil = new ArrayList(); + Iterator ite = fieldPhraseList.getPhraseList().iterator(); + WeightedPhraseInfo phraseInfo = null; + int startOffset = 0; + boolean taken = false; + while( true ){ + if( !taken ){ + if( !ite.hasNext() ) break; + phraseInfo = ite.next(); + } + taken = false; + if( phraseInfo == null ) break; + + // if the phrase violates the border of previous fragment, discard it and try next phrase + if( phraseInfo.getStartOffset() < startOffset ) continue; + + wpil.clear(); + wpil.add( phraseInfo ); + int st = phraseInfo.getStartOffset() - margin < startOffset ? + startOffset : phraseInfo.getStartOffset() - margin; + int en = st + fragCharSize; + if( phraseInfo.getEndOffset() > en ) + en = phraseInfo.getEndOffset(); + startOffset = en; + + while( true ){ + if( ite.hasNext() ){ + phraseInfo = ite.next(); + taken = true; + if( phraseInfo == null ) break; + } + else + break; + if( phraseInfo.getEndOffset() <= en ) + wpil.add( phraseInfo ); + else + break; + } + fieldFragList.add( st, en, wpil ); + } + return fieldFragList; + } +} diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java index d09eb15..110736c 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -125,14 +125,14 @@ public abstract class BaseFragmentsBuilder implements FragmentsBuilder { reader.document(docId, new StoredFieldVisitor() { @Override - public void stringField(FieldInfo fieldInfo, String value) throws IOException { + public void stringField(FieldInfo fieldInfo, String value) { FieldType ft = new FieldType(TextField.TYPE_STORED); ft.setStoreTermVectors(fieldInfo.hasVectors()); fields.add(new Field(fieldInfo.name, value, ft)); } @Override - public Status needsField(FieldInfo fieldInfo) throws IOException { + public Status needsField(FieldInfo fieldInfo) { return fieldInfo.name.equals(fieldName) ? Status.YES : Status.NO; } }); diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BoundaryScanner.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BoundaryScanner.java index 5e78996..217db2c 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BoundaryScanner.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BoundaryScanner.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BreakIteratorBoundaryScanner.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BreakIteratorBoundaryScanner.java index 518a472..4f506ec 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BreakIteratorBoundaryScanner.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BreakIteratorBoundaryScanner.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FastVectorHighlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FastVectorHighlighter.java index 9a3ffcf..5b20539 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FastVectorHighlighter.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FastVectorHighlighter.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldFragList.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldFragList.java index 92cdb02..7a0707c 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldFragList.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldFragList.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -27,7 +27,7 @@ import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseIn * FieldFragList has a list of "frag info" that is used by FragmentsBuilder class * to create fragments (snippets). */ -public class FieldFragList { +public abstract class FieldFragList { private List fragInfos = new ArrayList(); @@ -46,9 +46,7 @@ public class FieldFragList { * @param endOffset end offset of the fragment * @param phraseInfoList list of WeightedPhraseInfo objects */ - public void add( int startOffset, int endOffset, List phraseInfoList ){ - fragInfos.add( new WeightedFragInfo( startOffset, endOffset, phraseInfoList ) ); - } + public abstract void add( int startOffset, int endOffset, List phraseInfoList ); /** * return the list of WeightedFragInfos. @@ -61,20 +59,16 @@ public class FieldFragList { public static class WeightedFragInfo { - List subInfos; - float totalBoost; - int startOffset; - int endOffset; + private List subInfos; + private float totalBoost; + private int startOffset; + private int endOffset; - public WeightedFragInfo( int startOffset, int endOffset, List phraseInfoList ){ + public WeightedFragInfo( int startOffset, int endOffset, List subInfos, float totalBoost ){ this.startOffset = startOffset; this.endOffset = endOffset; - subInfos = new ArrayList(); - for( WeightedPhraseInfo phraseInfo : phraseInfoList ){ - SubInfo subInfo = new SubInfo( phraseInfo.text, phraseInfo.termsOffsets, phraseInfo.seqnum ); - subInfos.add( subInfo ); - totalBoost += phraseInfo.boost; - } + this.totalBoost = totalBoost; + this.subInfos = subInfos; } public List getSubInfos(){ @@ -104,12 +98,12 @@ public class FieldFragList { } public static class SubInfo { - final String text; // unnecessary member, just exists for debugging purpose - final List termsOffsets; // usually termsOffsets.size() == 1, + private final String text; // unnecessary member, just exists for debugging purpose + private final List termsOffsets; // usually termsOffsets.size() == 1, // but if position-gap > 1 and slop > 0 then size() could be greater than 1 - int seqnum; + private int seqnum; - SubInfo( String text, List termsOffsets, int seqnum ){ + public SubInfo( String text, List termsOffsets, int seqnum ){ this.text = text; this.termsOffsets = termsOffsets; this.seqnum = seqnum; diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldPhraseList.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldPhraseList.java index f2263c5..60288a5 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldPhraseList.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldPhraseList.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -42,13 +42,22 @@ public class FieldPhraseList { } /** + * return the list of WeightedPhraseInfo. + * + * @return phraseList. + */ + public List getPhraseList() { + return phraseList; + } + + /** * a constructor. * * @param fieldTermStack FieldTermStack object * @param fieldQuery FieldQuery object * @param phraseLimit maximum size of phraseList */ - public FieldPhraseList( FieldTermStack fieldTermStack, FieldQuery fieldQuery, int phraseLimit){ + public FieldPhraseList( FieldTermStack fieldTermStack, FieldQuery fieldQuery, int phraseLimit ){ final String field = fieldTermStack.getFieldName(); LinkedList phraseCandidate = new LinkedList(); @@ -96,29 +105,68 @@ public class FieldPhraseList { } } } - - void addIfNoOverlap( WeightedPhraseInfo wpi ){ - for( WeightedPhraseInfo existWpi : phraseList ){ - if( existWpi.isOffsetOverlap( wpi ) ) return; + + public void addIfNoOverlap( WeightedPhraseInfo wpi ){ + for( WeightedPhraseInfo existWpi : getPhraseList() ){ + if( existWpi.isOffsetOverlap( wpi ) ) { + // WeightedPhraseInfo.addIfNoOverlap() dumps the second part of, for example, hyphenated words (social-economics). + // The result is that all informations in TermInfo are lost and not available for further operations. + existWpi.getTermsInfos().addAll( wpi.getTermsInfos() ); + return; + } } - phraseList.add( wpi ); + getPhraseList().add( wpi ); } public static class WeightedPhraseInfo { - String text; // unnecessary member, just exists for debugging purpose - List termsOffsets; // usually termsOffsets.size() == 1, + private String text; // unnecessary member, just exists for debugging purpose + private List termsOffsets; // usually termsOffsets.size() == 1, // but if position-gap > 1 and slop > 0 then size() could be greater than 1 - float boost; // query boost - int seqnum; + private float boost; // query boost + private int seqnum; + private ArrayList termsInfos; + + /** + * @return the text + */ + public String getText() { + return text; + } + + /** + * @return the termsOffsets + */ + public List getTermsOffsets() { + return termsOffsets; + } + + /** + * @return the boost + */ + public float getBoost() { + return boost; + } + + /** + * @return the termInfos + */ + public List getTermsInfos() { + return termsInfos; + } + public WeightedPhraseInfo( LinkedList terms, float boost ){ this( terms, boost, 0 ); } - public WeightedPhraseInfo( LinkedList terms, float boost, int number ){ + public WeightedPhraseInfo( LinkedList terms, float boost, int seqnum ){ this.boost = boost; - this.seqnum = number; + this.seqnum = seqnum; + + // We keep TermInfos for further operations + termsInfos = new ArrayList( terms ); + termsOffsets = new ArrayList( terms.size() ); TermInfo ti = terms.get( 0 ); termsOffsets.add( new Toffs( ti.getStartOffset(), ti.getEndOffset() ) ); @@ -175,9 +223,16 @@ public class FieldPhraseList { return sb.toString(); } + /** + * @return the seqnum + */ + public int getSeqnum() { + return seqnum; + } + public static class Toffs { - int startOffset; - int endOffset; + private int startOffset; + private int endOffset; public Toffs( int startOffset, int endOffset ){ this.startOffset = startOffset; this.endOffset = endOffset; diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java index c5f8d76..d88d4f9 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -348,7 +348,7 @@ public class FieldQuery { return map; } - void add( Query query, IndexReader reader ) throws IOException { + void add( Query query, IndexReader reader ) { if( query instanceof TermQuery ){ addTerm( ((TermQuery)query).getTerm(), query.getBoost() ); } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java index f31e85f..9cf018e 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -94,6 +94,10 @@ public class FieldTermStack { final TermsEnum termsEnum = vector.iterator(null); DocsAndPositionsEnum dpEnum = null; BytesRef text; + + int numDocs = reader.numDocs() - reader.numDeletedDocs(); + float weight = 0; + while ((text = termsEnum.next()) != null) { UnicodeUtil.UTF8toUTF16(text, spare); final String term = spare.toString(); @@ -112,7 +116,9 @@ public class FieldTermStack { for(int i = 0;i < freq;i++) { int pos = dpEnum.nextPosition(); - termList.add(new TermInfo(term, dpEnum.startOffset(), dpEnum.endOffset(), pos)); + // For weight look here: http://lucene.apache.org/core/3_6_0/api/core/org/apache/lucene/search/DefaultSimilarity.html + weight = ( float ) ( Math.log( numDocs / ( double ) ( reader.docFreq( fieldName, text ) + 1 ) ) + 1.0 ); + termList.add( new TermInfo( term, dpEnum.startOffset(), dpEnum.endOffset(), pos, weight ) ); } } @@ -152,22 +158,27 @@ public class FieldTermStack { public static class TermInfo implements Comparable{ - final String text; - final int startOffset; - final int endOffset; - final int position; + private final String text; + private final int startOffset; + private final int endOffset; + private final int position; - TermInfo( String text, int startOffset, int endOffset, int position ){ + // IDF-weight of this term + private final float weight; + + public TermInfo( String text, int startOffset, int endOffset, int position, float weight ){ this.text = text; this.startOffset = startOffset; this.endOffset = endOffset; this.position = position; + this.weight = weight; } public String getText(){ return text; } public int getStartOffset(){ return startOffset; } public int getEndOffset(){ return endOffset; } public int getPosition(){ return position; } + public float getWeight(){ return weight; } @Override public String toString(){ @@ -176,7 +187,8 @@ public class FieldTermStack { return sb.toString(); } - public int compareTo( TermInfo o ) { + @Override + public int compareTo( TermInfo o ){ return ( this.position - o.position ); } } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FragListBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FragListBuilder.java index 0ed1d3e..8440cc9 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FragListBuilder.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FragListBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FragmentsBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FragmentsBuilder.java index 81d6c6b..8bb5a89 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FragmentsBuilder.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FragmentsBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilder.java index a2e6e29..112fb9b 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilder.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -65,12 +65,12 @@ public class ScoreOrderFragmentsBuilder extends BaseFragmentsBuilder { public static class ScoreComparator implements Comparator { public int compare( WeightedFragInfo o1, WeightedFragInfo o2 ) { - if( o1.totalBoost > o2.totalBoost ) return -1; - else if( o1.totalBoost < o2.totalBoost ) return 1; + if( o1.getTotalBoost() > o2.getTotalBoost() ) return -1; + else if( o1.getTotalBoost() < o2.getTotalBoost() ) return 1; // if same score then check startOffset else{ - if( o1.startOffset < o2.startOffset ) return -1; - else if( o1.startOffset > o2.startOffset ) return 1; + if( o1.getStartOffset() < o2.getStartOffset() ) return -1; + else if( o1.getStartOffset() > o2.getStartOffset() ) return 1; } return 0; } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleBoundaryScanner.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleBoundaryScanner.java index d76e8d4..ea048cb 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleBoundaryScanner.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleBoundaryScanner.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFieldFragList.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFieldFragList.java new file mode 100644 index 0000000..d9f0b47 --- /dev/null +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFieldFragList.java @@ -0,0 +1,54 @@ +package org.apache.lucene.search.vectorhighlight; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.ArrayList; +import java.util.List; + +import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo.SubInfo; +import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo; + +/** + * A simple implementation of {@link FieldFragList}. + */ +public class SimpleFieldFragList extends FieldFragList { + + /** + * a constructor. + * + * @param fragCharSize the length (number of chars) of a fragment + */ + public SimpleFieldFragList( int fragCharSize ) { + super( fragCharSize ); + } + + /* (non-Javadoc) + * @see org.apache.lucene.search.vectorhighlight.FieldFragList#add( int startOffset, int endOffset, List phraseInfoList ) + */ + @Override + public void add( int startOffset, int endOffset, List phraseInfoList ) { + float totalBoost = 0; + List subInfos = new ArrayList(); + for( WeightedPhraseInfo phraseInfo : phraseInfoList ){ + subInfos.add( new SubInfo( phraseInfo.getText(), phraseInfo.getTermsOffsets(), phraseInfo.getSeqnum() ) ); + totalBoost += phraseInfo.getBoost(); + } + getFragInfos().add( new WeightedFragInfo( startOffset, endOffset, subInfos, totalBoost ) ); + } + +} diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFragListBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFragListBuilder.java index 8a9447b..c155900 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFragListBuilder.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFragListBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,83 +17,25 @@ package org.apache.lucene.search.vectorhighlight; * limitations under the License. */ -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo; /** * A simple implementation of {@link FragListBuilder}. */ -public class SimpleFragListBuilder implements FragListBuilder { +public class SimpleFragListBuilder extends BaseFragListBuilder { - public static final int MARGIN_DEFAULT = 6; - public static final int MIN_FRAG_CHAR_SIZE_FACTOR = 3; - - final int margin; - final int minFragCharSize; - - public SimpleFragListBuilder( int margin ){ - if( margin < 0 ) - throw new IllegalArgumentException( "margin(" + margin + ") is too small. It must be 0 or higher." ); - - this.margin = margin; - this.minFragCharSize = Math.max( 1, margin * MIN_FRAG_CHAR_SIZE_FACTOR ); + public SimpleFragListBuilder() { + super(); } - public SimpleFragListBuilder(){ - this( MARGIN_DEFAULT ); + public SimpleFragListBuilder(int margin) { + super(margin); } - public FieldFragList createFieldFragList(FieldPhraseList fieldPhraseList, int fragCharSize) { - if( fragCharSize < minFragCharSize ) - throw new IllegalArgumentException( "fragCharSize(" + fragCharSize + ") is too small. It must be " + - minFragCharSize + " or higher." ); - - FieldFragList ffl = new FieldFragList( fragCharSize ); - - List wpil = new ArrayList(); - Iterator ite = fieldPhraseList.phraseList.iterator(); - WeightedPhraseInfo phraseInfo = null; - int startOffset = 0; - boolean taken = false; - while( true ){ - if( !taken ){ - if( !ite.hasNext() ) break; - phraseInfo = ite.next(); - } - taken = false; - if( phraseInfo == null ) break; - - // if the phrase violates the border of previous fragment, discard it and try next phrase - if( phraseInfo.getStartOffset() < startOffset ) continue; - - wpil.clear(); - wpil.add( phraseInfo ); - int st = phraseInfo.getStartOffset() - margin < startOffset ? - startOffset : phraseInfo.getStartOffset() - margin; - int en = st + fragCharSize; - if( phraseInfo.getEndOffset() > en ) - en = phraseInfo.getEndOffset(); - startOffset = en; - - while( true ){ - if( ite.hasNext() ){ - phraseInfo = ite.next(); - taken = true; - if( phraseInfo == null ) break; - } - else - break; - if( phraseInfo.getEndOffset() <= en ) - wpil.add( phraseInfo ); - else - break; - } - ffl.add( st, en, wpil ); - } - return ffl; + /* (non-Javadoc) + * @see org.apache.lucene.search.vectorhighlight.FragListBuilder#createFieldFragList(FieldPhraseList fieldPhraseList, int fragCharSize) + */ + public FieldFragList createFieldFragList( FieldPhraseList fieldPhraseList, int fragCharSize ){ + return createFieldFragList( fieldPhraseList, new SimpleFieldFragList( fragCharSize ), fragCharSize ); } } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilder.java index 2822cf3..026c97c 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilder.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SingleFragListBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SingleFragListBuilder.java index ae3b210..e8a2e17 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SingleFragListBuilder.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SingleFragListBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -38,7 +38,7 @@ public class SingleFragListBuilder implements FragListBuilder { public FieldFragList createFieldFragList(FieldPhraseList fieldPhraseList, int fragCharSize) { - FieldFragList ffl = new FieldFragList( fragCharSize ); + FieldFragList ffl = new SimpleFieldFragList( fragCharSize ); List wpil = new ArrayList(); Iterator ite = fieldPhraseList.phraseList.iterator(); diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/WeightedFieldFragList.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/WeightedFieldFragList.java new file mode 100644 index 0000000..54122ff --- /dev/null +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/WeightedFieldFragList.java @@ -0,0 +1,76 @@ +package org.apache.lucene.search.vectorhighlight; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; + +import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo.SubInfo; +import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo; +import org.apache.lucene.search.vectorhighlight.FieldTermStack.TermInfo; + +/** + * A weighted implementation of {@link FieldFragList}. + */ +public class WeightedFieldFragList extends FieldFragList { + + /** + * a constructor. + * + * @param fragCharSize the length (number of chars) of a fragment + */ + public WeightedFieldFragList( int fragCharSize ) { + super( fragCharSize ); + } + + /* (non-Javadoc) + * @see org.apache.lucene.search.vectorhighlight.FieldFragList#add( int startOffset, int endOffset, List phraseInfoList ) + */ + @Override + public void add( int startOffset, int endOffset, List phraseInfoList ) { + + float totalBoost = 0; + + List subInfos = new ArrayList(); + + HashSet distinctTerms = new HashSet(); + + int length = 0; + + for( WeightedPhraseInfo phraseInfo : phraseInfoList ){ + + subInfos.add( new SubInfo( phraseInfo.getText(), phraseInfo.getTermsOffsets(), phraseInfo.getSeqnum() ) ); + + for ( TermInfo ti : phraseInfo.getTermsInfos()) { + if ( distinctTerms.add( ti.getText() ) ) + totalBoost += ti.getWeight() * phraseInfo.getBoost(); + length++; + } + } + + // We want that terms per fragment (length) is included into the weight. Otherwise a one-word-query + // would cause an equal weight for all fragments regardless of how much words they contain. + // To avoid that fragments containing a high number of words possibly "outrank" more relevant fragments + // we "bend" the length with a standard-normalization a little bit. + totalBoost *= length * ( 1 / Math.sqrt( length ) ); + + getFragInfos().add( new WeightedFragInfo( startOffset, endOffset, subInfos, totalBoost ) ); + } + +} \ No newline at end of file diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/WeightedFragListBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/WeightedFragListBuilder.java new file mode 100644 index 0000000..655a6c7 --- /dev/null +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/WeightedFragListBuilder.java @@ -0,0 +1,41 @@ +package org.apache.lucene.search.vectorhighlight; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A weighted implementation of {@link FragListBuilder}. + */ +public class WeightedFragListBuilder extends BaseFragListBuilder { + + public WeightedFragListBuilder() { + super(); + } + + public WeightedFragListBuilder(int margin) { + super(margin); + } + + /* (non-Javadoc) + * @see org.apache.lucene.search.vectorhighlight.FragListBuilder#createFieldFragList(FieldPhraseList fieldPhraseList, int fragCharSize) + */ + @Override + public FieldFragList createFieldFragList( FieldPhraseList fieldPhraseList, int fragCharSize ){ + return createFieldFragList( fieldPhraseList, new WeightedFieldFragList( fragCharSize ), fragCharSize ); + } + +} diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/package.html b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/package.html index d713fa2..f8f1741 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/package.html +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/package.html @@ -27,9 +27,9 @@ This is an another highlighter implementation.
    • support multi-term (includes wildcard, range, regexp, etc) queries
    • need Java 1.5
    • highlight fields need to be stored with Positions and Offsets
      • -
    • take into account query boost to score fragments
      • +
    • take into account query boost and/or IDF-weight to score fragments
    • support colored highlight tags
      • -
    • pluggable FragListBuilder
      • +
    • pluggable FragListBuilder / FieldFragList
    • pluggable FragmentsBuilder
      • @@ -122,9 +122,8 @@ by reference to QueryPhraseMap and FieldTermStack.

      +----------------+-----------------+---+

      The type of each entry is WeightedPhraseInfo that consists of -an array of terms offsets and weight. The weight (Fast Vector Highlighter uses query boost to -calculate the weight) will be taken into account when Fast Vector Highlighter creates -{@link org.apache.lucene.search.vectorhighlight.FieldFragList} in the next step.

      +an array of terms offsets and weight. +

      Step 4.

      In Step 4, Fast Vector Highlighter creates FieldFragList by reference to FieldPhraseList. In this sample case, the following @@ -137,6 +136,59 @@ calculate the weight) will be taken into account when Fast Vector Highlighter cr |totalBoost=3 | +---------------------------------+ + +

      +The calculation for each FieldFragList.WeightedFragInfo.totalBoost (weight) +depends on the implementation of FieldFragList.add( ... ): +

      +  public void add( int startOffset, int endOffset, List<WeightedPhraseInfo> phraseInfoList ) {
+    float totalBoost = 0;
+    List<SubInfo> subInfos = new ArrayList<SubInfo>();
+    for( WeightedPhraseInfo phraseInfo : phraseInfoList ){
+      subInfos.add( new SubInfo( phraseInfo.getText(), phraseInfo.getTermsOffsets(), phraseInfo.getSeqnum() ) );
+      totalBoost += phraseInfo.getBoost();
+    }
+    getFragInfos().add( new WeightedFragInfo( startOffset, endOffset, subInfos, totalBoost ) );
+  }
+  
+
      +The used implementation of FieldFragList is noted in BaseFragListBuilder.createFieldFragList( ... ): +
      +  public FieldFragList createFieldFragList( FieldPhraseList fieldPhraseList, int fragCharSize ){
+    return createFieldFragList( fieldPhraseList, new SimpleFieldFragList( fragCharSize ), fragCharSize );
+  }
+
      +

      +Currently there are basically to approaches available: +

      +
        +
      • SimpleFragListBuilder using SimpleFieldFragList: sum-of-boosts-approach. The totalBoost is calculated by summarizing the query-boosts per term. Per default a term is boosted by 1.0
        • +
      • WeightedFragListBuilder using WeightedFieldFragList: sum-of-distinct-weights-approach. The totalBoost is calculated by summarizing the IDF-weights of distinct terms.
        • +
      +

      Comparison of the two approaches:

      + + + + + + + + + + + + + + + + + + + +
      + query = das alte testament (The Old Testament) +
      Terms in fragmentsum-of-distinct-weightssum-of-boosts
      das alte testament5.3396213.0
      das alte testament5.3396213.0
      das testament alte5.3396213.0
      das alte testament5.3396213.0
      das testament2.94556882.0
      das alte2.47595952.0
      das das das das1.50153574.0
      das das das1.30036813.0
      das das1.0617462.0
      alte1.01.0
      alte1.01.0
      das0.75076781.0
      das0.75076781.0
      das0.75076781.0
      das0.75076781.0
      das0.75076781.0
      +

      Step 5.

      In Step 5, by using FieldFragList and the field stored data, Fast Vector Highlighter creates highlighted snippets!

      diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java index 5c2b64f..2f7d56d 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -31,7 +31,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; @@ -45,21 +45,19 @@ import org.apache.lucene.search.spans.SpanNearQuery; import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.FixedBitSet; public class HighlighterPhraseTest extends LuceneTestCase { private static final String FIELD = "text"; - public void testConcurrentPhrase() throws CorruptIndexException, - LockObtainFailedException, IOException, InvalidTokenOffsetsException { + public void testConcurrentPhrase() throws IOException, InvalidTokenOffsetsException { final String TEXT = "the fox jumped"; final Directory directory = newDirectory(); final IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false))); try { final Document document = new Document(); - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setStoreTermVectorOffsets(true); customType.setStoreTermVectorPositions(true); customType.setStoreTermVectors(true); @@ -68,7 +66,7 @@ public class HighlighterPhraseTest extends LuceneTestCase { } finally { indexWriter.close(); } - final IndexReader indexReader = IndexReader.open(directory); + final IndexReader indexReader = DirectoryReader.open(directory); try { assertEquals(1, indexReader.numDocs()); final IndexSearcher indexSearcher = newSearcher(indexReader); @@ -93,8 +91,7 @@ public class HighlighterPhraseTest extends LuceneTestCase { } } - public void testConcurrentSpan() throws CorruptIndexException, - LockObtainFailedException, IOException, InvalidTokenOffsetsException { + public void testConcurrentSpan() throws IOException, InvalidTokenOffsetsException { final String TEXT = "the fox jumped"; final Directory directory = newDirectory(); final IndexWriter indexWriter = new IndexWriter(directory, @@ -102,7 +99,7 @@ public class HighlighterPhraseTest extends LuceneTestCase { try { final Document document = new Document(); - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setStoreTermVectorOffsets(true); customType.setStoreTermVectorPositions(true); customType.setStoreTermVectors(true); @@ -111,7 +108,7 @@ public class HighlighterPhraseTest extends LuceneTestCase { } finally { indexWriter.close(); } - final IndexReader indexReader = IndexReader.open(directory); + final IndexReader indexReader = DirectoryReader.open(directory); try { assertEquals(1, indexReader.numDocs()); final IndexSearcher indexSearcher = newSearcher(indexReader); @@ -128,19 +125,17 @@ public class HighlighterPhraseTest extends LuceneTestCase { } @Override - public void collect(int i) throws IOException { + public void collect(int i) { bitset.set(this.baseDoc + i); } @Override - public void setNextReader(AtomicReaderContext context) - throws IOException { + public void setNextReader(AtomicReaderContext context) { this.baseDoc = context.docBase; } @Override - public void setScorer(org.apache.lucene.search.Scorer scorer) - throws IOException { + public void setScorer(org.apache.lucene.search.Scorer scorer) { // Do Nothing } }); @@ -164,8 +159,7 @@ public class HighlighterPhraseTest extends LuceneTestCase { } } - public void testSparsePhrase() throws CorruptIndexException, - LockObtainFailedException, IOException, InvalidTokenOffsetsException { + public void testSparsePhrase() throws IOException, InvalidTokenOffsetsException { final String TEXT = "the fox did not jump"; final Directory directory = newDirectory(); final IndexWriter indexWriter = new IndexWriter(directory, @@ -173,7 +167,7 @@ public class HighlighterPhraseTest extends LuceneTestCase { try { final Document document = new Document(); - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setStoreTermVectorOffsets(true); customType.setStoreTermVectorPositions(true); customType.setStoreTermVectors(true); @@ -182,7 +176,7 @@ public class HighlighterPhraseTest extends LuceneTestCase { } finally { indexWriter.close(); } - final IndexReader indexReader = IndexReader.open(directory); + final IndexReader indexReader = DirectoryReader.open(directory); try { assertEquals(1, indexReader.numDocs()); final IndexSearcher indexSearcher = newSearcher(indexReader); @@ -207,8 +201,7 @@ public class HighlighterPhraseTest extends LuceneTestCase { } } - public void testSparsePhraseWithNoPositions() throws CorruptIndexException, - LockObtainFailedException, IOException, InvalidTokenOffsetsException { + public void testSparsePhraseWithNoPositions() throws IOException, InvalidTokenOffsetsException { final String TEXT = "the fox did not jump"; final Directory directory = newDirectory(); final IndexWriter indexWriter = new IndexWriter(directory, @@ -224,7 +217,7 @@ public class HighlighterPhraseTest extends LuceneTestCase { } finally { indexWriter.close(); } - final IndexReader indexReader = IndexReader.open(directory); + final IndexReader indexReader = DirectoryReader.open(directory); try { assertEquals(1, indexReader.numDocs()); final IndexSearcher indexSearcher = newSearcher(indexReader); @@ -247,15 +240,14 @@ public class HighlighterPhraseTest extends LuceneTestCase { } } - public void testSparseSpan() throws CorruptIndexException, - LockObtainFailedException, IOException, InvalidTokenOffsetsException { + public void testSparseSpan() throws IOException, InvalidTokenOffsetsException { final String TEXT = "the fox did not jump"; final Directory directory = newDirectory(); final IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false))); try { final Document document = new Document(); - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setStoreTermVectorOffsets(true); customType.setStoreTermVectorPositions(true); customType.setStoreTermVectors(true); @@ -264,7 +256,7 @@ public class HighlighterPhraseTest extends LuceneTestCase { } finally { indexWriter.close(); } - final IndexReader indexReader = IndexReader.open(directory); + final IndexReader indexReader = DirectoryReader.open(directory); try { assertEquals(1, indexReader.numDocs()); final IndexSearcher indexSearcher = newSearcher(indexReader); @@ -303,7 +295,7 @@ public class HighlighterPhraseTest extends LuceneTestCase { } @Override - public boolean incrementToken() throws IOException { + public boolean incrementToken() { this.i++; if (this.i >= this.tokens.length) { return false; @@ -343,7 +335,7 @@ public class HighlighterPhraseTest extends LuceneTestCase { } @Override - public boolean incrementToken() throws IOException { + public boolean incrementToken() { this.i++; if (this.i >= this.tokens.length) { return false; diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java index e4d5978..67ea75f 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -39,6 +39,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.IntField; import org.apache.lucene.document.StoredField; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig.OpenMode; @@ -1370,7 +1371,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte // now an ugly built of XML parsing to test the snippet is encoded OK DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); DocumentBuilder db = dbf.newDocumentBuilder(); - org.w3c.dom.Document doc = db.parse(new ByteArrayInputStream(xhtml.getBytes())); + org.w3c.dom.Document doc = db.parse(new ByteArrayInputStream(xhtml.getBytes("UTF-8"))); Element root = doc.getDocumentElement(); NodeList nodes = root.getElementsByTagName("body"); Element body = (Element) nodes.item(0); @@ -1457,7 +1458,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte } @Override - public boolean incrementToken() throws IOException { + public boolean incrementToken() { if(iter.hasNext()) { Token token = iter.next(); clearAttributes(); @@ -1508,7 +1509,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte } @Override - public boolean incrementToken() throws IOException { + public boolean incrementToken() { if(iter.hasNext()) { Token token = iter.next(); clearAttributes(); @@ -1621,7 +1622,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte private Document doc( String f, String v ){ Document doc = new Document(); - doc.add( new Field( f, v, TextField.TYPE_STORED)); + doc.add( new TextField( f, v, Field.Store.YES)); return doc; } @@ -1645,7 +1646,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte private void searchIndex() throws IOException, InvalidTokenOffsetsException { Query query = new TermQuery(new Term("t_text1", "random")); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); // This scorer can return negative idf -> null fragment Scorer scorer = new QueryTermScorer( query, searcher.getIndexReader(), "t_text1" ); @@ -1672,7 +1673,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte * writer = new IndexWriter(ramDir,bigramAnalyzer , true); Document d = new * Document(); Field f = new Field(FIELD_NAME, "java abc def", true, true, * true); d.add(f); writer.addDocument(d); writer.close(); IndexReader reader = - * IndexReader.open(ramDir); + * DirectoryReader.open(ramDir); * * IndexSearcher searcher=new IndexSearcher(reader); query = * QueryParser.parse("abc", FIELD_NAME, bigramAnalyzer); @@ -1739,28 +1740,28 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte addDoc(writer, text); } Document doc = new Document(); - doc.add(new IntField(NUMERIC_FIELD_NAME, 1)); + doc.add(new IntField(NUMERIC_FIELD_NAME, 1, Field.Store.NO)); doc.add(new StoredField(NUMERIC_FIELD_NAME, 1)); writer.addDocument(doc, analyzer); doc = new Document(); - doc.add(new IntField(NUMERIC_FIELD_NAME, 3)); + doc.add(new IntField(NUMERIC_FIELD_NAME, 3, Field.Store.NO)); doc.add(new StoredField(NUMERIC_FIELD_NAME, 3)); writer.addDocument(doc, analyzer); doc = new Document(); - doc.add(new IntField(NUMERIC_FIELD_NAME, 5)); + doc.add(new IntField(NUMERIC_FIELD_NAME, 5, Field.Store.NO)); doc.add(new StoredField(NUMERIC_FIELD_NAME, 5)); writer.addDocument(doc, analyzer); doc = new Document(); - doc.add(new IntField(NUMERIC_FIELD_NAME, 7)); + doc.add(new IntField(NUMERIC_FIELD_NAME, 7, Field.Store.NO)); doc.add(new StoredField(NUMERIC_FIELD_NAME, 7)); writer.addDocument(doc, analyzer); writer.forceMerge(1); writer.close(); - reader = IndexReader.open(ramDir); + reader = DirectoryReader.open(ramDir); numHighlights = 0; } @@ -1774,7 +1775,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte private void addDoc(IndexWriter writer, String text) throws IOException { Document d = new Document(); - Field f = new Field(FIELD_NAME, text, TextField.TYPE_STORED); + Field f = new TextField(FIELD_NAME, text, Field.Store.YES); d.add(f); writer.addDocument(d); diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java index d22d721..7e0070f 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java index 61735ec..8e8bea2 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,7 +19,8 @@ package org.apache.lucene.search.highlight; import java.io.IOException; -import org.apache.lucene.analysis.*; +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; @@ -27,7 +28,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; @@ -39,7 +40,6 @@ import org.apache.lucene.search.spans.SpanNearQuery; import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.LuceneTestCase; // LUCENE-2874 @@ -56,7 +56,7 @@ public class TokenSourcesTest extends LuceneTestCase { private final PositionIncrementAttribute positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class); @Override - public boolean incrementToken() throws IOException { + public boolean incrementToken() { this.i++; if (this.i >= this.tokens.length) { return false; @@ -84,15 +84,14 @@ public class TokenSourcesTest extends LuceneTestCase { } } - public void testOverlapWithOffset() throws CorruptIndexException, - LockObtainFailedException, IOException, InvalidTokenOffsetsException { + public void testOverlapWithOffset() throws IOException, InvalidTokenOffsetsException { final String TEXT = "the fox did not jump"; final Directory directory = newDirectory(); final IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, null)); try { final Document document = new Document(); - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setStoreTermVectors(true); customType.setStoreTermVectorOffsets(true); document.add(new Field(FIELD, new OverlappingTokenStream(), customType)); @@ -100,7 +99,7 @@ public class TokenSourcesTest extends LuceneTestCase { } finally { indexWriter.close(); } - final IndexReader indexReader = IndexReader.open(directory); + final IndexReader indexReader = DirectoryReader.open(directory); assertEquals(1, indexReader.numDocs()); final IndexSearcher indexSearcher = newSearcher(indexReader); try { @@ -128,15 +127,15 @@ public class TokenSourcesTest extends LuceneTestCase { } } - public void testOverlapWithPositionsAndOffset() throws CorruptIndexException, - LockObtainFailedException, IOException, InvalidTokenOffsetsException { + public void testOverlapWithPositionsAndOffset() + throws IOException, InvalidTokenOffsetsException { final String TEXT = "the fox did not jump"; final Directory directory = newDirectory(); final IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, null)); try { final Document document = new Document(); - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setStoreTermVectors(true); customType.setStoreTermVectorOffsets(true); customType.setStoreTermVectorPositions(true); @@ -145,7 +144,7 @@ public class TokenSourcesTest extends LuceneTestCase { } finally { indexWriter.close(); } - final IndexReader indexReader = IndexReader.open(directory); + final IndexReader indexReader = DirectoryReader.open(directory); try { assertEquals(1, indexReader.numDocs()); final IndexSearcher indexSearcher = newSearcher(indexReader); @@ -173,15 +172,15 @@ public class TokenSourcesTest extends LuceneTestCase { } } - public void testOverlapWithOffsetExactPhrase() throws CorruptIndexException, - LockObtainFailedException, IOException, InvalidTokenOffsetsException { + public void testOverlapWithOffsetExactPhrase() + throws IOException, InvalidTokenOffsetsException { final String TEXT = "the fox did not jump"; final Directory directory = newDirectory(); final IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, null)); try { final Document document = new Document(); - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setStoreTermVectors(true); customType.setStoreTermVectorOffsets(true); document.add(new Field(FIELD, new OverlappingTokenStream(), customType)); @@ -189,7 +188,7 @@ public class TokenSourcesTest extends LuceneTestCase { } finally { indexWriter.close(); } - final IndexReader indexReader = IndexReader.open(directory); + final IndexReader indexReader = DirectoryReader.open(directory); try { assertEquals(1, indexReader.numDocs()); final IndexSearcher indexSearcher = newSearcher(indexReader); @@ -218,15 +217,14 @@ public class TokenSourcesTest extends LuceneTestCase { } public void testOverlapWithPositionsAndOffsetExactPhrase() - throws CorruptIndexException, LockObtainFailedException, IOException, - InvalidTokenOffsetsException { + throws IOException, InvalidTokenOffsetsException { final String TEXT = "the fox did not jump"; final Directory directory = newDirectory(); final IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, null)); try { final Document document = new Document(); - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setStoreTermVectors(true); customType.setStoreTermVectorOffsets(true); document.add(new Field(FIELD, new OverlappingTokenStream(), customType)); @@ -234,7 +232,7 @@ public class TokenSourcesTest extends LuceneTestCase { } finally { indexWriter.close(); } - final IndexReader indexReader = IndexReader.open(directory); + final IndexReader indexReader = DirectoryReader.open(directory); try { assertEquals(1, indexReader.numDocs()); final IndexSearcher indexSearcher = newSearcher(indexReader); diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/HighlightCustomQueryTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/HighlightCustomQueryTest.java index d0a54cf..98ec317 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/HighlightCustomQueryTest.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/HighlightCustomQueryTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.highlight.custom; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/poshighlight/PosHighlighterTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/poshighlight/PosHighlighterTest.java index 434b0af..e22aac1 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/poshighlight/PosHighlighterTest.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/poshighlight/PosHighlighterTest.java @@ -368,6 +368,17 @@ public class PosHighlighterTest extends LuceneTestCase { frags[0]); close(); } + + public void testSloppyPhraseQuery() throws Exception { + insertDocs(analyzer, "a b c d a b c d e f"); + PhraseQuery pq = new PhraseQuery(); + pq.add(new Term(F, "c")); + pq.add(new Term(F, "a")); + pq.setSlop(2); + String frags[] = doSearch(pq, 50); + assertEquals("a b c d a b c d e f", frags[0]); + close(); + } public static class BlockPositionIteratorFilter implements PositionIntervalFilter { diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java index e0f66e0..d2593ed 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -32,6 +32,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig.OpenMode; @@ -322,7 +323,7 @@ public abstract class AbstractTestCase extends LuceneTestCase { } @Override - public void reset() throws IOException { + public void reset() { startTerm = 0; nextStartOffset = 0; snippet = null; @@ -364,7 +365,7 @@ public abstract class AbstractTestCase extends LuceneTestCase { writer.addDocument( doc ); writer.close(); if (reader != null) reader.close(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); } // make 1 doc with multi valued & not analyzed field @@ -383,7 +384,7 @@ public abstract class AbstractTestCase extends LuceneTestCase { writer.addDocument( doc ); writer.close(); if (reader != null) reader.close(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); } protected void makeIndexShortMV() throws Exception { diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/BreakIteratorBoundaryScannerTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/BreakIteratorBoundaryScannerTest.java index 28447a1..78f287f 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/BreakIteratorBoundaryScannerTest.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/BreakIteratorBoundaryScannerTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldPhraseListTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldPhraseListTest.java index df58435..45054e6 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldPhraseListTest.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldPhraseListTest.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java index c6aec47..14cdd58 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -810,13 +810,13 @@ public class FieldQueryTest extends AbstractTestCase { // "a" List phraseCandidate = new ArrayList(); - phraseCandidate.add( new TermInfo( "a", 0, 1, 0 ) ); + phraseCandidate.add( new TermInfo( "a", 0, 1, 0, 1 ) ); assertNull( fq.searchPhrase( F, phraseCandidate ) ); // "a b" - phraseCandidate.add( new TermInfo( "b", 2, 3, 1 ) ); + phraseCandidate.add( new TermInfo( "b", 2, 3, 1, 1 ) ); assertNull( fq.searchPhrase( F, phraseCandidate ) ); // "a b c" - phraseCandidate.add( new TermInfo( "c", 4, 5, 2 ) ); + phraseCandidate.add( new TermInfo( "c", 4, 5, 2, 1 ) ); assertNotNull( fq.searchPhrase( F, phraseCandidate ) ); assertNull( fq.searchPhrase( "x", phraseCandidate ) ); @@ -832,13 +832,13 @@ public class FieldQueryTest extends AbstractTestCase { // "a" phraseCandidate.clear(); - phraseCandidate.add( new TermInfo( "a", 0, 1, 0 ) ); + phraseCandidate.add( new TermInfo( "a", 0, 1, 0, 1 ) ); assertNotNull( fq.searchPhrase( F, phraseCandidate ) ); // "a b" - phraseCandidate.add( new TermInfo( "b", 2, 3, 1 ) ); + phraseCandidate.add( new TermInfo( "b", 2, 3, 1, 1 ) ); assertNull( fq.searchPhrase( F, phraseCandidate ) ); // "a b c" - phraseCandidate.add( new TermInfo( "c", 4, 5, 2 ) ); + phraseCandidate.add( new TermInfo( "c", 4, 5, 2, 1 ) ); assertNotNull( fq.searchPhrase( F, phraseCandidate ) ); assertNull( fq.searchPhrase( "x", phraseCandidate ) ); } @@ -852,9 +852,9 @@ public class FieldQueryTest extends AbstractTestCase { // "a b c" w/ position-gap = 2 List phraseCandidate = new ArrayList(); - phraseCandidate.add( new TermInfo( "a", 0, 1, 0 ) ); - phraseCandidate.add( new TermInfo( "b", 2, 3, 2 ) ); - phraseCandidate.add( new TermInfo( "c", 4, 5, 4 ) ); + phraseCandidate.add( new TermInfo( "a", 0, 1, 0, 1 ) ); + phraseCandidate.add( new TermInfo( "b", 2, 3, 2, 1 ) ); + phraseCandidate.add( new TermInfo( "c", 4, 5, 4, 1 ) ); assertNull( fq.searchPhrase( F, phraseCandidate ) ); // "a b c"~1 @@ -868,9 +868,9 @@ public class FieldQueryTest extends AbstractTestCase { // "a b c" w/ position-gap = 3 phraseCandidate.clear(); - phraseCandidate.add( new TermInfo( "a", 0, 1, 0 ) ); - phraseCandidate.add( new TermInfo( "b", 2, 3, 3 ) ); - phraseCandidate.add( new TermInfo( "c", 4, 5, 6 ) ); + phraseCandidate.add( new TermInfo( "a", 0, 1, 0, 1 ) ); + phraseCandidate.add( new TermInfo( "b", 2, 3, 3, 1 ) ); + phraseCandidate.add( new TermInfo( "c", 4, 5, 6, 1 ) ); assertNull( fq.searchPhrase( F, phraseCandidate ) ); } @@ -901,7 +901,7 @@ public class FieldQueryTest extends AbstractTestCase { assertNotNull (qpm); assertNull (fq.getFieldTermMap(F, "dog")); List phraseCandidate = new ArrayList(); - phraseCandidate.add( new TermInfo( "defg", 0, 12, 0 ) ); + phraseCandidate.add( new TermInfo( "defg", 0, 12, 0, 1 ) ); assertNotNull (fq.searchPhrase(F, phraseCandidate)); } diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldTermStackTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldTermStackTest.java index 68402b4..2059408 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldTermStackTest.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldTermStackTest.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java index 2035cd4..91f0773 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -303,7 +303,7 @@ public class IndexTimeSynonymTest extends AbstractTestCase { int p = 0; @Override - public boolean incrementToken() throws IOException { + public boolean incrementToken() { if( p >= tokens.length ) return false; clearAttributes(); tokens[p++].copyTo(reusableToken); diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilderTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilderTest.java index 1e7fe99..4124388 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilderTest.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilderTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleBoundaryScannerTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleBoundaryScannerTest.java index 2aa3262..343af84 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleBoundaryScannerTest.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleBoundaryScannerTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragListBuilderTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragListBuilderTest.java index 5f30c61..9c8e373 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragListBuilderTest.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragListBuilderTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java index 16fe102..874e151 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.vectorhighlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,7 +21,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig.OpenMode; @@ -138,7 +138,7 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase { IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( TEST_VERSION_CURRENT, analyzerW).setOpenMode(OpenMode.CREATE)); Document doc = new Document(); - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); customType.setStoreTermVectors(true); customType.setStoreTermVectorOffsets(true); customType.setStoreTermVectorPositions(true); @@ -147,7 +147,7 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase { writer.addDocument( doc ); writer.close(); if (reader != null) reader.close(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); } public void test1StrMV() throws Exception { diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SingleFragListBuilderTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SingleFragListBuilderTest.java index 0f42bae..740b63e 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SingleFragListBuilderTest.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SingleFragListBuilderTest.java @@ -4,7 +4,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/WeightedFragListBuilderTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/WeightedFragListBuilderTest.java new file mode 100644 index 0000000..9d7d566 --- /dev/null +++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/WeightedFragListBuilderTest.java @@ -0,0 +1,35 @@ +package org.apache.lucene.search.vectorhighlight; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class WeightedFragListBuilderTest extends AbstractTestCase { + + public void test2WeightedFragList() throws Exception { + + makeIndexLongMV(); + + FieldQuery fq = new FieldQuery( pqF( "the", "both" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + WeightedFragListBuilder wflb = new WeightedFragListBuilder(); + FieldFragList ffl = wflb.createFieldFragList( fpl, 100 ); + assertEquals( 1, ffl.getFragInfos().size() ); + assertEquals( "subInfos=(theboth((195,203)))/0.86791086(189,289)", ffl.getFragInfos().get( 0 ).toString() ); + } + +} diff --git a/lucene/ivy-settings.xml b/lucene/ivy-settings.xml index 8c82286..97376ff 100644 --- a/lucene/ivy-settings.xml +++ b/lucene/ivy-settings.xml @@ -29,7 +29,7 @@ - + diff --git a/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java b/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java index 05586cd..9502d04 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java @@ -38,12 +38,24 @@ public final class JoinUtil { *

      * Execute the returned query with a {@link IndexSearcher} to retrieve all documents that have the same terms in the * to field that match with documents matching the specified fromQuery and have the same terms in the from field. + *

      + * In the case a single document relates to more than one document the multipleValuesPerDocument option + * should be set to true. When the multipleValuesPerDocument is set to true only the + * the score from the first encountered join value originating from the 'from' side is mapped into the 'to' side. + * Even in the case when a second join value related to a specific document yields a higher score. Obviously this + * doesn't apply in the case that {@link ScoreMode#None} is used, since no scores are computed at all. + *

      + * Memory considerations: During joining all unique join values are kept in memory. On top of that when the scoreMode + * isn't set to {@link ScoreMode#None} a float value per unique join value is kept in memory for computing scores. + * When scoreMode is set to {@link ScoreMode#Avg} also an additional integer value is kept in memory per unique + * join value. * * @param fromField The from field to join from * @param multipleValuesPerDocument Whether the from field has multiple terms per document * @param toField The to field to join to * @param fromQuery The query to match documents on the from side * @param fromSearcher The searcher that executed the specified fromQuery + * @param scoreMode Instructs how scores from the fromQuery are mapped to the returned query * @return a {@link Query} instance that can be used to join documents based on the * terms in the from and to field * @throws IOException If I/O related errors occur @@ -52,10 +64,29 @@ public final class JoinUtil { boolean multipleValuesPerDocument, String toField, Query fromQuery, - IndexSearcher fromSearcher) throws IOException { - TermsCollector termsCollector = TermsCollector.create(fromField, multipleValuesPerDocument); - fromSearcher.search(fromQuery, termsCollector); - return new TermsQuery(toField, termsCollector.getCollectorTerms()); + IndexSearcher fromSearcher, + ScoreMode scoreMode) throws IOException { + switch (scoreMode) { + case None: + TermsCollector termsCollector = TermsCollector.create(fromField, multipleValuesPerDocument); + fromSearcher.search(fromQuery, termsCollector); + return new TermsQuery(toField, termsCollector.getCollectorTerms()); + case Total: + case Max: + case Avg: + TermsWithScoreCollector termsWithScoreCollector = + TermsWithScoreCollector.create(fromField, multipleValuesPerDocument, scoreMode); + fromSearcher.search(fromQuery, termsWithScoreCollector); + return new TermsIncludingScoreQuery( + toField, + multipleValuesPerDocument, + termsWithScoreCollector.getCollectedTerms(), + termsWithScoreCollector.getScoresPerTerm(), + fromQuery + ); + default: + throw new IllegalArgumentException(String.format("Score mode %s isn't supported.", scoreMode)); + } } } diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ScoreMode.java b/lucene/join/src/java/org/apache/lucene/search/join/ScoreMode.java new file mode 100644 index 0000000..5b6fc10 --- /dev/null +++ b/lucene/join/src/java/org/apache/lucene/search/join/ScoreMode.java @@ -0,0 +1,45 @@ +package org.apache.lucene.search.join; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * How to aggregate multiple child hit scores into a single parent score. + */ +public enum ScoreMode { + + /** + * Do no scoring. + */ + None, + + /** + * Parent hit's score is the average of all child scores. + */ + Avg, + + /** + * Parent hit's score is the max of all child scores. + */ + Max, + + /** + * Parent hit's score is the sum of all child scores. + */ + Total + +} diff --git a/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java new file mode 100644 index 0000000..ab40e90 --- /dev/null +++ b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java @@ -0,0 +1,277 @@ +package org.apache.lucene.search.join; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.ComplexExplanation; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; +import org.apache.lucene.search.positions.PositionIntervalIterator; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefHash; +import org.apache.lucene.util.FixedBitSet; + +import java.io.IOException; +import java.util.Set; + +class TermsIncludingScoreQuery extends Query { + + final String field; + final boolean multipleValuesPerDocument; + final BytesRefHash terms; + final float[] scores; + final int[] ords; + final Query originalQuery; + final Query unwrittenOriginalQuery; + + TermsIncludingScoreQuery(String field, boolean multipleValuesPerDocument, BytesRefHash terms, float[] scores, Query originalQuery) { + this.field = field; + this.multipleValuesPerDocument = multipleValuesPerDocument; + this.terms = terms; + this.scores = scores; + this.originalQuery = originalQuery; + this.ords = terms.sort(BytesRef.getUTF8SortedAsUnicodeComparator()); + this.unwrittenOriginalQuery = originalQuery; + } + + private TermsIncludingScoreQuery(String field, boolean multipleValuesPerDocument, BytesRefHash terms, float[] scores, int[] ords, Query originalQuery, Query unwrittenOriginalQuery) { + this.field = field; + this.multipleValuesPerDocument = multipleValuesPerDocument; + this.terms = terms; + this.scores = scores; + this.originalQuery = originalQuery; + this.ords = ords; + this.unwrittenOriginalQuery = unwrittenOriginalQuery; + } + + public String toString(String string) { + return String.format("TermsIncludingScoreQuery{field=%s;originalQuery=%s}", field, unwrittenOriginalQuery); + } + + @Override + public void extractTerms(Set terms) { + originalQuery.extractTerms(terms); + } + + @Override + public Query rewrite(IndexReader reader) throws IOException { + final Query originalQueryRewrite = originalQuery.rewrite(reader); + if (originalQueryRewrite != originalQuery) { + Query rewritten = new TermsIncludingScoreQuery(field, multipleValuesPerDocument, terms, scores, + ords, originalQueryRewrite, originalQuery); + rewritten.setBoost(getBoost()); + return rewritten; + } else { + return this; + } + } + + @Override + public Weight createWeight(IndexSearcher searcher) throws IOException { + final Weight originalWeight = originalQuery.createWeight(searcher); + return new Weight() { + + private TermsEnum segmentTermsEnum; + + public Explanation explain(AtomicReaderContext context, int doc) throws IOException { + SVInnerScorer scorer = (SVInnerScorer) scorer(context, true, false, context.reader().getLiveDocs()); + if (scorer != null) { + if (scorer.advance(doc) == doc) { + return scorer.explain(); + } + } + return new ComplexExplanation(false, 0.0f, "Not a match"); + } + + public Query getQuery() { + return TermsIncludingScoreQuery.this; + } + + public float getValueForNormalization() throws IOException { + return originalWeight.getValueForNormalization() * TermsIncludingScoreQuery.this.getBoost() * TermsIncludingScoreQuery.this.getBoost(); + } + + public void normalize(float norm, float topLevelBoost) { + originalWeight.normalize(norm, topLevelBoost * TermsIncludingScoreQuery.this.getBoost()); + } + + public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder, boolean topScorer, Bits acceptDocs) throws IOException { + Terms terms = context.reader().terms(field); + if (terms == null) { + return null; + } + + segmentTermsEnum = terms.iterator(segmentTermsEnum); + if (multipleValuesPerDocument) { + return new MVInnerScorer(this, acceptDocs, segmentTermsEnum, context.reader().maxDoc()); + } else { + return new SVInnerScorer(this, acceptDocs, segmentTermsEnum); + } + } + }; + } + + // This impl assumes that the 'join' values are used uniquely per doc per field. Used for one to many relations. + class SVInnerScorer extends Scorer { + + final BytesRef spare = new BytesRef(); + final Bits acceptDocs; + final TermsEnum termsEnum; + + int upto; + DocsEnum docsEnum; + DocsEnum reuse; + int scoreUpto; + + SVInnerScorer(Weight weight, Bits acceptDocs, TermsEnum termsEnum) { + super(weight); + this.acceptDocs = acceptDocs; + this.termsEnum = termsEnum; + } + + @Override + public PositionIntervalIterator positions(boolean needsPayloads, boolean needsOffsets) throws IOException { + throw new UnsupportedOperationException(); // nocommit - what to do here? + } + + public float score() throws IOException { + return scores[ords[scoreUpto]]; + } + + public Explanation explain() throws IOException { + return new ComplexExplanation(true, score(), "Score based on join value " + termsEnum.term().utf8ToString()); + } + + public int docID() { + return docsEnum != null ? docsEnum.docID() : DocIdSetIterator.NO_MORE_DOCS; + } + + public int nextDoc() throws IOException { + if (docsEnum != null) { + int docId = docsEnum.nextDoc(); + if (docId == DocIdSetIterator.NO_MORE_DOCS) { + docsEnum = null; + } else { + return docId; + } + } + + do { + if (upto == terms.size()) { + return DocIdSetIterator.NO_MORE_DOCS; + } + + scoreUpto = upto; + TermsEnum.SeekStatus status = termsEnum.seekCeil(terms.get(ords[upto++], spare), true); + if (status == TermsEnum.SeekStatus.FOUND) { + docsEnum = reuse = termsEnum.docs(acceptDocs, reuse, false); + } + } while (docsEnum == null); + + return docsEnum.nextDoc(); + } + + public int advance(int target) throws IOException { + int docId; + do { + docId = nextDoc(); + if (docId < target) { + int tempDocId = docsEnum.advance(target); + if (tempDocId == target) { + docId = tempDocId; + break; + } + } else if (docId == target) { + break; + } + docsEnum = null; // goto the next ord. + } while (docId != DocIdSetIterator.NO_MORE_DOCS); + return docId; + } + } + + // This impl that tracks whether a docid has already been emitted. This check makes sure that docs aren't emitted + // twice for different join values. This means that the first encountered join value determines the score of a document + // even if other join values yield a higher score. + class MVInnerScorer extends SVInnerScorer { + + final FixedBitSet alreadyEmittedDocs; + + MVInnerScorer(Weight weight, Bits acceptDocs, TermsEnum termsEnum, int maxDoc) { + super(weight, acceptDocs, termsEnum); + alreadyEmittedDocs = new FixedBitSet(maxDoc); + } + + public int nextDoc() throws IOException { + if (docsEnum != null) { + int docId; + do { + docId = docsEnum.nextDoc(); + if (docId == DocIdSetIterator.NO_MORE_DOCS) { + break; + } + } while (alreadyEmittedDocs.get(docId)); + if (docId == DocIdSetIterator.NO_MORE_DOCS) { + docsEnum = null; + } else { + alreadyEmittedDocs.set(docId); + return docId; + } + } + + for (;;) { + do { + if (upto == terms.size()) { + return DocIdSetIterator.NO_MORE_DOCS; + } + + scoreUpto = upto; + TermsEnum.SeekStatus status = termsEnum.seekCeil(terms.get(ords[upto++], spare), true); + if (status == TermsEnum.SeekStatus.FOUND) { + docsEnum = reuse = termsEnum.docs(acceptDocs, reuse, false); + } + } while (docsEnum == null); + + int docId; + do { + docId = docsEnum.nextDoc(); + if (docId == DocIdSetIterator.NO_MORE_DOCS) { + break; + } + } while (alreadyEmittedDocs.get(docId)); + if (docId == DocIdSetIterator.NO_MORE_DOCS) { + docsEnum = null; + } else { + alreadyEmittedDocs.set(docId); + return docId; + } + } + } + } + +} diff --git a/lucene/join/src/java/org/apache/lucene/search/join/TermsQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/TermsQuery.java index 71136c4..d20cc2a 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/TermsQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/TermsQuery.java @@ -74,7 +74,7 @@ class TermsQuery extends MultiTermQuery { private BytesRef seekTerm; private int upto = 0; - SeekingTermSetTermsEnum(TermsEnum tenum, BytesRefHash terms) throws IOException { + SeekingTermSetTermsEnum(TermsEnum tenum, BytesRefHash terms) { super(tenum); this.terms = terms; diff --git a/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java b/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java new file mode 100644 index 0000000..af7c2de --- /dev/null +++ b/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java @@ -0,0 +1,292 @@ +package org.apache.lucene.search.join; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.DocTermOrds; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefHash; + +import java.io.IOException; + +abstract class TermsWithScoreCollector extends Collector { + + private final static int INITIAL_ARRAY_SIZE = 256; + + final String field; + final BytesRefHash collectedTerms = new BytesRefHash(); + final ScoreMode scoreMode; + + Scorer scorer; + float[] scoreSums = new float[INITIAL_ARRAY_SIZE]; + + TermsWithScoreCollector(String field, ScoreMode scoreMode) { + this.field = field; + this.scoreMode = scoreMode; + } + + public BytesRefHash getCollectedTerms() { + return collectedTerms; + } + + public float[] getScoresPerTerm() { + return scoreSums; + } + + public void setScorer(Scorer scorer) throws IOException { + this.scorer = scorer; + } + + public boolean acceptsDocsOutOfOrder() { + return true; + } + + /** + * Chooses the right {@link TermsWithScoreCollector} implementation. + * + * @param field The field to collect terms for + * @param multipleValuesPerDocument Whether the field to collect terms for has multiple values per document. + * @return a {@link TermsWithScoreCollector} instance + */ + static TermsWithScoreCollector create(String field, boolean multipleValuesPerDocument, ScoreMode scoreMode) { + if (multipleValuesPerDocument) { + switch (scoreMode) { + case Avg: + return new MV.Avg(field); + default: + return new MV(field, scoreMode); + } + } else { + switch (scoreMode) { + case Avg: + return new SV.Avg(field); + default: + return new SV(field, scoreMode); + } + } + } + + // impl that works with single value per document + static class SV extends TermsWithScoreCollector { + + final BytesRef spare = new BytesRef(); + FieldCache.DocTerms fromDocTerms; + + SV(String field, ScoreMode scoreMode) { + super(field, scoreMode); + } + + public void collect(int doc) throws IOException { + int ord = collectedTerms.add(fromDocTerms.getTerm(doc, spare)); + if (ord < 0) { + ord = -ord - 1; + } else { + if (ord >= scoreSums.length) { + scoreSums = ArrayUtil.grow(scoreSums); + } + } + + float current = scorer.score(); + float existing = scoreSums[ord]; + if (Float.compare(existing, 0.0f) == 0) { + scoreSums[ord] = current; + } else { + switch (scoreMode) { + case Total: + scoreSums[ord] = scoreSums[ord] + current; + break; + case Max: + if (current > existing) { + scoreSums[ord] = current; + } + } + } + } + + public void setNextReader(AtomicReaderContext context) throws IOException { + fromDocTerms = FieldCache.DEFAULT.getTerms(context.reader(), field); + } + + static class Avg extends SV { + + int[] scoreCounts = new int[INITIAL_ARRAY_SIZE]; + + Avg(String field) { + super(field, ScoreMode.Avg); + } + + @Override + public void collect(int doc) throws IOException { + int ord = collectedTerms.add(fromDocTerms.getTerm(doc, spare)); + if (ord < 0) { + ord = -ord - 1; + } else { + if (ord >= scoreSums.length) { + scoreSums = ArrayUtil.grow(scoreSums); + scoreCounts = ArrayUtil.grow(scoreCounts); + } + } + + float current = scorer.score(); + float existing = scoreSums[ord]; + if (Float.compare(existing, 0.0f) == 0) { + scoreSums[ord] = current; + scoreCounts[ord] = 1; + } else { + scoreSums[ord] = scoreSums[ord] + current; + scoreCounts[ord]++; + } + } + + @Override + public float[] getScoresPerTerm() { + if (scoreCounts != null) { + for (int i = 0; i < scoreCounts.length; i++) { + scoreSums[i] = scoreSums[i] / scoreCounts[i]; + } + scoreCounts = null; + } + return scoreSums; + } + } + } + + // impl that works with multiple values per document + static class MV extends TermsWithScoreCollector { + + DocTermOrds fromDocTermOrds; + TermsEnum docTermsEnum; + DocTermOrds.TermOrdsIterator reuse; + + MV(String field, ScoreMode scoreMode) { + super(field, scoreMode); + } + + public void collect(int doc) throws IOException { + reuse = fromDocTermOrds.lookup(doc, reuse); + int[] buffer = new int[5]; + + int chunk; + do { + chunk = reuse.read(buffer); + if (chunk == 0) { + return; + } + + for (int idx = 0; idx < chunk; idx++) { + int key = buffer[idx]; + docTermsEnum.seekExact((long) key); + int ord = collectedTerms.add(docTermsEnum.term()); + if (ord < 0) { + ord = -ord - 1; + } else { + if (ord >= scoreSums.length) { + scoreSums = ArrayUtil.grow(scoreSums); + } + } + + final float current = scorer.score(); + final float existing = scoreSums[ord]; + if (Float.compare(existing, 0.0f) == 0) { + scoreSums[ord] = current; + } else { + switch (scoreMode) { + case Total: + scoreSums[ord] = existing + current; + break; + case Max: + if (current > existing) { + scoreSums[ord] = current; + } + } + } + } + } while (chunk >= buffer.length); + } + + public void setNextReader(AtomicReaderContext context) throws IOException { + fromDocTermOrds = FieldCache.DEFAULT.getDocTermOrds(context.reader(), field); + docTermsEnum = fromDocTermOrds.getOrdTermsEnum(context.reader()); + reuse = null; // LUCENE-3377 needs to be fixed first then this statement can be removed... + } + + static class Avg extends MV { + + int[] scoreCounts = new int[INITIAL_ARRAY_SIZE]; + + Avg(String field) { + super(field, ScoreMode.Avg); + } + + @Override + public void collect(int doc) throws IOException { + reuse = fromDocTermOrds.lookup(doc, reuse); + int[] buffer = new int[5]; + + int chunk; + do { + chunk = reuse.read(buffer); + if (chunk == 0) { + return; + } + + for (int idx = 0; idx < chunk; idx++) { + int key = buffer[idx]; + docTermsEnum.seekExact((long) key); + int ord = collectedTerms.add(docTermsEnum.term()); + if (ord < 0) { + ord = -ord - 1; + } else { + if (ord >= scoreSums.length) { + scoreSums = ArrayUtil.grow(scoreSums); + scoreCounts = ArrayUtil.grow(scoreCounts); + } + } + + float current = scorer.score(); + float existing = scoreSums[ord]; + if (Float.compare(existing, 0.0f) == 0) { + scoreSums[ord] = current; + scoreCounts[ord] = 1; + } else { + scoreSums[ord] = scoreSums[ord] + current; + scoreCounts[ord]++; + } + } + } while (chunk >= buffer.length); + } + + @Override + public float[] getScoresPerTerm() { + if (scoreCounts != null) { + for (int i = 0; i < scoreCounts.length; i++) { + scoreSums[i] = scoreSums[i] / scoreCounts[i]; + } + scoreCounts = null; + } + return scoreSums; + } + } + } + +} diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ToChildBlockJoinQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/ToChildBlockJoinQuery.java index d671f7a..6fb27f3 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/ToChildBlockJoinQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/ToChildBlockJoinQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.join; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java b/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java index bdbfe9c..f7144ad 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.join; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -109,6 +109,9 @@ public class ToParentBlockJoinCollector extends Collector { // only collector this.sort = sort; this.trackMaxScore = trackMaxScore; + if (trackMaxScore) { + maxScore = Float.MIN_VALUE; + } this.trackScores = trackScores; this.numParentHits = numParentHits; queue = FieldValueHitQueue.create(sort.getSort(), numParentHits); @@ -147,9 +150,7 @@ public class ToParentBlockJoinCollector extends Collector { if (trackMaxScore) { score = scorer.score(); - if (score > maxScore) { - maxScore = score; - } + maxScore = Math.max(maxScore, score); } // TODO: we could sweep all joinScorers here and @@ -203,7 +204,11 @@ public class ToParentBlockJoinCollector extends Collector { for (int i = 0; i < comparators.length; i++) { comparators[i].copy(comparatorSlot, parentDoc); } - //System.out.println(" startup: new OG doc=" + (docBase+parentDoc)); + //System.out.println(" startup: new OG doc=" + + //(docBase+parentDoc)); + if (!trackMaxScore && trackScores) { + score = scorer.score(); + } final OneGroup og = new OneGroup(comparatorSlot, docBase+parentDoc, score, joinScorers.length, trackScores); og.readerContext = currentReaderContext; copyGroups(og); @@ -437,7 +442,8 @@ public class ToParentBlockJoinCollector extends Collector { final TopDocs topDocs = collector.topDocs(withinGroupOffset, maxDocsPerGroup); - groups[groupIDX-offset] = new GroupDocs(topDocs.getMaxScore(), + groups[groupIDX-offset] = new GroupDocs(og.score, + topDocs.getMaxScore(), og.counts[slot], topDocs.scoreDocs, og.doc, @@ -446,7 +452,15 @@ public class ToParentBlockJoinCollector extends Collector { return new TopGroups(new TopGroups(sort.getSort(), withinGroupSort == null ? null : withinGroupSort.getSort(), - 0, totalGroupedHitCount, groups), + 0, totalGroupedHitCount, groups, maxScore), totalHitCount); } + + /** Returns the highest score across all collected parent + * hits, as long as trackMaxScores=true was passed {@link + * #ToParentBlockJoinCollector on construction}. Else, + * this returns Float.NaN */ + public float getMaxScore() { + return maxScore; + } } diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java index 79b1a62..82131d3 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.join; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,30 +17,22 @@ package org.apache.lucene.search.join; * limitations under the License. */ -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.Set; - import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; // javadocs +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.search.DocIdSet; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.Filter; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Scorer.ChildScorer; -import org.apache.lucene.search.Weight; +import org.apache.lucene.search.*; import org.apache.lucene.search.grouping.TopGroups; import org.apache.lucene.search.positions.PositionIntervalIterator; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; import org.apache.lucene.util.FixedBitSet; +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.Set; + /** * This query requires that you index * children and parent docs as a single block, using the @@ -83,24 +75,8 @@ import org.apache.lucene.util.FixedBitSet; * * @lucene.experimental */ - public class ToParentBlockJoinQuery extends Query { - /** How to aggregate multiple child hit scores into a - * single parent score. */ - public static enum ScoreMode { - /** Do no scoring. */ - None, - /** Parent hit's score is the average of all child - scores. */ - Avg, - /** Parent hit's score is the max of all child - scores. */ - Max, - /** Parent hit's score is the sum of all child - scores. */ - Total}; - private final Filter parentsFilter; private final Query childQuery; @@ -197,7 +173,8 @@ public class ToParentBlockJoinQuery extends Query { // acceptDocs when we score: final DocIdSet parents = parentsFilter.getDocIdSet(readerContext, null); - if (parents == null) { + if (parents == null + || parents.iterator().docID() == DocIdSetIterator.NO_MORE_DOCS) { // <-- means DocIdSet#EMPTY_DOCIDSET // No matches return null; } @@ -209,10 +186,14 @@ public class ToParentBlockJoinQuery extends Query { } @Override - public Explanation explain(AtomicReaderContext reader, int doc) throws IOException { - // TODO - throw new UnsupportedOperationException(getClass().getName() + - " cannot explain match on parent document"); + public Explanation explain(AtomicReaderContext context, int doc) throws IOException { + BlockJoinScorer scorer = (BlockJoinScorer) scorer(context, true, false, context.reader().getLiveDocs()); + if (scorer != null) { + if (scorer.advance(doc) == doc) { + return scorer.explain(context.docBase); + } + } + return new ComplexExplanation(false, 0.0f, "Not a match"); } @Override @@ -227,6 +208,7 @@ public class ToParentBlockJoinQuery extends Query { private final ScoreMode scoreMode; private final Bits acceptDocs; private int parentDoc = -1; + private int prevParentDoc; private float parentScore; private int nextChildDoc; @@ -383,7 +365,7 @@ public class ToParentBlockJoinQuery extends Query { return nextDoc(); } - final int prevParentDoc = parentBits.prevSetBit(parentTarget-1); + prevParentDoc = parentBits.prevSetBit(parentTarget-1); //System.out.println(" rolled back to prevParentDoc=" + prevParentDoc + " vs parentDoc=" + parentDoc); assert prevParentDoc >= parentDoc; @@ -407,6 +389,15 @@ public class ToParentBlockJoinQuery extends Query { //nocommit - what can we do here? throw new UnsupportedOperationException(); } + + public Explanation explain(int docBase) throws IOException { + int start = docBase + prevParentDoc + 1; // +1 b/c prevParentDoc is previous parent doc + int end = docBase + parentDoc - 1; // -1 b/c parentDoc is parent doc + return new ComplexExplanation( + true, score(), String.format("Score based on child doc range from %d to %d", start, end) + ); + } + } @Override diff --git a/lucene/join/src/java/org/apache/lucene/search/join/package.html b/lucene/join/src/java/org/apache/lucene/search/join/package.html index 5cea27a..036ef63 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/package.html +++ b/lucene/join/src/java/org/apache/lucene/search/join/package.html @@ -56,7 +56,7 @@ any query matching parent documents, creating the joined query matching only child documents. -

      Search-time joins

      +

      Query-time joins

      The query time joining is index term based and implemented as two pass search. The first pass collects all the terms from a fromField @@ -68,22 +68,26 @@

    • fromField: The from field to join from.
    • fromQuery: The query executed to collect the from terms. This is usually the user specified query.
    • multipleValuesPerDocument: Whether the fromField contains more than one value per document +
    • scoreMode: Defines how scores are translated to the other join side. If you don't care about scoring + use {@link org.apache.lucene.search.join.ScoreMode#None} mode. This will disable scoring and is therefore more + efficient (requires less memory and is faster).
    • toField: The to field to join to

      Basically the query-time joining is accessible from one static method. The user of this method supplies the method with the described input and a IndexSearcher where the from terms need to be collected from. The returned query can be executed with the same IndexSearcher, but also with another IndexSearcher. - Example usage of the {@link org.apache.lucene.search.join.JoinUtil#createJoinQuery(String, boolean, String, org.apache.lucene.search.Query, org.apache.lucene.search.IndexSearcher) + Example usage of the {@link org.apache.lucene.search.join.JoinUtil#createJoinQuery(String, boolean, String, org.apache.lucene.search.Query, org.apache.lucene.search.IndexSearcher, org.apache.lucene.search.join.ScoreMode) JoinUtil.createJoinQuery()} : 

         String fromField = "from"; // Name of the from field
   boolean multipleValuesPerDocument = false; // Set only yo true in the case when your fromField has multiple values per document in your index
   String toField = "to"; // Name of the to field
+  ScoreMode scoreMode = ScoreMode.Max // Defines how the scores are translated into the other side of the join.
   Query fromQuery = new TermQuery(new Term("content", searchTerm)); // Query executed to collect from values to join to the to values
 
-  Query joinQuery = JoinUtil.createJoinQuery(fromField, multipleValuesPerDocument, toField, fromQuery, fromSearcher);
+  Query joinQuery = JoinUtil.createJoinQuery(fromField, multipleValuesPerDocument, toField, fromQuery, fromSearcher, scoreMode);
   TopDocs topDocs = toSearcher.search(joinQuery, 10); // Note: toSearcher can be the same as the fromSearcher
   // Render topDocs...
      diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java index 34f7b65..b9e6aa8 100644 --- a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java +++ b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.join; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -38,17 +38,17 @@ public class TestBlockJoin extends LuceneTestCase { // One resume... private Document makeResume(String name, String country) { Document resume = new Document(); - resume.add(newField("docType", "resume", StringField.TYPE_UNSTORED)); - resume.add(newField("name", name, StringField.TYPE_STORED)); - resume.add(newField("country", country, StringField.TYPE_UNSTORED)); + resume.add(newStringField("docType", "resume", Field.Store.NO)); + resume.add(newStringField("name", name, Field.Store.YES)); + resume.add(newStringField("country", country, Field.Store.NO)); return resume; } // ... has multiple jobs private Document makeJob(String skill, int year) { Document job = new Document(); - job.add(newField("skill", skill, StringField.TYPE_STORED)); - job.add(new IntField("year", year)); + job.add(newStringField("skill", skill, Field.Store.YES)); + job.add(new IntField("year", year, Field.Store.NO)); job.add(new StoredField("year", year)); return job; } @@ -56,8 +56,8 @@ public class TestBlockJoin extends LuceneTestCase { // ... has multiple qualifications private Document makeQualification(String qualification, int year) { Document job = new Document(); - job.add(newField("qualification", qualification, StringField.TYPE_STORED)); - job.add(new IntField("year", year)); + job.add(newStringField("qualification", qualification, Field.Store.YES)); + job.add(new IntField("year", year, Field.Store.NO)); return job; } @@ -96,18 +96,19 @@ public class TestBlockJoin extends LuceneTestCase { // Wrap the child document query to 'join' any matches // up to corresponding parent: - ToParentBlockJoinQuery childJoinQuery = new ToParentBlockJoinQuery(childQuery, parentsFilter, ToParentBlockJoinQuery.ScoreMode.Avg); + ToParentBlockJoinQuery childJoinQuery = new ToParentBlockJoinQuery(childQuery, parentsFilter, ScoreMode.Avg); // Combine the parent and nested child queries into a single query for a candidate BooleanQuery fullQuery = new BooleanQuery(); fullQuery.add(new BooleanClause(parentQuery, Occur.MUST)); fullQuery.add(new BooleanClause(childJoinQuery, Occur.MUST)); - ToParentBlockJoinCollector c = new ToParentBlockJoinCollector(Sort.RELEVANCE, 1, true, false); + ToParentBlockJoinCollector c = new ToParentBlockJoinCollector(Sort.RELEVANCE, 1, true, true); s.search(fullQuery, c); TopGroups results = c.getTopGroups(childJoinQuery, null, 0, 10, 0, true); + assertFalse(Float.isNaN(results.maxScore)); //assertEquals(1, results.totalHitCount); assertEquals(1, results.totalGroupedHitCount); @@ -115,6 +116,7 @@ public class TestBlockJoin extends LuceneTestCase { final GroupDocs group = results.groups[0]; assertEquals(1, group.totalHits); + assertFalse(Float.isNaN(group.score)); Document childDoc = s.doc(group.scoreDocs[0].doc); //System.out.println(" doc=" + group.scoreDocs[0].doc); @@ -198,7 +200,7 @@ public class TestBlockJoin extends LuceneTestCase { // Wrap the child document query to 'join' any matches // up to corresponding parent: - ToParentBlockJoinQuery childJoinQuery = new ToParentBlockJoinQuery(childQuery, parentsFilter, ToParentBlockJoinQuery.ScoreMode.Avg); + ToParentBlockJoinQuery childJoinQuery = new ToParentBlockJoinQuery(childQuery, parentsFilter, ScoreMode.Avg); assertEquals("no filter - both passed", 2, s.search(childJoinQuery, 10).totalHits); @@ -245,9 +247,9 @@ public class TestBlockJoin extends LuceneTestCase { } private Document getParentDoc(IndexReader reader, Filter parents, int childDocID) throws IOException { - final AtomicReaderContext[] leaves = reader.getTopReaderContext().leaves(); + final List leaves = reader.getTopReaderContext().leaves(); final int subIndex = ReaderUtil.subIndex(childDocID, leaves); - final AtomicReaderContext leaf = leaves[subIndex]; + final AtomicReaderContext leaf = leaves.get(subIndex); final FixedBitSet bits = (FixedBitSet) parents.getDocIdSet(leaf, null); return leaf.reader().document(bits.nextSetBit(childDocID - leaf.docBase)); } @@ -259,7 +261,7 @@ public class TestBlockJoin extends LuceneTestCase { w.close(); IndexSearcher s = newSearcher(r); - ToParentBlockJoinQuery q = new ToParentBlockJoinQuery(new MatchAllDocsQuery(), new QueryWrapperFilter(new MatchAllDocsQuery()), ToParentBlockJoinQuery.ScoreMode.Avg); + ToParentBlockJoinQuery q = new ToParentBlockJoinQuery(new MatchAllDocsQuery(), new QueryWrapperFilter(new MatchAllDocsQuery()), ScoreMode.Avg); s.search(q, 10); BooleanQuery bq = new BooleanQuery(); bq.setBoost(2f); // we boost the BQ @@ -339,23 +341,21 @@ public class TestBlockJoin extends LuceneTestCase { for(int parentDocID=0;parentDocID joinDocs = new ArrayList(); @@ -379,15 +379,13 @@ public class TestBlockJoin extends LuceneTestCase { Document joinChildDoc = new Document(); joinDocs.add(joinChildDoc); - Field childID = newField("childID", ""+childDocID, StringField.TYPE_STORED); + Field childID = newStringField("childID", ""+childDocID, Field.Store.YES); childDoc.add(childID); joinChildDoc.add(childID); for(int childFieldID=0;childFieldID()); @@ -260,9 +400,9 @@ public class TestJoinUtil extends LuceneTestCase { context.randomValueFromDocs.put(value, new ArrayList()); } - context.fromDocuments.get(linkValue).add(doc); - context.randomValueFromDocs.get(value).add(doc); - document.add(newField(random(), "from", linkValue, TextField.TYPE_STORED)); + context.fromDocuments.get(linkValue).add(docs[i]); + context.randomValueFromDocs.get(value).add(docs[i]); + document.add(newTextField(random(), "from", linkValue, Field.Store.NO)); } else { if (!context.toDocuments.containsKey(linkValue)) { context.toDocuments.put(linkValue, new ArrayList()); @@ -271,9 +411,9 @@ public class TestJoinUtil extends LuceneTestCase { context.randomValueToDocs.put(value, new ArrayList()); } - context.toDocuments.get(linkValue).add(doc); - context.randomValueToDocs.get(value).add(doc); - document.add(newField(random(), "to", linkValue, TextField.TYPE_STORED)); + context.toDocuments.get(linkValue).add(docs[i]); + context.randomValueToDocs.get(value).add(docs[i]); + document.add(newTextField(random(), "to", linkValue, Field.Store.NO)); } } @@ -289,12 +429,235 @@ public class TestJoinUtil extends LuceneTestCase { w.commit(); } if (VERBOSE) { - System.out.println("Added document[" + i + "]: " + document); + System.out.println("Added document[" + docs[i].id + "]: " + document); } } + + // Pre-compute all possible hits for all unique random values. On top of this also compute all possible score for + // any ScoreMode. + IndexSearcher fromSearcher = newSearcher(fromWriter.getReader()); + IndexSearcher toSearcher = newSearcher(toWriter.getReader()); + for (int i = 0; i < context.randomUniqueValues.length; i++) { + String uniqueRandomValue = context.randomUniqueValues[i]; + final String fromField; + final String toField; + final Map> queryVals; + if (context.randomFrom[i]) { + fromField = "from"; + toField = "to"; + queryVals = context.fromHitsToJoinScore; + } else { + fromField = "to"; + toField = "from"; + queryVals = context.toHitsToJoinScore; + } + final Map joinValueToJoinScores = new HashMap(); + if (multipleValuesPerDocument) { + fromSearcher.search(new TermQuery(new Term("value", uniqueRandomValue)), new Collector() { + + private Scorer scorer; + private DocTermOrds docTermOrds; + private TermsEnum docTermsEnum; + private DocTermOrds.TermOrdsIterator reuse; + + public void collect(int doc) throws IOException { + if (docTermOrds.isEmpty()) { + return; + } + + reuse = docTermOrds.lookup(doc, reuse); + int[] buffer = new int[5]; + + int chunk; + do { + chunk = reuse.read(buffer); + if (chunk == 0) { + return; + } + + for (int idx = 0; idx < chunk; idx++) { + int key = buffer[idx]; + docTermsEnum.seekExact((long) key); + BytesRef joinValue = docTermsEnum.term(); + if (joinValue == null) { + continue; + } + + JoinScore joinScore = joinValueToJoinScores.get(joinValue); + if (joinScore == null) { + joinValueToJoinScores.put(BytesRef.deepCopyOf(joinValue), joinScore = new JoinScore()); + } + joinScore.addScore(scorer.score()); + } + } while (chunk >= buffer.length); + } + + public void setNextReader(AtomicReaderContext context) throws IOException { + docTermOrds = FieldCache.DEFAULT.getDocTermOrds(context.reader(), fromField); + docTermsEnum = docTermOrds.getOrdTermsEnum(context.reader()); + reuse = null; + } + + public void setScorer(Scorer scorer) { + this.scorer = scorer; + } + + public boolean acceptsDocsOutOfOrder() { + return false; + } + }); + } else { + fromSearcher.search(new TermQuery(new Term("value", uniqueRandomValue)), new Collector() { + + private Scorer scorer; + private FieldCache.DocTerms terms; + private final BytesRef spare = new BytesRef(); + + public void collect(int doc) throws IOException { + BytesRef joinValue = terms.getTerm(doc, spare); + if (joinValue == null) { + return; + } + + JoinScore joinScore = joinValueToJoinScores.get(joinValue); + if (joinScore == null) { + joinValueToJoinScores.put(BytesRef.deepCopyOf(joinValue), joinScore = new JoinScore()); + } + joinScore.addScore(scorer.score()); + } + + public void setNextReader(AtomicReaderContext context) throws IOException { + terms = FieldCache.DEFAULT.getTerms(context.reader(), fromField); + } + + public void setScorer(Scorer scorer) { + this.scorer = scorer; + } + + public boolean acceptsDocsOutOfOrder() { + return false; + } + }); + } + + final Map docToJoinScore = new HashMap(); + if (multipleValuesPerDocument) { + toSearcher.search(new MatchAllDocsQuery(), new Collector() { + + private DocTermOrds docTermOrds; + private TermsEnum docTermsEnum; + private DocTermOrds.TermOrdsIterator reuse; + private int docBase; + + public void collect(int doc) throws IOException { + if (docTermOrds.isEmpty()) { + return; + } + + reuse = docTermOrds.lookup(doc, reuse); + int[] buffer = new int[5]; + + int chunk; + do { + chunk = reuse.read(buffer); + if (chunk == 0) { + return; + } + + for (int idx = 0; idx < chunk; idx++) { + int key = buffer[idx]; + docTermsEnum.seekExact((long) key); + JoinScore joinScore = joinValueToJoinScores.get(docTermsEnum.term()); + if (joinScore == null) { + continue; + } + Integer basedDoc = docBase + doc; + // First encountered join value determines the score. + // Something to keep in mind for many-to-many relations. + if (!docToJoinScore.containsKey(basedDoc)) { + docToJoinScore.put(basedDoc, joinScore); + } + } + } while (chunk >= buffer.length); + } + + public void setNextReader(AtomicReaderContext context) throws IOException { + docBase = context.docBase; + docTermOrds = FieldCache.DEFAULT.getDocTermOrds(context.reader(), toField); + docTermsEnum = docTermOrds.getOrdTermsEnum(context.reader()); + reuse = null; + } + + public boolean acceptsDocsOutOfOrder() {return false;} + public void setScorer(Scorer scorer) {} + }); + } else { + toSearcher.search(new MatchAllDocsQuery(), new Collector() { + + private FieldCache.DocTerms terms; + private int docBase; + private final BytesRef spare = new BytesRef(); + + public void collect(int doc) { + JoinScore joinScore = joinValueToJoinScores.get(terms.getTerm(doc, spare)); + if (joinScore == null) { + return; + } + docToJoinScore.put(docBase + doc, joinScore); + } + + public void setNextReader(AtomicReaderContext context) throws IOException { + terms = FieldCache.DEFAULT.getTerms(context.reader(), toField); + docBase = context.docBase; + } + + public boolean acceptsDocsOutOfOrder() {return false;} + public void setScorer(Scorer scorer) {} + }); + } + queryVals.put(uniqueRandomValue, docToJoinScore); + } + + fromSearcher.getIndexReader().close(); + toSearcher.getIndexReader().close(); + return context; } + private TopDocs createExpectedTopDocs(String queryValue, + final boolean from, + final ScoreMode scoreMode, + IndexIterationContext context) { + + Map hitsToJoinScores; + if (from) { + hitsToJoinScores = context.fromHitsToJoinScore.get(queryValue); + } else { + hitsToJoinScores = context.toHitsToJoinScore.get(queryValue); + } + List> hits = new ArrayList>(hitsToJoinScores.entrySet()); + Collections.sort(hits, new Comparator>() { + + public int compare(Map.Entry hit1, Map.Entry hit2) { + float score1 = hit1.getValue().score(scoreMode); + float score2 = hit2.getValue().score(scoreMode); + + int cmp = Float.compare(score2, score1); + if (cmp != 0) { + return cmp; + } + return hit1.getKey() - hit2.getKey(); + } + + }); + ScoreDoc[] scoreDocs = new ScoreDoc[Math.min(10, hits.size())]; + for (int i = 0; i < scoreDocs.length; i++) { + Map.Entry hit = hits.get(i); + scoreDocs[i] = new ScoreDoc(hit.getKey(), hit.getValue().score(scoreMode)); + } + return new TopDocs(hits.size(), scoreDocs, hits.isEmpty() ? Float.NaN : hits.get(0).getValue().score(scoreMode)); + } + private FixedBitSet createExpectedResult(String queryValue, boolean from, IndexReader topLevelReader, IndexIterationContext context) throws IOException { final Map> randomValueDocs; final Map> linkValueDocuments; @@ -339,6 +702,9 @@ public class TestJoinUtil extends LuceneTestCase { Map> randomValueFromDocs = new HashMap>(); Map> randomValueToDocs = new HashMap>(); + Map> fromHitsToJoinScore = new HashMap>(); + Map> toHitsToJoinScore = new HashMap>(); + } private static class RandomDoc { @@ -346,12 +712,44 @@ public class TestJoinUtil extends LuceneTestCase { final String id; final List linkValues; final String value; + final boolean from; - private RandomDoc(String id, int numberOfLinkValues, String value) { + private RandomDoc(String id, int numberOfLinkValues, String value, boolean from) { this.id = id; + this.from = from; linkValues = new ArrayList(numberOfLinkValues); this.value = value; } } + private static class JoinScore { + + float maxScore; + float total; + int count; + + void addScore(float score) { + total += score; + if (score > maxScore) { + maxScore = score; + } + count++; + } + + float score(ScoreMode mode) { + switch (mode) { + case None: + return 1.0f; + case Total: + return total; + case Avg: + return total / count; + case Max: + return maxScore; + } + throw new IllegalArgumentException("Unsupported ScoreMode: " + mode); + } + + } + } diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java index 3f8f8ba..cf1724b 100644 --- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java +++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java @@ -1,6 +1,6 @@ package org.apache.lucene.index.memory; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -32,7 +32,6 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; -import org.apache.lucene.document.Document; import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.FieldInfo; @@ -443,7 +442,7 @@ public class MemoryIndex { } @Override - public void setScorer(Scorer scorer) throws IOException { + public void setScorer(Scorer scorer) { this.scorer = scorer; } @@ -770,13 +769,13 @@ public class MemoryIndex { } @Override - public long getSumDocFreq() throws IOException { + public long getSumDocFreq() { // each term has df=1 return info.sortedTerms.length; } @Override - public int getDocCount() throws IOException { + public int getDocCount() { return info.sortedTerms.length > 0 ? 1 : 0; } @@ -1065,7 +1064,7 @@ public class MemoryIndex { } @Override - public DocValues docValues(String field) throws IOException { + public DocValues docValues(String field) { return null; } @@ -1075,7 +1074,7 @@ public class MemoryIndex { private Similarity cachedSimilarity; @Override - public DocValues normValues(String field) throws IOException { + public DocValues normValues(String field) { DocValues norms = cachedNormValues; Similarity sim = getSimilarity(); if (!field.equals(cachedFieldName) || sim != cachedSimilarity) { // not cached? diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndexNormDocValues.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndexNormDocValues.java index 36a3f9d..6a69512 100644 --- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndexNormDocValues.java +++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndexNormDocValues.java @@ -1,5 +1,5 @@ package org.apache.lucene.index.memory; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java b/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java index a0ceaca..f67c684 100644 --- a/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java +++ b/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.index.memory; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -33,7 +33,6 @@ import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.TextField; import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocsAndPositionsEnum; @@ -126,8 +125,8 @@ public class MemoryIndexTest extends BaseTokenStreamTestCase { IndexWriter writer = new IndexWriter(ramdir, new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()))); Document doc = new Document(); - Field field1 = newField("foo", fooField.toString(), TextField.TYPE_UNSTORED); - Field field2 = newField("term", termField.toString(), TextField.TYPE_UNSTORED); + Field field1 = newTextField("foo", fooField.toString(), Field.Store.NO); + Field field2 = newTextField("term", termField.toString(), Field.Store.NO); doc.add(field1); doc.add(field2); writer.addDocument(doc); diff --git a/lucene/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java b/lucene/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java index 91177e0..95d234a 100644 --- a/lucene/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java +++ b/lucene/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -245,7 +245,7 @@ public class BalancedSegmentMergePolicy extends LogByteSizeMergePolicy { @Override public MergeSpecification findForcedDeletesMerges(SegmentInfos infos) - throws CorruptIndexException, IOException { + throws IOException { final int numSegs = infos.size(); final int numLargeSegs = (numSegs < _numLargeSegments ? numSegs : _numLargeSegments); MergeSpecification spec = null; @@ -335,7 +335,7 @@ public class BalancedSegmentMergePolicy extends LogByteSizeMergePolicy { } } - private OneMerge findOneSegmentToExpunge(SegmentInfos infos, int maxNumSegments) throws IOException { + private OneMerge findOneSegmentToExpunge(SegmentInfos infos, int maxNumSegments) { int expungeCandidate = -1; int maxDelCount = 0; diff --git a/lucene/misc/src/java/org/apache/lucene/index/CompoundFileExtractor.java b/lucene/misc/src/java/org/apache/lucene/index/CompoundFileExtractor.java index d8ef54a..2b80764 100644 --- a/lucene/misc/src/java/org/apache/lucene/index/CompoundFileExtractor.java +++ b/lucene/misc/src/java/org/apache/lucene/index/CompoundFileExtractor.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java b/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java index 1127e18..31fa60d 100644 --- a/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java +++ b/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -27,7 +27,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; -import org.apache.lucene.index.IndexWriter; // Required for javadocs import org.apache.lucene.store.FSDirectory; /** @@ -46,11 +45,6 @@ import org.apache.lucene.store.FSDirectory; * @lucene.experimental You can easily * accidentally remove segments from your index so be * careful! - * - *

      NOTE: this tool is unaware of documents added - * atomically via {@link IndexWriter#addDocuments} or {@link - * IndexWriter#updateDocuments}, which means it can easily - * break up such document groups. */ public class IndexSplitter { public SegmentInfos infos; diff --git a/lucene/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java b/lucene/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java index d15d417..2fc612b 100644 --- a/lucene/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java +++ b/lucene/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,13 +20,13 @@ package org.apache.lucene.index; import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.List; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.ReaderUtil; import org.apache.lucene.util.Version; /** @@ -101,7 +101,8 @@ public class MultiPassIndexSplitter { .setOpenMode(OpenMode.CREATE)); System.err.println("Writing part " + (i + 1) + " ..."); // pass the subreaders directly, as our wrapper's numDocs/hasDeletetions are not up-to-date - w.addIndexes(input.getSequentialSubReaders()); + final List sr = input.getSequentialSubReaders(); + w.addIndexes(sr.toArray(new IndexReader[sr.size()])); // TODO: maybe take List here? w.close(); } System.err.println("Done."); @@ -177,34 +178,36 @@ public class MultiPassIndexSplitter { /** * This class emulates deletions on the underlying index. */ - private static final class FakeDeleteIndexReader extends MultiReader { + private static final class FakeDeleteIndexReader extends BaseCompositeReader { - public FakeDeleteIndexReader(IndexReader reader) throws IOException { + public FakeDeleteIndexReader(IndexReader reader) { super(initSubReaders(reader)); } - private static AtomicReader[] initSubReaders(IndexReader reader) throws IOException { - final ArrayList subs = new ArrayList(); - new ReaderUtil.Gather(reader) { - @Override - protected void add(int base, AtomicReader r) { - subs.add(new FakeDeleteAtomicIndexReader(r)); - } - }.run(); - return subs.toArray(new AtomicReader[subs.size()]); + private static FakeDeleteAtomicIndexReader[] initSubReaders(IndexReader reader) { + final List leaves = reader.getTopReaderContext().leaves(); + final FakeDeleteAtomicIndexReader[] subs = new FakeDeleteAtomicIndexReader[leaves.size()]; + int i = 0; + for (final AtomicReaderContext ctx : leaves) { + subs[i++] = new FakeDeleteAtomicIndexReader(ctx.reader()); + } + return subs; } public void deleteDocument(int docID) { final int i = readerIndex(docID); - ((FakeDeleteAtomicIndexReader) subReaders[i]).deleteDocument(docID - starts[i]); + getSequentialSubReaders().get(i).deleteDocument(docID - readerBase(i)); } public void undeleteAll() { - for (IndexReader r : subReaders) { - ((FakeDeleteAtomicIndexReader) r).undeleteAll(); + for (FakeDeleteAtomicIndexReader r : getSequentialSubReaders()) { + r.undeleteAll(); } } + @Override + protected void doClose() {} + // no need to override numDocs/hasDeletions, // as we pass the subreaders directly to IW.addIndexes(). } diff --git a/lucene/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java b/lucene/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java index b89d275..9044004 100644 --- a/lucene/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java +++ b/lucene/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,6 +18,7 @@ package org.apache.lucene.index; */ import java.io.IOException; +import java.util.List; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.search.DocIdSet; @@ -101,10 +102,11 @@ public class PKIndexSplitter { boolean success = false; final IndexWriter w = new IndexWriter(target, config); try { - final AtomicReaderContext[] leaves = reader.getTopReaderContext().leaves(); - final IndexReader[] subReaders = new IndexReader[leaves.length]; - for (int i = 0; i < leaves.length; i++) { - subReaders[i] = new DocumentFilteredAtomicIndexReader(leaves[i], preserveFilter, negateFilter); + final List leaves = reader.getTopReaderContext().leaves(); + final IndexReader[] subReaders = new IndexReader[leaves.size()]; + int i = 0; + for (final AtomicReaderContext ctx : leaves) { + subReaders[i++] = new DocumentFilteredAtomicIndexReader(ctx, preserveFilter, negateFilter); } w.addIndexes(subReaders); success = true; diff --git a/lucene/misc/src/java/org/apache/lucene/misc/GetTermInfo.java b/lucene/misc/src/java/org/apache/lucene/misc/GetTermInfo.java index 45ab8d5..8aadc8d 100644 --- a/lucene/misc/src/java/org/apache/lucene/misc/GetTermInfo.java +++ b/lucene/misc/src/java/org/apache/lucene/misc/GetTermInfo.java @@ -1,6 +1,6 @@ package org.apache.lucene.misc; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,6 +21,7 @@ import java.io.File; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; /** @@ -47,7 +48,7 @@ public class GetTermInfo { } public static void getTermInfo(Directory dir, String field, BytesRef termtext) throws Exception { - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); long totalTF = HighFreqTerms.getTotalTermFreq(reader, field, termtext); System.out.printf("%s:%s \t totalTF = %,d \t doc freq = %,d \n", field, termtext.utf8ToString(), totalTF, reader.docFreq(field, termtext)); diff --git a/lucene/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java b/lucene/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java index 52faf49..7073fee 100644 --- a/lucene/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java +++ b/lucene/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java @@ -1,6 +1,6 @@ package org.apache.lucene.misc; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,9 +18,12 @@ package org.apache.lucene.misc; */ import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.Fields; +import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.FieldsEnum; import org.apache.lucene.index.Terms; @@ -30,7 +33,6 @@ import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.PriorityQueue; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.ReaderUtil; import java.io.File; import java.io.IOException; @@ -78,7 +80,7 @@ public class HighFreqTerms { } } - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); TermStats[] terms = getHighFreqTerms(reader, numTerms, field); if (!IncludeTermFreqs) { //default HighFreqTerms behavior @@ -183,33 +185,29 @@ public class HighFreqTerms { } public static long getTotalTermFreq(IndexReader reader, final String field, final BytesRef termText) throws Exception { - final long totalTF[] = new long[1]; - - new ReaderUtil.Gather(reader) { - - @Override - protected void add(int base, AtomicReader r) throws IOException { - Bits liveDocs = r.getLiveDocs(); - if (liveDocs == null) { - // TODO: we could do this up front, during the scan - // (next()), instead of after-the-fact here w/ seek, - // if the codec supports it and there are no del - // docs... - final long totTF = r.totalTermFreq(field, termText); - if (totTF != -1) { - totalTF[0] += totTF; - return; - } - } - DocsEnum de = r.termDocsEnum(liveDocs, field, termText, true); - if (de != null) { - while (de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) - totalTF[0] += de.freq(); - } + long totalTF = 0L; + for (final AtomicReaderContext ctx : reader.getTopReaderContext().leaves()) { + AtomicReader r = ctx.reader(); + Bits liveDocs = r.getLiveDocs(); + if (liveDocs == null) { + // TODO: we could do this up front, during the scan + // (next()), instead of after-the-fact here w/ seek, + // if the codec supports it and there are no del + // docs... + final long totTF = r.totalTermFreq(field, termText); + if (totTF != -1) { + totalTF += totTF; + continue; + } // otherwise we fall-through } - }.run(); + DocsEnum de = r.termDocsEnum(liveDocs, field, termText, true); + if (de != null) { + while (de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) + totalTF += de.freq(); + } + } - return totalTF[0]; + return totalTF; } } diff --git a/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java b/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java index b52dcba..3224416 100644 --- a/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java +++ b/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/misc/src/java/org/apache/lucene/misc/TermStats.java b/lucene/misc/src/java/org/apache/lucene/misc/TermStats.java index 22de9b6..b7b4311 100644 --- a/lucene/misc/src/java/org/apache/lucene/misc/TermStats.java +++ b/lucene/misc/src/java/org/apache/lucene/misc/TermStats.java @@ -1,6 +1,6 @@ package org.apache.lucene.misc; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/misc/src/java/org/apache/lucene/store/NativePosixUtil.java b/lucene/misc/src/java/org/apache/lucene/store/NativePosixUtil.java index 7c78e85..0618318 100644 --- a/lucene/misc/src/java/org/apache/lucene/store/NativePosixUtil.java +++ b/lucene/misc/src/java/org/apache/lucene/store/NativePosixUtil.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java b/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java index c34364f..3b8ea42 100644 --- a/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java +++ b/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF @@ -199,7 +199,7 @@ public class NativeUnixDirectory extends FSDirectory { //} @Override - public void flush() throws IOException { + public void flush() { // TODO -- I don't think this method is necessary? } @@ -257,7 +257,7 @@ public class NativeUnixDirectory extends FSDirectory { } @Override - public long length() throws IOException { + public long length() { return fileLength + bufferPos; } diff --git a/lucene/misc/src/java/org/apache/lucene/store/WindowsDirectory.java b/lucene/misc/src/java/org/apache/lucene/store/WindowsDirectory.java index 1941434..3ad31f5 100644 --- a/lucene/misc/src/java/org/apache/lucene/store/WindowsDirectory.java +++ b/lucene/misc/src/java/org/apache/lucene/store/WindowsDirectory.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/lucene/misc/src/test/org/apache/lucene/index/TestBalancedSegmentMergePolicy.java b/lucene/misc/src/test/org/apache/lucene/index/TestBalancedSegmentMergePolicy.java index 11444bc..8db6164 100644 --- a/lucene/misc/src/test/org/apache/lucene/index/TestBalancedSegmentMergePolicy.java +++ b/lucene/misc/src/test/org/apache/lucene/index/TestBalancedSegmentMergePolicy.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -44,7 +44,7 @@ public class TestBalancedSegmentMergePolicy extends LuceneTestCase { mp.setMergePolicyParams(newMergePolicyParams(random())); iwc.setMergePolicy(mp); iw = new RandomIndexWriter(random(), dir, iwc); - LineFileDocs docs = new LineFileDocs(random(), !Codec.getDefault().getName().equals("Lucene3x")); + LineFileDocs docs = new LineFileDocs(random(), true); int numDocs = atLeast(200); for (int i = 0; i < numDocs; i++) { iw.addDocument(docs.nextDoc()); @@ -67,7 +67,7 @@ public class TestBalancedSegmentMergePolicy extends LuceneTestCase { int numSegments = _TestUtil.nextInt(random(), 1, 4); iw.forceMerge(numSegments); DirectoryReader ir = iw.getReader(); - assertTrue(ir.getSequentialSubReaders().length <= numSegments); + assertTrue(ir.getSequentialSubReaders().size() <= numSegments); ir.close(); } diff --git a/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java b/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java index d711756..f3502dc 100644 --- a/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java +++ b/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -59,7 +59,7 @@ public class TestIndexSplitter extends LuceneTestCase { } iw.commit(); DirectoryReader iwReader = iw.getReader(); - assertEquals(3, iwReader.getSequentialSubReaders().length); + assertEquals(3, iwReader.getSequentialSubReaders().size()); iwReader.close(); iw.close(); // we should have 2 segments now @@ -87,7 +87,7 @@ public class TestIndexSplitter extends LuceneTestCase { // now remove the copied segment from src IndexSplitter.main(new String[] {dir.getAbsolutePath(), "-d", splitSegName}); r = DirectoryReader.open(fsDir); - assertEquals(2, r.getSequentialSubReaders().length); + assertEquals(2, r.getSequentialSubReaders().size()); r.close(); fsDir.close(); } diff --git a/lucene/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java b/lucene/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java index a6837ac..7f15357 100644 --- a/lucene/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java +++ b/lucene/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java @@ -1,5 +1,5 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,8 +18,7 @@ package org.apache.lucene.index; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; @@ -37,15 +36,15 @@ public class TestMultiPassIndexSplitter extends LuceneTestCase { Document doc; for (int i = 0; i < NUM_DOCS; i++) { doc = new Document(); - doc.add(newField("id", i + "", StringField.TYPE_STORED)); - doc.add(newField("f", i + " " + i, TextField.TYPE_STORED)); + doc.add(newStringField("id", i + "", Field.Store.YES)); + doc.add(newTextField("f", i + " " + i, Field.Store.YES)); w.addDocument(doc); if (i%3==0) w.commit(); } w.commit(); w.deleteDocuments(new Term("id", "" + (NUM_DOCS-1))); w.close(); - input = IndexReader.open(dir); + input = DirectoryReader.open(dir); } @Override @@ -67,7 +66,7 @@ public class TestMultiPassIndexSplitter extends LuceneTestCase { }; splitter.split(TEST_VERSION_CURRENT, input, dirs, false); IndexReader ir; - ir = IndexReader.open(dirs[0]); + ir = DirectoryReader.open(dirs[0]); assertTrue(ir.numDocs() - NUM_DOCS / 3 <= 1); // rounding error Document doc = ir.document(0); assertEquals("0", doc.get("id")); @@ -75,7 +74,7 @@ public class TestMultiPassIndexSplitter extends LuceneTestCase { assertEquals(TermsEnum.SeekStatus.NOT_FOUND, te.seekCeil(new BytesRef("1"))); assertNotSame("1", te.term().utf8ToString()); ir.close(); - ir = IndexReader.open(dirs[1]); + ir = DirectoryReader.open(dirs[1]); assertTrue(ir.numDocs() - NUM_DOCS / 3 <= 1); doc = ir.document(0); assertEquals("1", doc.get("id")); @@ -84,7 +83,7 @@ public class TestMultiPassIndexSplitter extends LuceneTestCase { assertNotSame("0", te.term().utf8ToString()); ir.close(); - ir = IndexReader.open(dirs[2]); + ir = DirectoryReader.open(dirs[2]); assertTrue(ir.numDocs() - NUM_DOCS / 3 <= 1); doc = ir.document(0); assertEquals("2", doc.get("id")); @@ -112,19 +111,19 @@ public class TestMultiPassIndexSplitter extends LuceneTestCase { }; splitter.split(TEST_VERSION_CURRENT, input, dirs, true); IndexReader ir; - ir = IndexReader.open(dirs[0]); + ir = DirectoryReader.open(dirs[0]); assertTrue(ir.numDocs() - NUM_DOCS / 3 <= 1); Document doc = ir.document(0); assertEquals("0", doc.get("id")); int start = ir.numDocs(); ir.close(); - ir = IndexReader.open(dirs[1]); + ir = DirectoryReader.open(dirs[1]); assertTrue(ir.numDocs() - NUM_DOCS / 3 <= 1); doc = ir.document(0); assertEquals(start + "", doc.get("id")); start += ir.numDocs(); ir.close(); - ir = IndexReader.open(dirs[2]); + ir = DirectoryReader.open(dirs[2]); assertTrue(ir.numDocs() - NUM_DOCS / 3 <= 1); doc = ir.document(0); assertEquals(start + "", doc.get("id")); diff --git a/lucene/misc/src/test/org/apache/lucene/index/TestPKIndexSplitter.java b/lucene/misc/src/test/org/apache/lucene/index/TestPKIndexSplitter.java index ffdf51c..1f0e68b 100644 --- a/lucene/misc/src/test/org/apache/lucene/index/TestPKIndexSplitter.java +++ b/lucene/misc/src/test/org/apache/lucene/index/TestPKIndexSplitter.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF @@ -23,8 +23,7 @@ import java.text.NumberFormat; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; @@ -75,8 +74,8 @@ public class TestPKIndexSplitter extends LuceneTestCase { newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); splitter.split(); - IndexReader ir1 = IndexReader.open(dir1); - IndexReader ir2 = IndexReader.open(dir2); + IndexReader ir1 = DirectoryReader.open(dir1); + IndexReader ir2 = DirectoryReader.open(dir2); assertEquals(leftCount, ir1.numDocs()); assertEquals(rightCount, ir2.numDocs()); @@ -104,15 +103,15 @@ public class TestPKIndexSplitter extends LuceneTestCase { StringBuilder sb = new StringBuilder(); Document doc = new Document(); String id = format.format(n); - doc.add(newField("id", id, StringField.TYPE_STORED)); - doc.add(newField("indexname", indexName, StringField.TYPE_STORED)); + doc.add(newStringField("id", id, Field.Store.YES)); + doc.add(newStringField("indexname", indexName, Field.Store.YES)); sb.append("a"); sb.append(n); - doc.add(newField("field1", sb.toString(), TextField.TYPE_STORED)); + doc.add(newTextField("field1", sb.toString(), Field.Store.YES)); sb.append(" b"); sb.append(n); for (int i = 1; i < numFields; i++) { - doc.add(newField("field" + (i + 1), sb.toString(), TextField.TYPE_STORED)); + doc.add(newTextField("field" + (i + 1), sb.toString(), Field.Store.YES)); } return doc; } diff --git a/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java b/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java index c2db3d3..b59f7fc 100644 --- a/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java +++ b/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java @@ -1,5 +1,5 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java b/lucene/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java index 21090b2..e469825 100644 --- a/lucene/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java +++ b/lucene/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java @@ -1,6 +1,6 @@ package org.apache.lucene.misc; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,7 +22,8 @@ import java.util.Random; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.store.Directory; @@ -45,7 +46,7 @@ public class TestHighFreqTerms extends LuceneTestCase { TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)) .setMaxBufferedDocs(2)); indexDocs(writer); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); _TestUtil.checkIndex(dir); } @@ -205,9 +206,9 @@ public class TestHighFreqTerms extends LuceneTestCase { Document doc = new Document(); String content = getContent(i); - doc.add(newField(rnd, "FIELD_1", content, TextField.TYPE_STORED)); + doc.add(newTextField(rnd, "FIELD_1", content, Field.Store.YES)); //add a different field - doc.add(newField(rnd, "different_field", "diff", TextField.TYPE_STORED)); + doc.add(newTextField(rnd, "different_field", "diff", Field.Store.YES)); writer.addDocument(doc); } @@ -215,7 +216,7 @@ public class TestHighFreqTerms extends LuceneTestCase { //highest freq terms for a specific field. for (int i = 1; i <= 10; i++) { Document doc = new Document(); - doc.add(newField(rnd, "different_field", "diff", TextField.TYPE_STORED)); + doc.add(newTextField(rnd, "different_field", "diff", Field.Store.YES)); writer.addDocument(doc); } // add some docs where tf < df so we can see if sorting works @@ -226,7 +227,7 @@ public class TestHighFreqTerms extends LuceneTestCase { for (int i = 0; i < highTF; i++) { content += "highTF "; } - doc.add(newField(rnd, "FIELD_1", content, TextField.TYPE_STORED)); + doc.add(newTextField(rnd, "FIELD_1", content, Field.Store.YES)); writer.addDocument(doc); // highTF medium df =5 int medium_df = 5; @@ -237,7 +238,7 @@ public class TestHighFreqTerms extends LuceneTestCase { for (int j = 0; j < tf; j++) { newcontent += "highTFmedDF "; } - newdoc.add(newField(rnd, "FIELD_1", newcontent, TextField.TYPE_STORED)); + newdoc.add(newTextField(rnd, "FIELD_1", newcontent, Field.Store.YES)); writer.addDocument(newdoc); } // add a doc with high tf in field different_field @@ -247,7 +248,7 @@ public class TestHighFreqTerms extends LuceneTestCase { for (int i = 0; i < targetTF; i++) { content += "TF150 "; } - doc.add(newField(rnd, "different_field", content, TextField.TYPE_STORED)); + doc.add(newTextField(rnd, "different_field", content, Field.Store.YES)); writer.addDocument(doc); writer.close(); diff --git a/lucene/module-build.xml b/lucene/module-build.xml index 62cfd96..771016a 100644 --- a/lucene/module-build.xml +++ b/lucene/module-build.xml @@ -312,9 +312,7 @@ - - - + diff --git a/lucene/queries/build.xml b/lucene/queries/build.xml index 24e9480..5edb64d 100644 --- a/lucene/queries/build.xml +++ b/lucene/queries/build.xml @@ -20,5 +20,5 @@ Filters and Queries that add to core Lucene - + diff --git a/lucene/queries/src/java/org/apache/lucene/queries/BooleanFilter.java b/lucene/queries/src/java/org/apache/lucene/queries/BooleanFilter.java index f1ac009..2a2d00e 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/BooleanFilter.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/BooleanFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/BoostingQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/BoostingQuery.java index eb48f52..037879e 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/BoostingQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/BoostingQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -38,9 +38,9 @@ import org.apache.lucene.search.*; * and is documented here: http://wiki.apache.org/lucene-java/CommunityContributions */ public class BoostingQuery extends Query { - private float boost; // the amount to boost by - private Query match; // query to match - private Query context; // boost when matches too + private final float boost; // the amount to boost by + private final Query match; // query to match + private final Query context; // boost when matches too public BoostingQuery(Query match, Query context, float boost) { this.match = match; diff --git a/lucene/queries/src/java/org/apache/lucene/queries/ChainedFilter.java b/lucene/queries/src/java/org/apache/lucene/queries/ChainedFilter.java index 8e58a51..6f8c9a8 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/ChainedFilter.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/ChainedFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -53,7 +53,7 @@ public class ChainedFilter extends Filter { /** * Logical operation when none is declared. Defaults to OR. */ - public static int DEFAULT = OR; + public static final int DEFAULT = OR; /** * The filter chain diff --git a/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreProvider.java b/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreProvider.java index b923383..862334c 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreProvider.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreProvider.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -78,8 +78,8 @@ public class CustomScoreProvider { return customScore(doc, subQueryScore, 1); } float score = subQueryScore; - for(int i = 0; i < valSrcScores.length; i++) { - score *= valSrcScores[i]; + for (float valSrcScore : valSrcScores) { + score *= valSrcScore; } return score; } @@ -127,13 +127,13 @@ public class CustomScoreProvider { return subQueryExpl; } float valSrcScore = 1; - for (int i = 0; i < valSrcExpls.length; i++) { - valSrcScore *= valSrcExpls[i].getValue(); + for (Explanation valSrcExpl : valSrcExpls) { + valSrcScore *= valSrcExpl.getValue(); } Explanation exp = new Explanation( valSrcScore * subQueryExpl.getValue(), "custom score: product of:"); exp.addDetail(subQueryExpl); - for (int i = 0; i < valSrcExpls.length; i++) { - exp.addDetail(valSrcExpls[i]); + for (Explanation valSrcExpl : valSrcExpls) { + exp.addDetail(valSrcExpl); } return exp; } diff --git a/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java index 9d7f5dd..1d7785b 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -108,8 +108,8 @@ public class CustomScoreQuery extends Query { @Override public void extractTerms(Set terms) { subQuery.extractTerms(terms); - for(int i = 0; i < scoringQueries.length; i++) { - scoringQueries[i].extractTerms(terms); + for (Query scoringQuery : scoringQueries) { + scoringQuery.extractTerms(terms); } } @@ -130,8 +130,8 @@ public class CustomScoreQuery extends Query { public String toString(String field) { StringBuilder sb = new StringBuilder(name()).append("("); sb.append(subQuery.toString(field)); - for(int i = 0; i < scoringQueries.length; i++) { - sb.append(", ").append(scoringQueries[i].toString(field)); + for (Query scoringQuery : scoringQueries) { + sb.append(", ").append(scoringQuery.toString(field)); } sb.append(")"); sb.append(strict?" STRICT" : ""); @@ -200,11 +200,11 @@ public class CustomScoreQuery extends Query { @Override public float getValueForNormalization() throws IOException { float sum = subQueryWeight.getValueForNormalization(); - for(int i = 0; i < valSrcWeights.length; i++) { + for (Weight valSrcWeight : valSrcWeights) { if (qStrict) { - valSrcWeights[i].getValueForNormalization(); // do not include ValueSource part in the query normalization + valSrcWeight.getValueForNormalization(); // do not include ValueSource part in the query normalization } else { - sum += valSrcWeights[i].getValueForNormalization(); + sum += valSrcWeight.getValueForNormalization(); } } sum *= getBoost() * getBoost(); // boost each sub-weight @@ -216,11 +216,11 @@ public class CustomScoreQuery extends Query { public void normalize(float norm, float topLevelBoost) { topLevelBoost *= getBoost(); // incorporate boost subQueryWeight.normalize(norm, topLevelBoost); - for(int i = 0; i < valSrcWeights.length; i++) { + for (Weight valSrcWeight : valSrcWeights) { if (qStrict) { - valSrcWeights[i].normalize(1, 1); // do not normalize the ValueSource part + valSrcWeight.normalize(1, 1); // do not normalize the ValueSource part } else { - valSrcWeights[i].normalize(norm, topLevelBoost); + valSrcWeight.normalize(norm, topLevelBoost); } } } @@ -284,14 +284,14 @@ public class CustomScoreQuery extends Query { */ private class CustomScorer extends Scorer { private final float qWeight; - private Scorer subQueryScorer; - private Scorer[] valSrcScorers; + private final Scorer subQueryScorer; + private final Scorer[] valSrcScorers; private final CustomScoreProvider provider; - private float vScores[]; // reused in score() to avoid allocating this array for each doc + private final float[] vScores; // reused in score() to avoid allocating this array for each doc // constructor private CustomScorer(CustomScoreProvider provider, CustomWeight w, float qWeight, - Scorer subQueryScorer, Scorer[] valSrcScorers) throws IOException { + Scorer subQueryScorer, Scorer[] valSrcScorers) { super(w); this.qWeight = qWeight; this.subQueryScorer = subQueryScorer; @@ -304,8 +304,8 @@ public class CustomScoreQuery extends Query { public int nextDoc() throws IOException { int doc = subQueryScorer.nextDoc(); if (doc != NO_MORE_DOCS) { - for (int i = 0; i < valSrcScorers.length; i++) { - valSrcScorers[i].advance(doc); + for (Scorer valSrcScorer : valSrcScorers) { + valSrcScorer.advance(doc); } } return doc; @@ -329,8 +329,8 @@ public class CustomScoreQuery extends Query { public int advance(int target) throws IOException { int doc = subQueryScorer.advance(target); if (doc != NO_MORE_DOCS) { - for (int i = 0; i < valSrcScorers.length; i++) { - valSrcScorers[i].advance(doc); + for (Scorer valSrcScorer : valSrcScorers) { + valSrcScorer.advance(doc); } } return doc; diff --git a/lucene/queries/src/java/org/apache/lucene/queries/FilterClause.java b/lucene/queries/src/java/org/apache/lucene/queries/FilterClause.java index 1168706..f31439a 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/FilterClause.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/FilterClause.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/TermsFilter.java b/lucene/queries/src/java/org/apache/lucene/queries/TermsFilter.java index 4ce87fa..a50a91f 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/TermsFilter.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/TermsFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -65,7 +65,7 @@ public class TermsFilter extends Filter { BytesRef br = new BytesRef(); String lastField = null; - Terms termsC = null; + Terms termsC; TermsEnum termsEnum = null; DocsEnum docs = null; for (Term term : terms) { @@ -80,6 +80,7 @@ public class TermsFilter extends Filter { if (terms != null) { // TODO this check doesn't make sense, decide which variable its supposed to be for br.copyBytes(term.bytes()); + assert termsEnum != null; if (termsEnum.seekCeil(br) == TermsEnum.SeekStatus.FOUND) { docs = termsEnum.docs(acceptDocs, docs, false); while (docs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/BoostedQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/BoostedQuery.java index 5469ea2..c56b205 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/BoostedQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/BoostedQuery.java @@ -36,7 +36,7 @@ import java.util.Map; // something has to give public class BoostedQuery extends Query { private Query q; - private ValueSource boostVal; // optional, can be null + private final ValueSource boostVal; // optional, can be null public BoostedQuery(Query subQuery, ValueSource boostVal) { this.q = subQuery; @@ -66,7 +66,7 @@ public class BoostedQuery extends Query { } private class BoostedWeight extends Weight { - IndexSearcher searcher; + final IndexSearcher searcher; Weight qWeight; Map fcontext; diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java index dafa802..2396d10 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java @@ -39,7 +39,7 @@ import java.util.Map; * */ public class FunctionQuery extends Query { - ValueSource func; + final ValueSource func; /** * @param func defines the function to be used for scoring @@ -62,10 +62,10 @@ public class FunctionQuery extends Query { public void extractTerms(Set terms) {} protected class FunctionWeight extends Weight { - protected IndexSearcher searcher; + protected final IndexSearcher searcher; protected float queryNorm; protected float queryWeight; - protected Map context; + protected final Map context; public FunctionWeight(IndexSearcher searcher) throws IOException { this.searcher = searcher; diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionValues.java index d0866d5..c1890f4 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionValues.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionValues.java @@ -61,7 +61,7 @@ public abstract class FunctionValues { } target.copyChars(s); return true; - }; + } /** Native Java Object representation of the value */ public Object objectVal(int doc) { diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java index e4894ee..91c94cf 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java @@ -131,7 +131,7 @@ public abstract class ValueSource { private final double[] values; private FunctionValues docVals; private double bottom; - private Map fcontext; + private final Map fcontext; ValueSourceComparator(Map fcontext, int numHits) { this.fcontext = fcontext; @@ -187,7 +187,7 @@ public abstract class ValueSource { @Override public int compareDocToValue(int doc, Double valueObj) { - final double value = valueObj.doubleValue(); + final double value = valueObj; final double docValue = docVals.doubleVal(doc); if (docValue < value) { return -1; diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java index 31c7277..63f345b 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java @@ -26,7 +26,7 @@ import org.apache.lucene.util.Bits; import java.io.IOException; public class ValueSourceScorer extends Scorer { - protected IndexReader reader; + protected final IndexReader reader; private int doc = -1; protected final int maxDoc; protected final FunctionValues values; diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/BoolDocValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/BoolDocValues.java index 8ab782c..b942e8a 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/BoolDocValues.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/BoolDocValues.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries.function.docvalues; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java index f0dc1a6..89ffd19 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DoubleDocValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DoubleDocValues.java index cb879e7..44cc440 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DoubleDocValues.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DoubleDocValues.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries.function.docvalues; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/FloatDocValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/FloatDocValues.java index ba9988b..231c264 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/FloatDocValues.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/FloatDocValues.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries.function.docvalues; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/IntDocValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/IntDocValues.java index 2a3c86f..7276afc 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/IntDocValues.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/IntDocValues.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries.function.docvalues; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/LongDocValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/LongDocValues.java index dc5be02..26e4fde 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/LongDocValues.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/LongDocValues.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries.function.docvalues; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/StrDocValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/StrDocValues.java index 095b21d..726c566 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/StrDocValues.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/StrDocValues.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries.function.docvalues; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BoolFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BoolFunction.java index 93f1e28..6430686 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BoolFunction.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BoolFunction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ByteFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ByteFieldSource.java index 62e803d..c11ee78 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ByteFieldSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ByteFieldSource.java @@ -1,5 +1,5 @@ package org.apache.lucene.queries.function.valuesource; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -33,7 +33,7 @@ import org.apache.lucene.search.FieldCache; public class ByteFieldSource extends FieldCacheSource { - private FieldCache.ByteParser parser; + private final FieldCache.ByteParser parser; public ByteFieldSource(String field) { this(field, null); diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ConstNumberSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ConstNumberSource.java index 425ce52..5110002 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ConstNumberSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ConstNumberSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ConstValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ConstValueSource.java index 2c6349c..457a87d 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ConstValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ConstValueSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DefFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DefFunction.java index f19aca9..b025891 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DefFunction.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DefFunction.java @@ -1,5 +1,5 @@ package org.apache.lucene.queries.function.valuesource; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DivFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DivFloatFunction.java index 6ef2d5f..cb45b5b 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DivFloatFunction.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DivFloatFunction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DocFreqValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DocFreqValueSource.java index b85a84d..3b42539 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DocFreqValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DocFreqValueSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -124,10 +124,10 @@ class ConstDoubleDocValues extends DoubleDocValues { * @lucene.internal */ public class DocFreqValueSource extends ValueSource { - protected String field; - protected String indexedField; - protected String val; - protected BytesRef indexedBytes; + protected final String field; + protected final String indexedField; + protected final String val; + protected final BytesRef indexedBytes; public DocFreqValueSource(String field, String val, String indexedField, BytesRef indexedBytes) { this.field = field; diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleConstValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleConstValueSource.java index 641a9c5..5fad61e 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleConstValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleConstValueSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java index 32c7f73..97f2414 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -40,7 +40,7 @@ import org.apache.lucene.util.mutable.MutableValueDouble; public class DoubleFieldSource extends FieldCacheSource { - protected FieldCache.DoubleParser parser; + protected final FieldCache.DoubleParser parser; public DoubleFieldSource(String field) { this(field, null); diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DualFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DualFloatFunction.java index b6771fe..cace63c 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DualFloatFunction.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DualFloatFunction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FieldCacheSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FieldCacheSource.java index 20944ee..ebdb72c 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FieldCacheSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FieldCacheSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -27,8 +27,8 @@ import org.apache.lucene.search.FieldCache; * */ public abstract class FieldCacheSource extends ValueSource { - protected String field; - protected FieldCache cache = FieldCache.DEFAULT; + protected final String field; + protected final FieldCache cache = FieldCache.DEFAULT; public FieldCacheSource(String field) { this.field=field; @@ -58,6 +58,6 @@ public abstract class FieldCacheSource extends ValueSource { @Override public int hashCode() { return cache.hashCode() + field.hashCode(); - }; + } } diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java index 908d1aa..4486bd0 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -38,7 +38,7 @@ import org.apache.lucene.util.mutable.MutableValueFloat; public class FloatFieldSource extends FieldCacheSource { - protected FieldCache.FloatParser parser; + protected final FieldCache.FloatParser parser; public FloatFieldSource(String field) { this(field, null); @@ -108,5 +108,5 @@ public class FloatFieldSource extends FieldCacheSource { int h = parser==null ? Float.class.hashCode() : parser.getClass().hashCode(); h += super.hashCode(); return h; - }; + } } diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java index 4a3558a..319d755 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IfFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IfFunction.java index c9d9523..ff693b5 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IfFunction.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IfFunction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -31,9 +31,9 @@ import java.util.Map; public class IfFunction extends BoolFunction { - private ValueSource ifSource; - private ValueSource trueSource; - private ValueSource falseSource; + private final ValueSource ifSource; + private final ValueSource trueSource; + private final ValueSource falseSource; public IfFunction(ValueSource ifSource, ValueSource trueSource, ValueSource falseSource) { diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java index cf4a47f..8efe2de 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -172,5 +172,5 @@ public class IntFieldSource extends FieldCacheSource { int h = parser==null ? Integer.class.hashCode() : parser.getClass().hashCode(); h += super.hashCode(); return h; - }; + } } diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java index 7bb5014..1d8a8b1 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,11 +22,11 @@ import java.util.Map; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.docvalues.IntDocValues; import org.apache.lucene.search.FieldCache.DocTerms; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.ReaderUtil; import org.apache.lucene.util.packed.PackedInts; /** @@ -57,16 +57,15 @@ public class JoinDocFreqValueSource extends FieldCacheSource { final IndexReader top = ReaderUtil.getTopLevelContext(readerContext).reader(); return new IntDocValues(this) { - BytesRef ref = new BytesRef(); + final BytesRef ref = new BytesRef(); @Override public int intVal(int doc) { try { terms.getTerm(doc, ref); - int v = top.docFreq( qfield, ref ); //System.out.println( NAME+"["+field+"="+ref.utf8ToString()+"=("+qfield+":"+v+")]" ); - return v; + return top.docFreq( qfield, ref ); } catch (IOException e) { throw new RuntimeException("caught exception in function "+description()+" : doc="+doc, e); @@ -86,5 +85,5 @@ public class JoinDocFreqValueSource extends FieldCacheSource { @Override public int hashCode() { return qfield.hashCode() + super.hashCode(); - }; + } } diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LinearFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LinearFloatFunction.java index 7a930c4..34eed4b 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LinearFloatFunction.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LinearFloatFunction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LiteralValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LiteralValueSource.java index 3b5dea7..1762a80 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LiteralValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LiteralValueSource.java @@ -1,5 +1,5 @@ package org.apache.lucene.queries.function.valuesource; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -78,9 +78,8 @@ public class LiteralValueSource extends ValueSource { LiteralValueSource that = (LiteralValueSource) o; - if (!string.equals(that.string)) return false; + return string.equals(that.string); - return true; } public static final int hash = LiteralValueSource.class.hashCode(); diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java index 7628457..58a1d68 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -40,7 +40,7 @@ import org.apache.lucene.util.mutable.MutableValueLong; public class LongFieldSource extends FieldCacheSource { - protected FieldCache.LongParser parser; + protected final FieldCache.LongParser parser; public LongFieldSource(String field) { this(field, null); diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxDocValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxDocValueSource.java index 64ae927..789f047 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxDocValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxDocValueSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java index 5734434..9f998fd 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java index 67235a3..abae980 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiBoolFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiBoolFunction.java index 75edfae..2c89302 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiBoolFunction.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiBoolFunction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFloatFunction.java index e61d072..1131d21 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFloatFunction.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFloatFunction.java @@ -1,5 +1,5 @@ package org.apache.lucene.queries.function.valuesource; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java index 6951ae5..e6cdfae 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java @@ -1,5 +1,5 @@ package org.apache.lucene.queries.function.valuesource; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiValueSource.java index 13c899f..d6430c0 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiValueSource.java @@ -1,5 +1,5 @@ package org.apache.lucene.queries.function.valuesource; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java index 0a4ac53..88b357c 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -29,7 +29,7 @@ import java.io.IOException; import java.util.Map; public class NormValueSource extends ValueSource { - protected String field; + protected final String field; public NormValueSource(String field) { this.field = field; } diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NumDocsValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NumDocsValueSource.java index cce8738..5c60b5e 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NumDocsValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NumDocsValueSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,9 +17,9 @@ package org.apache.lucene.queries.function.valuesource; import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; -import org.apache.lucene.util.ReaderUtil; import java.io.IOException; import java.util.Map; diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NumericIndexDocValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NumericIndexDocValueSource.java index aa17061..a45e000 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NumericIndexDocValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NumericIndexDocValueSource.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries.function.valuesource; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/OrdFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/OrdFieldSource.java index 56ebf2c..04d5d0a 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/OrdFieldSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/OrdFieldSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,12 +21,12 @@ import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.CompositeReader; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.SlowCompositeReaderWrapper; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.docvalues.IntDocValues; import org.apache.lucene.search.FieldCache; -import org.apache.lucene.util.ReaderUtil; import org.apache.lucene.util.mutable.MutableValue; import org.apache.lucene.util.mutable.MutableValueInt; @@ -52,7 +52,7 @@ import java.util.Map; */ public class OrdFieldSource extends ValueSource { - protected String field; + protected final String field; public OrdFieldSource(String field) { this.field = field; @@ -124,6 +124,6 @@ public class OrdFieldSource extends ValueSource { @Override public int hashCode() { return hcode + field.hashCode(); - }; + } } diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/PowFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/PowFloatFunction.java index a65b4d2..7a35c94 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/PowFloatFunction.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/PowFloatFunction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ProductFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ProductFloatFunction.java index 102fff9..c957462 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ProductFloatFunction.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ProductFloatFunction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/QueryValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/QueryValueSource.java index 9caf765..3e22e8e 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/QueryValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/QueryValueSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,12 +18,12 @@ package org.apache.lucene.queries.function.valuesource; import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.docvalues.FloatDocValues; import org.apache.lucene.search.*; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.ReaderUtil; import org.apache.lucene.util.mutable.MutableValue; import org.apache.lucene.util.mutable.MutableValueFloat; @@ -237,7 +237,6 @@ class QueryDocValues extends FloatDocValues { // a match! mval.value = scorer.score(); mval.exists = true; - return; } catch (IOException e) { throw new RuntimeException("caught exception in QueryDocVals("+q+") doc="+doc, e); } diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/RangeMapFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/RangeMapFloatFunction.java index 041970d..79df9b9 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/RangeMapFloatFunction.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/RangeMapFloatFunction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReciprocalFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReciprocalFloatFunction.java index 9a3c57e..63aa5d6 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReciprocalFloatFunction.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReciprocalFloatFunction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReverseOrdFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReverseOrdFieldSource.java index a8ab36b..708c2b7 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReverseOrdFieldSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReverseOrdFieldSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,12 +21,12 @@ import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.CompositeReader; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.SlowCompositeReaderWrapper; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.docvalues.IntDocValues; import org.apache.lucene.search.FieldCache; -import org.apache.lucene.util.ReaderUtil; import java.io.IOException; import java.util.Map; @@ -53,7 +53,7 @@ import java.util.Map; */ public class ReverseOrdFieldSource extends ValueSource { - public String field; + public final String field; public ReverseOrdFieldSource(String field) { this.field = field; @@ -95,6 +95,6 @@ public class ReverseOrdFieldSource extends ValueSource { @Override public int hashCode() { return hcode + field.hashCode(); - }; + } } diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ScaleFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ScaleFloatFunction.java index ebc2da7..3b247a3 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ScaleFloatFunction.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ScaleFloatFunction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,13 +18,14 @@ package org.apache.lucene.queries.function.valuesource; import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.docvalues.FloatDocValues; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.util.ReaderUtil; import java.io.IOException; +import java.util.List; import java.util.Map; /** @@ -60,7 +61,7 @@ public class ScaleFloatFunction extends ValueSource { } private ScaleInfo createScaleInfo(Map context, AtomicReaderContext readerContext) throws IOException { - final AtomicReaderContext[] leaves = ReaderUtil.getTopLevelContext(readerContext).leaves(); + final List leaves = ReaderUtil.getTopLevelContext(readerContext).leaves(); float minVal = Float.POSITIVE_INFINITY; float maxVal = Float.NEGATIVE_INFINITY; diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ShortFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ShortFieldSource.java index 6807c88..f3fbe0b 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ShortFieldSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ShortFieldSource.java @@ -1,5 +1,5 @@ package org.apache.lucene.queries.function.valuesource; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SimpleBoolFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SimpleBoolFunction.java index ee2faf7..f4a7f74 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SimpleBoolFunction.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SimpleBoolFunction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SimpleFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SimpleFloatFunction.java index 9106aa3..824ac55 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SimpleFloatFunction.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SimpleFloatFunction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SingleFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SingleFunction.java index 2e95e4c..28e3616 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SingleFunction.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SingleFunction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SumFloatFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SumFloatFunction.java index 1bfa380..573a484 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SumFloatFunction.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SumFloatFunction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SumTotalTermFreqValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SumTotalTermFreqValueSource.java index e8ef8bd..54e9dac 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SumTotalTermFreqValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/SumTotalTermFreqValueSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -34,7 +34,7 @@ import java.util.Map; * @lucene.internal */ public class SumTotalTermFreqValueSource extends ValueSource { - protected String indexedField; + protected final String indexedField; public SumTotalTermFreqValueSource(String indexedField) { this.indexedField = indexedField; diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java index 1131f9b..4682efd 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries.function.valuesource; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -87,12 +87,12 @@ public class TFValueSource extends TermFreqValueSource { } @Override - public int nextDoc() throws IOException { + public int nextDoc() { return DocIdSetIterator.NO_MORE_DOCS; } @Override - public int advance(int target) throws IOException { + public int advance(int target) { return DocIdSetIterator.NO_MORE_DOCS; } }; diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TermFreqValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TermFreqValueSource.java index 67d19d1..f2b2699 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TermFreqValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TermFreqValueSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -80,12 +80,12 @@ public class TermFreqValueSource extends DocFreqValueSource { } @Override - public int nextDoc() throws IOException { + public int nextDoc() { return DocIdSetIterator.NO_MORE_DOCS; } @Override - public int advance(int target) throws IOException { + public int advance(int target) { return DocIdSetIterator.NO_MORE_DOCS; } }; diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TotalTermFreqValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TotalTermFreqValueSource.java index 7480624..52cede1 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TotalTermFreqValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TotalTermFreqValueSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -32,10 +32,10 @@ import java.util.Map; * @lucene.internal */ public class TotalTermFreqValueSource extends ValueSource { - protected String field; - protected String indexedField; - protected String val; - protected BytesRef indexedBytes; + protected final String field; + protected final String indexedField; + protected final String val; + protected final BytesRef indexedBytes; public TotalTermFreqValueSource(String field, String val, String indexedField, BytesRef indexedBytes) { this.field = field; diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/VectorValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/VectorValueSource.java index b8dc453..8d443b4 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/VectorValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/VectorValueSource.java @@ -1,5 +1,5 @@ package org.apache.lucene.queries.function.valuesource; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -210,9 +210,8 @@ public class VectorValueSource extends MultiValueSource { VectorValueSource that = (VectorValueSource) o; - if (!sources.equals(that.sources)) return false; + return sources.equals(that.sources); - return true; } @Override diff --git a/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java b/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java index 6229a0c..d8b9776 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java @@ -717,8 +717,8 @@ public final class MoreLikeThis { if (vector == null) { Document d = ir.document(docNum); IndexableField fields[] = d.getFields(fieldName); - for (int j = 0; j < fields.length; j++) { - final String stringValue = fields[j].stringValue(); + for (IndexableField field : fields) { + final String stringValue = field.stringValue(); if (stringValue != null) { addTermFrequencies(new StringReader(stringValue), termFreqMap, fieldName); } diff --git a/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThisQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThisQuery.java index d26efb5..1931e72 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThisQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThisQuery.java @@ -3,7 +3,7 @@ */ package org.apache.lucene.queries.mlt; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -41,7 +41,7 @@ public class MoreLikeThisQuery extends Query { private String likeText; private String[] moreLikeFields; private Analyzer analyzer; - private String fieldName; + private final String fieldName; private float percentTermsToMatch = 0.3f; private int minTermFrequency = 1; private int maxQueryTerms = 5; @@ -49,7 +49,7 @@ public class MoreLikeThisQuery extends Query { private int minDocFreq = -1; /** - * @param moreLikeFields + * @param moreLikeFields fields used for similarity measure */ public MoreLikeThisQuery(String likeText, String[] moreLikeFields, Analyzer analyzer, String fieldName) { this.likeText = likeText; diff --git a/lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java b/lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java index 61903db..b5d4822 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,7 +20,7 @@ package org.apache.lucene.queries; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.RandomIndexWriter; @@ -69,10 +69,10 @@ public class BooleanFilterTest extends LuceneTestCase { private void addDoc(RandomIndexWriter writer, String accessRights, String price, String date, String inStock) throws IOException { Document doc = new Document(); - doc.add(newField("accessRights", accessRights, TextField.TYPE_STORED)); - doc.add(newField("price", price, TextField.TYPE_STORED)); - doc.add(newField("date", date, TextField.TYPE_STORED)); - doc.add(newField("inStock", inStock, TextField.TYPE_STORED)); + doc.add(newTextField("accessRights", accessRights, Field.Store.YES)); + doc.add(newTextField("price", price, Field.Store.YES)); + doc.add(newTextField("date", date, Field.Store.YES)); + doc.add(newTextField("inStock", inStock, Field.Store.YES)); writer.addDocument(doc); } diff --git a/lucene/queries/src/test/org/apache/lucene/queries/BoostingQueryTest.java b/lucene/queries/src/test/org/apache/lucene/queries/BoostingQueryTest.java index ef4e9ed..fc9a973 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/BoostingQueryTest.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/BoostingQueryTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queries/src/test/org/apache/lucene/queries/ChainedFilterTest.java b/lucene/queries/src/test/org/apache/lucene/queries/ChainedFilterTest.java index 78dca3f..6b78b48 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/ChainedFilterTest.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/ChainedFilterTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,7 +21,7 @@ import java.util.Calendar; import java.util.GregorianCalendar; import org.apache.lucene.document.Document; -import org.apache.lucene.document.StringField; +import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; @@ -62,9 +62,9 @@ public class ChainedFilterTest extends LuceneTestCase { for (int i = 0; i < MAX; i++) { Document doc = new Document(); - doc.add(newField("key", "" + (i + 1), StringField.TYPE_STORED)); - doc.add(newField("owner", (i < MAX / 2) ? "bob" : "sue", StringField.TYPE_STORED)); - doc.add(newField("date", cal.getTime().toString(), StringField.TYPE_STORED)); + doc.add(newStringField("key", "" + (i + 1), Field.Store.YES)); + doc.add(newStringField("owner", (i < MAX / 2) ? "bob" : "sue", Field.Store.YES)); + doc.add(newStringField("date", cal.getTime().toString(), Field.Store.YES)); writer.addDocument(doc); cal.add(Calendar.DATE, 1); diff --git a/lucene/queries/src/test/org/apache/lucene/queries/TermsFilterTest.java b/lucene/queries/src/test/org/apache/lucene/queries/TermsFilterTest.java index 7381ca8..03ae0a0 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/TermsFilterTest.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/TermsFilterTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,7 +20,7 @@ package org.apache.lucene.queries; import java.util.HashSet; import org.apache.lucene.document.Document; -import org.apache.lucene.document.StringField; +import org.apache.lucene.document.Field; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiReader; @@ -58,7 +58,7 @@ public class TermsFilterTest extends LuceneTestCase { for (int i = 0; i < 100; i++) { Document doc = new Document(); int term = i * 10; //terms are units of 10; - doc.add(newField(fieldName, "" + term, StringField.TYPE_STORED)); + doc.add(newStringField(fieldName, "" + term, Field.Store.YES)); w.addDocument(doc); } IndexReader reader = new SlowCompositeReaderWrapper(w.getReader()); @@ -92,7 +92,7 @@ public class TermsFilterTest extends LuceneTestCase { Directory rd1 = newDirectory(); RandomIndexWriter w1 = new RandomIndexWriter(random(), rd1); Document doc = new Document(); - doc.add(newField(fieldName, "content1", StringField.TYPE_STORED)); + doc.add(newStringField(fieldName, "content1", Field.Store.YES)); w1.addDocument(doc); IndexReader reader1 = w1.getReader(); w1.close(); @@ -101,7 +101,7 @@ public class TermsFilterTest extends LuceneTestCase { Directory rd2 = newDirectory(); RandomIndexWriter w2 = new RandomIndexWriter(random(), rd2); doc = new Document(); - doc.add(newField(fieldName, "content2", StringField.TYPE_STORED)); + doc.add(newStringField(fieldName, "content2", Field.Store.YES)); w2.addDocument(doc); IndexReader reader2 = w2.getReader(); w2.close(); diff --git a/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java b/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java index 5233aba..bcfd61e 100755 --- a/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,8 +20,16 @@ package org.apache.lucene.queries; import org.apache.lucene.queries.function.FunctionQuery; import org.apache.lucene.queries.function.FunctionTestSetup; import org.apache.lucene.queries.function.ValueSource; -import org.apache.lucene.queries.function.valuesource.FloatFieldSource; -import org.apache.lucene.search.*; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryUtils; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.search.TopDocs; import org.junit.BeforeClass; import org.junit.Test; import java.io.IOException; @@ -29,6 +37,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; @@ -177,7 +186,7 @@ public class TestCustomScoreQuery extends FunctionTestSetup { final int[] values = FieldCache.DEFAULT.getInts(context.reader(), INT_FIELD, false); return new CustomScoreProvider(context) { @Override - public float customScore(int doc, float subScore, float valSrcScore) throws IOException { + public float customScore(int doc, float subScore, float valSrcScore) { assertTrue(doc <= context.reader().maxDoc()); return values[doc]; } @@ -199,7 +208,7 @@ public class TestCustomScoreQuery extends FunctionTestSetup { final Query q = new CustomExternalQuery(q1); log(q); - IndexReader r = IndexReader.open(dir); + IndexReader r = DirectoryReader.open(dir); IndexSearcher s = new IndexSearcher(r); TopDocs hits = s.search(q, 1000); assertEquals(N_DOCS, hits.totalHits); @@ -213,7 +222,7 @@ public class TestCustomScoreQuery extends FunctionTestSetup { @Test public void testRewrite() throws Exception { - IndexReader r = IndexReader.open(dir); + IndexReader r = DirectoryReader.open(dir); final IndexSearcher s = new IndexSearcher(r); Query q = new TermQuery(new Term(TEXT_FIELD, "first")); @@ -238,7 +247,7 @@ public class TestCustomScoreQuery extends FunctionTestSetup { private void doTestCustomScore(ValueSource valueSource, double dboost) throws Exception { float boost = (float) dboost; FunctionQuery functionQuery = new FunctionQuery(valueSource); - IndexReader r = IndexReader.open(dir); + IndexReader r = DirectoryReader.open(dir); IndexSearcher s = new IndexSearcher(r); // regular (boolean) query. diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java b/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java index f6d7622..416967f 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java @@ -135,7 +135,7 @@ public abstract class FunctionTestSetup extends LuceneTestCase { f = newField(ID_FIELD, id2String(scoreAndID), customType); // for debug purposes d.add(f); - FieldType customType2 = new FieldType(TextField.TYPE_UNSTORED); + FieldType customType2 = new FieldType(TextField.TYPE_NOT_STORED); customType2.setOmitNorms(true); f = newField(TEXT_FIELD, "text of doc" + scoreAndID + textLine(i), customType2); // for regular search d.add(f); diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestBoostedQuery.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestBoostedQuery.java index 5a9a0c2..a8f7592 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestBoostedQuery.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestBoostedQuery.java @@ -1,6 +1,5 @@ package org.apache.lucene.queries.function; -import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; @@ -23,7 +22,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.junit.AfterClass; import org.junit.BeforeClass; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -56,7 +55,7 @@ public class TestBoostedQuery extends LuceneTestCase { iwConfig.setMergePolicy(newLogMergePolicy()); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConfig); Document document = new Document(); - Field idField = new StringField("id", ""); + Field idField = new StringField("id", "", Field.Store.NO); document.add(idField); iw.addDocument(document); ir = iw.getReader(); diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java index 5aea75a..8383f77 100755 --- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries.function; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,6 +17,7 @@ package org.apache.lucene.queries.function; * limitations under the License. */ +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.queries.function.FunctionQuery; import org.apache.lucene.queries.function.ValueSource; @@ -81,7 +82,7 @@ public class TestFieldScoreQuery extends FunctionTestSetup { // Test that FieldScoreQuery returns docs in expected order. private void doTestRank (ValueSource valueSource) throws Exception { FunctionQuery functionQuery = new FunctionQuery(valueSource); - IndexReader r = IndexReader.open(dir); + IndexReader r = DirectoryReader.open(dir); IndexSearcher s = new IndexSearcher(r); log("test: "+ functionQuery); QueryUtils.check(random(), functionQuery,s); @@ -130,7 +131,7 @@ public class TestFieldScoreQuery extends FunctionTestSetup { // Test that FieldScoreQuery returns docs with expected score. private void doTestExactScore (ValueSource valueSource) throws Exception { FunctionQuery functionQuery = new FunctionQuery(valueSource); - IndexReader r = IndexReader.open(dir); + IndexReader r = DirectoryReader.open(dir); IndexSearcher s = new IndexSearcher(r); TopDocs td = s.search(functionQuery,null,1000); assertEquals("All docs should be matched!",N_DOCS,td.totalHits); diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestOrdValues.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestOrdValues.java index 0890971..c07323c 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestOrdValues.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestOrdValues.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries.function; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,11 +17,15 @@ package org.apache.lucene.queries.function; * limitations under the License. */ -import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.queries.function.valuesource.OrdFieldSource; import org.apache.lucene.queries.function.valuesource.ReverseOrdFieldSource; -import org.apache.lucene.search.*; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryUtils; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; import org.junit.BeforeClass; import org.junit.Test; @@ -47,7 +51,7 @@ public class TestOrdValues extends FunctionTestSetup { * Test OrdFieldSource */ @Test - public void testOrdFieldRank() throws CorruptIndexException, Exception { + public void testOrdFieldRank() throws Exception { doTestRank(ID_FIELD, true); } @@ -55,13 +59,13 @@ public class TestOrdValues extends FunctionTestSetup { * Test ReverseOrdFieldSource */ @Test - public void testReverseOrdFieldRank() throws CorruptIndexException, Exception { + public void testReverseOrdFieldRank() throws Exception { doTestRank(ID_FIELD, false); } // Test that queries based on reverse/ordFieldScore scores correctly - private void doTestRank(String field, boolean inOrder) throws CorruptIndexException, Exception { - IndexReader r = IndexReader.open(dir); + private void doTestRank(String field, boolean inOrder) throws Exception { + IndexReader r = DirectoryReader.open(dir); IndexSearcher s = new IndexSearcher(r); ValueSource vs; if (inOrder) { @@ -97,7 +101,7 @@ public class TestOrdValues extends FunctionTestSetup { * Test exact score for OrdFieldSource */ @Test - public void testOrdFieldExactScore() throws CorruptIndexException, Exception { + public void testOrdFieldExactScore() throws Exception { doTestExactScore(ID_FIELD, true); } @@ -105,14 +109,14 @@ public class TestOrdValues extends FunctionTestSetup { * Test exact score for ReverseOrdFieldSource */ @Test - public void testReverseOrdFieldExactScore() throws CorruptIndexException, Exception { + public void testReverseOrdFieldExactScore() throws Exception { doTestExactScore(ID_FIELD, false); } // Test that queries based on reverse/ordFieldScore returns docs with expected score. - private void doTestExactScore(String field, boolean inOrder) throws CorruptIndexException, Exception { - IndexReader r = IndexReader.open(dir); + private void doTestExactScore(String field, boolean inOrder) throws Exception { + IndexReader r = DirectoryReader.open(dir); IndexSearcher s = new IndexSearcher(r); ValueSource vs; if (inOrder) { diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java index de8a33f..cd856d3 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries.function; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,7 +21,6 @@ import java.util.Arrays; import java.util.List; import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; @@ -98,23 +97,23 @@ public class TestValueSources extends LuceneTestCase { iwConfig.setMergePolicy(newLogMergePolicy()); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConfig); Document document = new Document(); - Field idField = new StringField("id", ""); + Field idField = new StringField("id", "", Field.Store.NO); document.add(idField); - Field byteField = new StringField("byte", ""); + Field byteField = new StringField("byte", "", Field.Store.NO); document.add(byteField); - Field doubleField = new StringField("double", ""); + Field doubleField = new StringField("double", "", Field.Store.NO); document.add(doubleField); - Field floatField = new StringField("float", ""); + Field floatField = new StringField("float", "", Field.Store.NO); document.add(floatField); - Field intField = new StringField("int", ""); + Field intField = new StringField("int", "", Field.Store.NO); document.add(intField); - Field longField = new StringField("long", ""); + Field longField = new StringField("long", "", Field.Store.NO); document.add(longField); - Field shortField = new StringField("short", ""); + Field shortField = new StringField("short", "", Field.Store.NO); document.add(shortField); - Field stringField = new StringField("string", ""); + Field stringField = new StringField("string", "", Field.Store.NO); document.add(stringField); - Field textField = new TextField("text", ""); + Field textField = new TextField("text", "", Field.Store.NO); document.add(textField); for (String [] doc : documents) { @@ -311,13 +310,8 @@ public class TestValueSources extends LuceneTestCase { } public void testSumTotalTermFreq() throws Exception { - if (Codec.getDefault().getName().equals("Lucene3x")) { - assertHits(new FunctionQuery(new SumTotalTermFreqValueSource("text")), - new float[] { -1f, -1f }); - } else { - assertHits(new FunctionQuery(new SumTotalTermFreqValueSource("text")), + assertHits(new FunctionQuery(new SumTotalTermFreqValueSource("text")), new float[] { 8f, 8f }); - } } public void testTermFreq() throws Exception { @@ -346,15 +340,9 @@ public class TestValueSources extends LuceneTestCase { } public void testTotalTermFreq() throws Exception { - if (Codec.getDefault().getName().equals("Lucene3x")) { - assertHits(new FunctionQuery( - new TotalTermFreqValueSource("bogus", "bogus", "text", new BytesRef("test"))), - new float[] { -1f, -1f }); - } else { - assertHits(new FunctionQuery( - new TotalTermFreqValueSource("bogus", "bogus", "text", new BytesRef("test"))), - new float[] { 4f, 4f }); - } + assertHits(new FunctionQuery( + new TotalTermFreqValueSource("bogus", "bogus", "text", new BytesRef("test"))), + new float[] { 4f, 4f }); } void assertHits(Query q, float scores[]) throws Exception { diff --git a/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java b/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java index 3635711..0929385 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java @@ -1,6 +1,6 @@ package org.apache.lucene.queries.mlt; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -26,7 +26,7 @@ import java.util.Map; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.BooleanClause; @@ -65,7 +65,7 @@ public class TestMoreLikeThis extends LuceneTestCase { private void addDoc(RandomIndexWriter writer, String text) throws IOException { Document doc = new Document(); - doc.add(newField("text", text, TextField.TYPE_STORED)); + doc.add(newTextField("text", text, Field.Store.YES)); writer.addDocument(doc); } diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java index 01e240c..67357d3 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.analyzing; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -50,6 +50,7 @@ public class AnalyzingQueryParser extends org.apache.lucene.queryparser.classic. */ public AnalyzingQueryParser(Version matchVersion, String field, Analyzer analyzer) { super(matchVersion, field, analyzer); + setAnalyzeRangeTerms(true); } /** @@ -278,72 +279,4 @@ public class AnalyzingQueryParser extends org.apache.lucene.queryparser.classic. return (nextToken == null) ? null : super.getFuzzyQuery(field, nextToken, minSimilarity); } - - /** - * Overrides super class, by passing terms through analyzer. - * @exception ParseException - */ - @Override - protected Query getRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) - throws ParseException { - // get Analyzer from superclass and tokenize the terms - TokenStream source = null; - CharTermAttribute termAtt = null; - boolean multipleTokens = false; - - if (part1 != null) { - // part1 - try { - source = getAnalyzer().tokenStream(field, new StringReader(part1)); - termAtt = source.addAttribute(CharTermAttribute.class); - source.reset(); - multipleTokens = false; - - - if (source.incrementToken()) { - part1 = termAtt.toString(); - } - multipleTokens = source.incrementToken(); - } catch (IOException e) { - // ignore - } - try { - source.end(); - source.close(); - } catch (IOException e) { - // ignore - } - if (multipleTokens) { - throw new ParseException("Cannot build RangeQuery with analyzer " + getAnalyzer().getClass() - + " - tokens were added to part1"); - } - } - - if (part2 != null) { - try { - // part2 - source = getAnalyzer().tokenStream(field, new StringReader(part2)); - termAtt = source.addAttribute(CharTermAttribute.class); - source.reset(); - if (source.incrementToken()) { - part2 = termAtt.toString(); - } - multipleTokens = source.incrementToken(); - } catch (IOException e) { - // ignore - } - try { - source.end(); - source.close(); - } catch (IOException e) { - // ignore - } - if (multipleTokens) { - throw new ParseException("Cannot build RangeQuery with analyzer " + getAnalyzer().getClass() - + " - tokens were added to part2"); - } - } - return super.getRangeQuery(field, part1, part2, startInclusive, endInclusive); - } - } diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/FastCharStream.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/FastCharStream.java index 3960cc4..8c995be 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/FastCharStream.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/FastCharStream.java @@ -1,7 +1,7 @@ // FastCharStream.java package org.apache.lucene.queryparser.classic; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/MultiFieldQueryParser.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/MultiFieldQueryParser.java index b55f99d..e4f01f0 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/MultiFieldQueryParser.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/MultiFieldQueryParser.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.classic; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.java index c0df24d..8180f153 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.java @@ -76,14 +76,6 @@ import org.apache.lucene.util.Version; *

      NOTE: there is a new QueryParser in contrib, which matches * the same syntax as this class, but is more modular, * enabling substantial customization to how a query is created. - * - * - *

      NOTE: You must specify the required {@link Version} - * compatibility when creating QueryParser: - *

      */ public class QueryParser extends QueryParserBase implements QueryParserConstants { /** The default operator for parsing queries. @@ -92,7 +84,7 @@ public class QueryParser extends QueryParserBase implements QueryParserConstants static public enum Operator { OR, AND } /** Create a query parser. - * @param matchVersion Lucene version to match. See above. + * @param matchVersion Lucene version to match. * @param f the default field for query terms. * @param a used to find terms in the query text. */ diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java index 3d99fc4..3e62f78 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -88,18 +88,14 @@ public abstract class QueryParserBase { } /** Initializes a query parser. Called by the QueryParser constructor - * @param matchVersion Lucene version to match. See here. + * @param matchVersion Lucene version to match. * @param f the default field for query terms. * @param a used to find terms in the query text. */ public void init(Version matchVersion, String f, Analyzer a) { analyzer = a; field = f; - if (matchVersion.onOrAfter(Version.LUCENE_31)) { - setAutoGeneratePhraseQueries(false); - } else { - setAutoGeneratePhraseQueries(true); - } + setAutoGeneratePhraseQueries(false); } // the generated parser will create these in QueryParser @@ -482,11 +478,7 @@ public abstract class QueryParserBase { PositionIncrementAttribute posIncrAtt = null; int numTokens = 0; - try { - buffer.reset(); - } catch (IOException e) { - throw new ParseException("Unable to initialize TokenStream to analyze query text", e); - } + buffer.reset(); if (buffer.hasAttribute(TermToBytesRefAttribute.class)) { termAtt = buffer.getAttribute(TermToBytesRefAttribute.class); @@ -1076,7 +1068,7 @@ public abstract class QueryParserBase { } // extracted from the .jj grammar - Query handleBoost(Query q, Token boost) throws ParseException { + Query handleBoost(Query q, Token boost) { if (boost != null) { float f = (float) 1.0; try { diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/package.html b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/package.html index de3a420..607e750 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/package.html +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/package.html @@ -51,6 +51,9 @@ enabling substantial customization to how a query is created. Wildcard Searches
    • +Regular expression Searches +
      • +
    • Fuzzy Searches
    • @@ -189,6 +192,12 @@ enabling substantial customization to how a query is created.

      You can also use the wildcard searches in the middle of a term.

      te*t

      Note: You cannot use a * or ? symbol as the first character of a search.

      + +

      Regular Expression Searches

      +

      Lucene supports regular expression searches matching a pattern between forward slashes "/". The syntax may change across releases, but the current supported +syntax is documented in the {@link org.apache.lucene.util.automaton.RegExp RegExp} class. For example to find documents containing "moat" or "boat": +

      +
      /[mb]oat/

      Fuzzy Searches

      Lucene supports fuzzy searches based on Damerau-Levenshtein Distance. To do a fuzzy search use the tilde, "~", symbol at the end of a Single word Term. For example to search for a term similar in spelling to "roam" use the fuzzy search:

      @@ -294,7 +303,7 @@ enabling substantial customization to how a query is created.

      Escaping Special Characters

      Lucene supports escaping special characters that are part of the query syntax. The current list special characters are

      -

      + - && || ! ( ) { } [ ] ^ " ~ * ? : \

      +

      + - && || ! ( ) { } [ ] ^ " ~ * ? : \ /

      To escape these character use the \ before the character. For example to search for (1+1):2 use the query:

      \(1\+1\)\:2
      diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/complexPhrase/ComplexPhraseQueryParser.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/complexPhrase/ComplexPhraseQueryParser.java index f7611c7..1a1faa4 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/complexPhrase/ComplexPhraseQueryParser.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/complexPhrase/ComplexPhraseQueryParser.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.complexPhrase; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/ext/ExtendableQueryParser.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/ext/ExtendableQueryParser.java index fdda8c3..8c0e26b 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/ext/ExtendableQueryParser.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/ext/ExtendableQueryParser.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.ext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/ext/ExtensionQuery.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/ext/ExtensionQuery.java index 4789633..423595e 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/ext/ExtensionQuery.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/ext/ExtensionQuery.java @@ -2,7 +2,7 @@ package org.apache.lucene.queryparser.ext; import org.apache.lucene.queryparser.classic.QueryParser; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/ext/Extensions.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/ext/Extensions.java index 1de3c1f..1a82e98 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/ext/Extensions.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/ext/Extensions.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.ext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/ext/ParserExtension.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/ext/ParserExtension.java index 294a8a3..88573ef 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/ext/ParserExtension.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/ext/ParserExtension.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.ext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/QueryNodeError.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/QueryNodeError.java index a22d137..e9f14bc 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/QueryNodeError.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/QueryNodeError.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/QueryNodeException.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/QueryNodeException.java index c2f3779..c4f0785 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/QueryNodeException.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/QueryNodeException.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/QueryNodeParseException.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/QueryNodeParseException.java index b309ec3..6ca2606 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/QueryNodeParseException.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/QueryNodeParseException.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/QueryParserHelper.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/QueryParserHelper.java index 1cd2f5d..e48f9ab 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/QueryParserHelper.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/QueryParserHelper.java @@ -6,7 +6,7 @@ import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser; import org.apache.lucene.queryparser.flexible.core.processors.QueryNodeProcessor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/builders/QueryBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/builders/QueryBuilder.java index 445f8e5..cc3bbe7 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/builders/QueryBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/builders/QueryBuilder.java @@ -3,7 +3,7 @@ package org.apache.lucene.queryparser.flexible.core.builders; import org.apache.lucene.queryparser.flexible.core.QueryNodeException; import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/builders/QueryTreeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/builders/QueryTreeBuilder.java index f77bc96..6aed7d6 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/builders/QueryTreeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/builders/QueryTreeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/AbstractQueryConfig.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/AbstractQueryConfig.java index a2f4f5a..fc6d79d 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/AbstractQueryConfig.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/AbstractQueryConfig.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.config; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/ConfigurationKey.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/ConfigurationKey.java index ac8138d..1ced76d 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/ConfigurationKey.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/ConfigurationKey.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.config; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/FieldConfig.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/FieldConfig.java index b59c579..b3876ca 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/FieldConfig.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/FieldConfig.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.config; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/FieldConfigListener.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/FieldConfigListener.java index a1f7911..f729377 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/FieldConfigListener.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/FieldConfigListener.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.config; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/QueryConfigHandler.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/QueryConfigHandler.java index b60bdbe..03e6110 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/QueryConfigHandler.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/QueryConfigHandler.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.config; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/messages/QueryParserMessages.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/messages/QueryParserMessages.java index 757f4f6..6ceb7df 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/messages/QueryParserMessages.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/messages/QueryParserMessages.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.messages; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/AndQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/AndQueryNode.java index 430395b..1090a47 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/AndQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/AndQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/AnyQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/AnyQueryNode.java index 22ad5b4..ad8ff39 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/AnyQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/AnyQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/BooleanQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/BooleanQueryNode.java index 3b60919..2ef6b96 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/BooleanQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/BooleanQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/BoostQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/BoostQueryNode.java index 4da96fa..ec53ca6 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/BoostQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/BoostQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,7 +21,6 @@ import java.util.List; import org.apache.lucene.queryparser.flexible.messages.MessageImpl; import org.apache.lucene.queryparser.flexible.core.QueryNodeError; -import org.apache.lucene.queryparser.flexible.core.QueryNodeException; import org.apache.lucene.queryparser.flexible.core.messages.QueryParserMessages; import org.apache.lucene.queryparser.flexible.core.parser.EscapeQuerySyntax; @@ -43,10 +42,8 @@ public class BoostQueryNode extends QueryNodeImpl { * the query to be boosted * @param value * the boost value, it may vary from 0.0 to 1.0 - * - * @throws QueryNodeException */ - public BoostQueryNode(QueryNode query, float value) throws QueryNodeException { + public BoostQueryNode(QueryNode query, float value) { if (query == null) { throw new QueryNodeError(new MessageImpl( QueryParserMessages.NODE_ACTION_NOT_SUPPORTED, "query", "null")); diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/DeletedQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/DeletedQueryNode.java index f29201b..22db473 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/DeletedQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/DeletedQueryNode.java @@ -2,7 +2,7 @@ package org.apache.lucene.queryparser.flexible.core.nodes; import org.apache.lucene.queryparser.flexible.core.parser.EscapeQuerySyntax; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/FieldQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/FieldQueryNode.java index e276d3b..0a8b53b 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/FieldQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/FieldQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/FieldValuePairQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/FieldValuePairQueryNode.java index 6188af3..daf0ae1 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/FieldValuePairQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/FieldValuePairQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/FieldableNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/FieldableNode.java index f069215..6ad248a 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/FieldableNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/FieldableNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/FuzzyQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/FuzzyQueryNode.java index 9a3ffcd..b009a4b 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/FuzzyQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/FuzzyQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/GroupQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/GroupQueryNode.java index e4fcc0c..dfb818f 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/GroupQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/GroupQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/MatchAllDocsQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/MatchAllDocsQueryNode.java index ffcf8a7..0e1efd4 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/MatchAllDocsQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/MatchAllDocsQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/MatchNoDocsQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/MatchNoDocsQueryNode.java index 90e8ced..06fbaf9 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/MatchNoDocsQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/MatchNoDocsQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/ModifierQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/ModifierQueryNode.java index f092ee9..3c6c9d7 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/ModifierQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/ModifierQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/NoTokenFoundQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/NoTokenFoundQueryNode.java index b5e503a..a40de2e 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/NoTokenFoundQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/NoTokenFoundQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/OpaqueQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/OpaqueQueryNode.java index 4edcf89..ba08a45 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/OpaqueQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/OpaqueQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/OrQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/OrQueryNode.java index eed480c..3ba90b0 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/OrQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/OrQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/PathQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/PathQueryNode.java index ea89768..2bc7dd0 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/PathQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/PathQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/PhraseSlopQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/PhraseSlopQueryNode.java index c4c592a..7272111 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/PhraseSlopQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/PhraseSlopQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,23 +19,17 @@ package org.apache.lucene.queryparser.flexible.core.nodes; import org.apache.lucene.queryparser.flexible.messages.MessageImpl; import org.apache.lucene.queryparser.flexible.core.QueryNodeError; -import org.apache.lucene.queryparser.flexible.core.QueryNodeException; import org.apache.lucene.queryparser.flexible.core.messages.QueryParserMessages; import org.apache.lucene.queryparser.flexible.core.parser.EscapeQuerySyntax; -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; public class PhraseSlopQueryNode extends QueryNodeImpl implements FieldableNode { private int value = 0; /** - * @throws QueryNodeException - * @throws QueryNodeParseException - * @exception QueryNodeParseException - * throw in overridden method to disallow + * @exception QueryNodeError throw in overridden method to disallow */ - public PhraseSlopQueryNode(QueryNode query, int value) - throws QueryNodeException { + public PhraseSlopQueryNode(QueryNode query, int value) { if (query == null) { throw new QueryNodeError(new MessageImpl( QueryParserMessages.NODE_ACTION_NOT_SUPPORTED, "query", "null")); diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/ProximityQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/ProximityQueryNode.java index 75d1ca3..43aefbb 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/ProximityQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/ProximityQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/QueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/QueryNode.java index 1e2af9e..b7aefb2 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/QueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/QueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/QueryNodeImpl.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/QueryNodeImpl.java index 3177df2..2c049ef 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/QueryNodeImpl.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/QueryNodeImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/QuotedFieldQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/QuotedFieldQueryNode.java index 9e2f7cf..a369b35 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/QuotedFieldQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/QuotedFieldQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/RangeQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/RangeQueryNode.java index 033f9bd..a076b58 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/RangeQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/RangeQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/SlopQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/SlopQueryNode.java index a29ef75..c04d615 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/SlopQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/SlopQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/TextableQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/TextableQueryNode.java index fa05986..8476ae7 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/TextableQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/TextableQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/TokenizedPhraseQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/TokenizedPhraseQueryNode.java index 9a09105..081d8e4 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/TokenizedPhraseQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/TokenizedPhraseQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/ValueQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/ValueQueryNode.java index f97ab3e..9b18156 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/ValueQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/ValueQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/parser/EscapeQuerySyntax.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/parser/EscapeQuerySyntax.java index ddbd126..1bc1704 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/parser/EscapeQuerySyntax.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/parser/EscapeQuerySyntax.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.parser; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/parser/SyntaxParser.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/parser/SyntaxParser.java index 29d4a98..857f03c 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/parser/SyntaxParser.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/parser/SyntaxParser.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.parser; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/processors/NoChildOptimizationQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/processors/NoChildOptimizationQueryNodeProcessor.java index 0aaee46..dcdad7a 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/processors/NoChildOptimizationQueryNodeProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/processors/NoChildOptimizationQueryNodeProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/processors/QueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/processors/QueryNodeProcessor.java index ad47cdc..b192ecd 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/processors/QueryNodeProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/processors/QueryNodeProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/processors/QueryNodeProcessorImpl.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/processors/QueryNodeProcessorImpl.java index a34acf5..adc9303 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/processors/QueryNodeProcessorImpl.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/processors/QueryNodeProcessorImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/processors/QueryNodeProcessorPipeline.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/processors/QueryNodeProcessorPipeline.java index dee780c..577d47b 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/processors/QueryNodeProcessorPipeline.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/processors/QueryNodeProcessorPipeline.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/processors/RemoveDeletedQueryNodesProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/processors/RemoveDeletedQueryNodesProcessor.java index dd0933c..ddde390 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/processors/RemoveDeletedQueryNodesProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/processors/RemoveDeletedQueryNodesProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/util/QueryNodeOperation.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/util/QueryNodeOperation.java index 51b359d..1c91c30 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/util/QueryNodeOperation.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/util/QueryNodeOperation.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/util/StringUtils.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/util/StringUtils.java index 35cf537..0ae3597 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/util/StringUtils.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/util/StringUtils.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/util/UnescapedCharSequence.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/util/UnescapedCharSequence.java index 83e5be8..e3c649d 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/util/UnescapedCharSequence.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/util/UnescapedCharSequence.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/messages/Message.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/messages/Message.java index 9422913..e4a9d2c 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/messages/Message.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/messages/Message.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.messages; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/messages/MessageImpl.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/messages/MessageImpl.java index e056f62..133aee2 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/messages/MessageImpl.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/messages/MessageImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.messages; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/messages/NLS.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/messages/NLS.java index ad2b627..ce164b0 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/messages/NLS.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/messages/NLS.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.messages; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/messages/NLSException.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/messages/NLSException.java index 0e7a1e9..2ac4674 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/messages/NLSException.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/messages/NLSException.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.messages; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/precedence/PrecedenceQueryParser.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/precedence/PrecedenceQueryParser.java index 785ae1d..a795439 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/precedence/PrecedenceQueryParser.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/precedence/PrecedenceQueryParser.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.precedence; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/precedence/processors/BooleanModifiersQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/precedence/processors/BooleanModifiersQueryNodeProcessor.java index 754ba11..fe4fb13 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/precedence/processors/BooleanModifiersQueryNodeProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/precedence/processors/BooleanModifiersQueryNodeProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.precedence.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/precedence/processors/PrecedenceQueryNodeProcessorPipeline.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/precedence/processors/PrecedenceQueryNodeProcessorPipeline.java index 70be2a1..8b8837b 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/precedence/processors/PrecedenceQueryNodeProcessorPipeline.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/precedence/processors/PrecedenceQueryNodeProcessorPipeline.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.precedence.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/QueryParserUtil.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/QueryParserUtil.java index 73aa6bb..a7318f4 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/QueryParserUtil.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/QueryParserUtil.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/StandardQueryParser.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/StandardQueryParser.java index 18844c6..bd2235c 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/StandardQueryParser.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/StandardQueryParser.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/AnyQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/AnyQueryNodeBuilder.java index e07ab11..9e7c093 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/AnyQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/AnyQueryNodeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/BooleanQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/BooleanQueryNodeBuilder.java index d5435e9..da58d0c 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/BooleanQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/BooleanQueryNodeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -83,8 +83,7 @@ public class BooleanQueryNodeBuilder implements StandardQueryBuilder { } - private static BooleanClause.Occur getModifierValue(QueryNode node) - throws QueryNodeException { + private static BooleanClause.Occur getModifierValue(QueryNode node) { if (node instanceof ModifierQueryNode) { ModifierQueryNode mNode = ((ModifierQueryNode) node); diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/BoostQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/BoostQueryNodeBuilder.java index ee86c96..d45779b 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/BoostQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/BoostQueryNodeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/DummyQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/DummyQueryNodeBuilder.java index 1fc7f35..c6708de 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/DummyQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/DummyQueryNodeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/FieldQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/FieldQueryNodeBuilder.java index 738ecae..e97892a 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/FieldQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/FieldQueryNodeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/FuzzyQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/FuzzyQueryNodeBuilder.java index 4cc36d8..6908959 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/FuzzyQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/FuzzyQueryNodeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/GroupQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/GroupQueryNodeBuilder.java index 0e4ece9..d9e6cdc 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/GroupQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/GroupQueryNodeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/MatchAllDocsQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/MatchAllDocsQueryNodeBuilder.java index 4c9b3e3..bbcbb10 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/MatchAllDocsQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/MatchAllDocsQueryNodeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/MatchNoDocsQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/MatchNoDocsQueryNodeBuilder.java index 0a33f75..d913874 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/MatchNoDocsQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/MatchNoDocsQueryNodeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/ModifierQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/ModifierQueryNodeBuilder.java index 169d8bd..b2cbbc0 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/ModifierQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/ModifierQueryNodeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/MultiPhraseQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/MultiPhraseQueryNodeBuilder.java index 256bba6..0105d46 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/MultiPhraseQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/MultiPhraseQueryNodeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/NumericRangeQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/NumericRangeQueryNodeBuilder.java index cce60af..e5a4d92 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/NumericRangeQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/NumericRangeQueryNodeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/PhraseQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/PhraseQueryNodeBuilder.java index e5e2405..f502053 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/PhraseQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/PhraseQueryNodeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/PrefixWildcardQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/PrefixWildcardQueryNodeBuilder.java index f35f5e5..3b541ab 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/PrefixWildcardQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/PrefixWildcardQueryNodeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/RegexpQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/RegexpQueryNodeBuilder.java index 796ce2b..57de72c 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/RegexpQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/RegexpQueryNodeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/SlopQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/SlopQueryNodeBuilder.java index e580a0e..4b6f864 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/SlopQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/SlopQueryNodeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/StandardBooleanQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/StandardBooleanQueryNodeBuilder.java index 15e4cb4..ba91dc3 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/StandardBooleanQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/StandardBooleanQueryNodeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -84,8 +84,7 @@ public class StandardBooleanQueryNodeBuilder implements StandardQueryBuilder { } - private static BooleanClause.Occur getModifierValue(QueryNode node) - throws QueryNodeException { + private static BooleanClause.Occur getModifierValue(QueryNode node) { if (node instanceof ModifierQueryNode) { ModifierQueryNode mNode = ((ModifierQueryNode) node); diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/StandardQueryBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/StandardQueryBuilder.java index 6f8dca9..203c0a7 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/StandardQueryBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/StandardQueryBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/StandardQueryTreeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/StandardQueryTreeBuilder.java index 789a2fb..c0cd179 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/StandardQueryTreeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/StandardQueryTreeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/TermRangeQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/TermRangeQueryNodeBuilder.java index 6539e61..468b9f5 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/TermRangeQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/TermRangeQueryNodeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/WildcardQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/WildcardQueryNodeBuilder.java index bc5c7c8..e228439 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/WildcardQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/WildcardQueryNodeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/FieldBoostMapFCListener.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/FieldBoostMapFCListener.java index fd2b476..71f417a 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/FieldBoostMapFCListener.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/FieldBoostMapFCListener.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.config; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/FieldDateResolutionFCListener.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/FieldDateResolutionFCListener.java index 2cd2b1f..fc8bb08 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/FieldDateResolutionFCListener.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/FieldDateResolutionFCListener.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.config; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/FuzzyConfig.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/FuzzyConfig.java index 90596f7..892f534 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/FuzzyConfig.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/FuzzyConfig.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.config; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/NumberDateFormat.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/NumberDateFormat.java index 345876b..fd0bfe5 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/NumberDateFormat.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/NumberDateFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.config; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/NumericConfig.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/NumericConfig.java index af89d42..75ea68e 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/NumericConfig.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/NumericConfig.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.config; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/NumericFieldConfigListener.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/NumericFieldConfigListener.java index e3092b3..b6abdc3 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/NumericFieldConfigListener.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/NumericFieldConfigListener.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.config; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/StandardQueryConfigHandler.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/StandardQueryConfigHandler.java index 4b5dc2f..7f814dd 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/StandardQueryConfigHandler.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/StandardQueryConfigHandler.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.config; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/AbstractRangeQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/AbstractRangeQueryNode.java index fdc6dd5..201654c 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/AbstractRangeQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/AbstractRangeQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/BooleanModifierNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/BooleanModifierNode.java index 2f27bf8..1ad670a 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/BooleanModifierNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/BooleanModifierNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/MultiPhraseQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/MultiPhraseQueryNode.java index 5d399f3..e562063 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/MultiPhraseQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/MultiPhraseQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/NumericQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/NumericQueryNode.java index 90ce408..139da06 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/NumericQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/NumericQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/NumericRangeQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/NumericRangeQueryNode.java index 68d0037..ee154aa 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/NumericRangeQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/NumericRangeQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/PrefixWildcardQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/PrefixWildcardQueryNode.java index 7b5bd4e..ce3b3b1 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/PrefixWildcardQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/PrefixWildcardQueryNode.java @@ -2,7 +2,7 @@ package org.apache.lucene.queryparser.flexible.standard.nodes; import org.apache.lucene.queryparser.flexible.core.nodes.FieldQueryNode; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/RegexpQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/RegexpQueryNode.java index 5345fb6..efa0989 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/RegexpQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/RegexpQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/StandardBooleanQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/StandardBooleanQueryNode.java index 48d2187..07808bf 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/StandardBooleanQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/StandardBooleanQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/TermRangeQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/TermRangeQueryNode.java index b96e059..05583e0 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/TermRangeQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/TermRangeQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/WildcardQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/WildcardQueryNode.java index fc17961..2fae752 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/WildcardQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/WildcardQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/EscapeQuerySyntaxImpl.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/EscapeQuerySyntaxImpl.java index 6a457ee..5d36b3a 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/EscapeQuerySyntaxImpl.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/EscapeQuerySyntaxImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.parser; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.java index e017985..fd5a73c 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.java @@ -1,7 +1,7 @@ /* Generated By:JavaCC: Do not edit this line. StandardSyntaxParser.java */ package org.apache.lucene.queryparser.flexible.standard.parser; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParserTokenManager.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParserTokenManager.java index e7fc1cd..2097ea1 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParserTokenManager.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParserTokenManager.java @@ -1,6 +1,6 @@ /* Generated By:JavaCC: Do not edit this line. StandardSyntaxParserTokenManager.java */ package org.apache.lucene.queryparser.flexible.standard.parser; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/AllowLeadingWildcardProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/AllowLeadingWildcardProcessor.java index 830b494..c2009ab 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/AllowLeadingWildcardProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/AllowLeadingWildcardProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/AnalyzerQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/AnalyzerQueryNodeProcessor.java index dad5469..3f99355 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/AnalyzerQueryNodeProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/AnalyzerQueryNodeProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -42,6 +42,7 @@ import org.apache.lucene.queryparser.flexible.core.nodes.TokenizedPhraseQueryNod import org.apache.lucene.queryparser.flexible.core.processors.QueryNodeProcessorImpl; import org.apache.lucene.queryparser.flexible.standard.config.StandardQueryConfigHandler.ConfigurationKeys; import org.apache.lucene.queryparser.flexible.standard.nodes.MultiPhraseQueryNode; +import org.apache.lucene.queryparser.flexible.standard.nodes.RegexpQueryNode; import org.apache.lucene.queryparser.flexible.standard.nodes.StandardBooleanQueryNode; import org.apache.lucene.queryparser.flexible.standard.nodes.WildcardQueryNode; @@ -106,6 +107,7 @@ public class AnalyzerQueryNodeProcessor extends QueryNodeProcessorImpl { if (node instanceof TextableQueryNode && !(node instanceof WildcardQueryNode) && !(node instanceof FuzzyQueryNode) + && !(node instanceof RegexpQueryNode) && !(node.getParent() instanceof RangeQueryNode)) { FieldQueryNode fieldNode = ((FieldQueryNode) node); diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/BooleanSingleChildOptimizationQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/BooleanSingleChildOptimizationQueryNodeProcessor.java index 7588341..f6887cb 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/BooleanSingleChildOptimizationQueryNodeProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/BooleanSingleChildOptimizationQueryNodeProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/BoostQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/BoostQueryNodeProcessor.java index 264ba68..02387fd 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/BoostQueryNodeProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/BoostQueryNodeProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/DefaultPhraseSlopQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/DefaultPhraseSlopQueryNodeProcessor.java index 5f92a12..bea4675 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/DefaultPhraseSlopQueryNodeProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/DefaultPhraseSlopQueryNodeProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/FuzzyQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/FuzzyQueryNodeProcessor.java index f7daf5d..5b20742 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/FuzzyQueryNodeProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/FuzzyQueryNodeProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/GroupQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/GroupQueryNodeProcessor.java index 051df6a..723f4f8 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/GroupQueryNodeProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/GroupQueryNodeProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/LowercaseExpandedTermsQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/LowercaseExpandedTermsQueryNodeProcessor.java index 8ee9936..80bedeb 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/LowercaseExpandedTermsQueryNodeProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/LowercaseExpandedTermsQueryNodeProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/MatchAllDocsQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/MatchAllDocsQueryNodeProcessor.java index 7a19d01..f3c4ec1 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/MatchAllDocsQueryNodeProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/MatchAllDocsQueryNodeProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/MultiFieldQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/MultiFieldQueryNodeProcessor.java index 13c25b9..4dfaa79 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/MultiFieldQueryNodeProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/MultiFieldQueryNodeProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/MultiTermRewriteMethodProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/MultiTermRewriteMethodProcessor.java index 1f0fc72..f14b3bb 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/MultiTermRewriteMethodProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/MultiTermRewriteMethodProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/NumericQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/NumericQueryNodeProcessor.java index efea548..7c18eb1 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/NumericQueryNodeProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/NumericQueryNodeProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/NumericRangeQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/NumericRangeQueryNodeProcessor.java index 4fdb2b3..7710fbc 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/NumericRangeQueryNodeProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/NumericRangeQueryNodeProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/OpenRangeQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/OpenRangeQueryNodeProcessor.java index 4848b3f..ab2c349 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/OpenRangeQueryNodeProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/OpenRangeQueryNodeProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/PhraseSlopQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/PhraseSlopQueryNodeProcessor.java index 2a0ca6f..5d62fea 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/PhraseSlopQueryNodeProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/PhraseSlopQueryNodeProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/RemoveEmptyNonLeafQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/RemoveEmptyNonLeafQueryNodeProcessor.java index 852dd8e..addee51 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/RemoveEmptyNonLeafQueryNodeProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/RemoveEmptyNonLeafQueryNodeProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/StandardQueryNodeProcessorPipeline.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/StandardQueryNodeProcessorPipeline.java index bd8a29c..7915f93 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/StandardQueryNodeProcessorPipeline.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/StandardQueryNodeProcessorPipeline.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/TermRangeQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/TermRangeQueryNodeProcessor.java index 7220b1f..caa0936 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/TermRangeQueryNodeProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/TermRangeQueryNodeProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/WildcardQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/WildcardQueryNodeProcessor.java index e572fb9..b9fe6c0 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/WildcardQueryNodeProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/WildcardQueryNodeProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard.processors; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/FastCharStream.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/FastCharStream.java index e035e61..4116eec 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/FastCharStream.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/FastCharStream.java @@ -1,6 +1,6 @@ // FastCharStream.java package org.apache.lucene.queryparser.surround.parser; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/AndQuery.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/AndQuery.java index 08583cf..e6b24b4 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/AndQuery.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/AndQuery.java @@ -1,5 +1,5 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/BasicQueryFactory.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/BasicQueryFactory.java index 08ff4f7..c328b20 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/BasicQueryFactory.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/BasicQueryFactory.java @@ -1,5 +1,5 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/ComposedQuery.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/ComposedQuery.java index db51d3d..dbaee65 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/ComposedQuery.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/ComposedQuery.java @@ -1,5 +1,5 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/DistanceQuery.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/DistanceQuery.java index 6d400b3..ce1e1fb 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/DistanceQuery.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/DistanceQuery.java @@ -1,5 +1,5 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/DistanceRewriteQuery.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/DistanceRewriteQuery.java index 9c1129d..9110e32 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/DistanceRewriteQuery.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/DistanceRewriteQuery.java @@ -1,5 +1,5 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/DistanceSubQuery.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/DistanceSubQuery.java index 8460b5e..9b37934 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/DistanceSubQuery.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/DistanceSubQuery.java @@ -1,5 +1,5 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/FieldsQuery.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/FieldsQuery.java index bcffd7c..1d31a66 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/FieldsQuery.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/FieldsQuery.java @@ -1,5 +1,5 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/NotQuery.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/NotQuery.java index c724be1..3256295 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/NotQuery.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/NotQuery.java @@ -1,5 +1,5 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/OrQuery.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/OrQuery.java index bafa7db..808cfb9 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/OrQuery.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/OrQuery.java @@ -1,5 +1,5 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/RewriteQuery.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/RewriteQuery.java index 64e53fb..73a4909 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/RewriteQuery.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/RewriteQuery.java @@ -1,5 +1,5 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SimpleTerm.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SimpleTerm.java index 6a36361..1c2e904 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SimpleTerm.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SimpleTerm.java @@ -1,5 +1,5 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SimpleTermRewriteQuery.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SimpleTermRewriteQuery.java index 963db84..e7dedd2 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SimpleTermRewriteQuery.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SimpleTermRewriteQuery.java @@ -1,5 +1,5 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SpanNearClauseFactory.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SpanNearClauseFactory.java index 264cd73..f11c800 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SpanNearClauseFactory.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SpanNearClauseFactory.java @@ -1,5 +1,5 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndBooleanQuery.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndBooleanQuery.java index fc836af..0a9cabb 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndBooleanQuery.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndBooleanQuery.java @@ -1,5 +1,5 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndPrefixQuery.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndPrefixQuery.java index 16ea64d..31e8f3a 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndPrefixQuery.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndPrefixQuery.java @@ -1,5 +1,5 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndQuery.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndQuery.java index ff02042..e9e0b94 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndQuery.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndQuery.java @@ -1,5 +1,5 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -73,7 +73,7 @@ public abstract class SrndQuery implements Cloneable { /** For subclasses of {@link SrndQuery} within the package * {@link org.apache.lucene.queryparser.surround.query} * it is not necessary to override this method, - * @see #toString(). + * @see #toString() */ @Override public int hashCode() { @@ -83,7 +83,7 @@ public abstract class SrndQuery implements Cloneable { /** For subclasses of {@link SrndQuery} within the package * {@link org.apache.lucene.queryparser.surround.query} * it is not necessary to override this method, - * @see #toString(). + * @see #toString() */ @Override public boolean equals(Object obj) { diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndTermQuery.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndTermQuery.java index a9ff4b7..fbaddda 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndTermQuery.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndTermQuery.java @@ -1,5 +1,5 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndTruncQuery.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndTruncQuery.java index c40d798..e2c9738 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndTruncQuery.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/SrndTruncQuery.java @@ -1,5 +1,5 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/TooManyBasicQueries.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/TooManyBasicQueries.java index c7080d6..d39b34a 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/TooManyBasicQueries.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/TooManyBasicQueries.java @@ -1,5 +1,5 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/CoreParser.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/CoreParser.java index bcbccd6..ae200c5 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/CoreParser.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/CoreParser.java @@ -11,7 +11,7 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import java.io.InputStream; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/CorePlusExtensionsParser.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/CorePlusExtensionsParser.java index 0676b55..2cf7306 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/CorePlusExtensionsParser.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/CorePlusExtensionsParser.java @@ -4,7 +4,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.queryparser.xml.builders.*; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/DOMUtils.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/DOMUtils.java index 5b30b00..b3f5caa 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/DOMUtils.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/DOMUtils.java @@ -8,7 +8,7 @@ import org.xml.sax.InputSource; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import java.io.Reader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/FilterBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/FilterBuilder.java index 691cfe5..11538b4 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/FilterBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/FilterBuilder.java @@ -5,7 +5,7 @@ package org.apache.lucene.queryparser.xml; import org.apache.lucene.search.Filter; import org.w3c.dom.Element; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/FilterBuilderFactory.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/FilterBuilderFactory.java index 30677d1..694dea1 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/FilterBuilderFactory.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/FilterBuilderFactory.java @@ -7,7 +7,7 @@ import org.apache.lucene.search.Filter; import org.w3c.dom.Element; import java.util.HashMap; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/ParserException.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/ParserException.java index 8ef33c7..63f2af9 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/ParserException.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/ParserException.java @@ -2,7 +2,7 @@ * Created on 25-Jan-2006 */ package org.apache.lucene.queryparser.xml; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/QueryBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/QueryBuilder.java index 9008265..8e676d7 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/QueryBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/QueryBuilder.java @@ -2,7 +2,7 @@ package org.apache.lucene.queryparser.xml; import org.apache.lucene.search.Query; import org.w3c.dom.Element; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/QueryBuilderFactory.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/QueryBuilderFactory.java index 0fab708..3590415 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/QueryBuilderFactory.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/QueryBuilderFactory.java @@ -7,7 +7,7 @@ import org.apache.lucene.search.Query; import org.w3c.dom.Element; import java.util.HashMap; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/QueryTemplateManager.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/QueryTemplateManager.java index 12525ff..8cc48de 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/QueryTemplateManager.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/QueryTemplateManager.java @@ -18,7 +18,7 @@ import java.util.Enumeration; import java.util.HashMap; import java.util.Properties; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -97,7 +97,7 @@ public class QueryTemplateManager { * Fast means of constructing query using a precompiled stylesheet */ public static String getQueryAsXmlString(Properties formProperties, Templates template) - throws SAXException, IOException, ParserConfigurationException, TransformerException { + throws ParserConfigurationException, TransformerException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); StreamResult result = new StreamResult(baos); transformCriteria(formProperties, template, result); @@ -120,7 +120,7 @@ public class QueryTemplateManager { * Fast means of constructing query using a cached,precompiled stylesheet */ public static Document getQueryAsDOM(Properties formProperties, Templates template) - throws SAXException, IOException, ParserConfigurationException, TransformerException { + throws ParserConfigurationException, TransformerException { DOMResult result = new DOMResult(); transformCriteria(formProperties, template, result); return (Document) result.getNode(); @@ -159,13 +159,13 @@ public class QueryTemplateManager { * Fast transformation using a pre-compiled stylesheet (suitable for production environments) */ public static void transformCriteria(Properties formProperties, Templates template, Result result) - throws SAXException, IOException, ParserConfigurationException, TransformerException { + throws ParserConfigurationException, TransformerException { transformCriteria(formProperties, template.newTransformer(), result); } public static void transformCriteria(Properties formProperties, Transformer transformer, Result result) - throws SAXException, IOException, ParserConfigurationException, TransformerException { + throws ParserConfigurationException, TransformerException { dbf.setNamespaceAware(true); //Create an XML document representing the search index document. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/BooleanFilterBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/BooleanFilterBuilder.java index 36f153a..98f13b2 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/BooleanFilterBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/BooleanFilterBuilder.java @@ -14,7 +14,7 @@ import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/BooleanQueryBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/BooleanQueryBuilder.java index 53f64a7..59f1d7d 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/BooleanQueryBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/BooleanQueryBuilder.java @@ -12,7 +12,7 @@ import org.apache.lucene.queryparser.xml.QueryBuilder; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/BoostingQueryBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/BoostingQueryBuilder.java index 6d4725c..92596d6 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/BoostingQueryBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/BoostingQueryBuilder.java @@ -6,7 +6,7 @@ import org.apache.lucene.queryparser.xml.DOMUtils; import org.apache.lucene.queryparser.xml.ParserException; import org.apache.lucene.queryparser.xml.QueryBuilder; import org.w3c.dom.Element; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/BoostingTermBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/BoostingTermBuilder.java index d5ebee9..49b9dfd 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/BoostingTermBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/BoostingTermBuilder.java @@ -8,7 +8,7 @@ import org.apache.lucene.queryparser.xml.DOMUtils; import org.apache.lucene.queryparser.xml.ParserException; import org.w3c.dom.Element; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/CachedFilterBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/CachedFilterBuilder.java index 246138a..06f7fe5 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/CachedFilterBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/CachedFilterBuilder.java @@ -11,7 +11,7 @@ import org.apache.lucene.search.QueryWrapperFilter; import org.w3c.dom.Element; import java.util.Map; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/ConstantScoreQueryBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/ConstantScoreQueryBuilder.java index 0a24b45..2ca21ef 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/ConstantScoreQueryBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/ConstantScoreQueryBuilder.java @@ -7,7 +7,7 @@ import org.apache.lucene.queryparser.xml.FilterBuilderFactory; import org.apache.lucene.queryparser.xml.ParserException; import org.apache.lucene.queryparser.xml.QueryBuilder; import org.w3c.dom.Element; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/DisjunctionMaxQueryBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/DisjunctionMaxQueryBuilder.java index 22c842e..e69916d 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/DisjunctionMaxQueryBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/DisjunctionMaxQueryBuilder.java @@ -1,5 +1,5 @@ package org.apache.lucene.queryparser.xml.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/DuplicateFilterBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/DuplicateFilterBuilder.java index 5537511..69cd6d2 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/DuplicateFilterBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/DuplicateFilterBuilder.java @@ -10,7 +10,7 @@ import org.apache.lucene.sandbox.queries.DuplicateFilter; import org.apache.lucene.search.Filter; import org.w3c.dom.Element; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/FilteredQueryBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/FilteredQueryBuilder.java index 27c4d7d..16c8e32 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/FilteredQueryBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/FilteredQueryBuilder.java @@ -12,7 +12,7 @@ import org.apache.lucene.queryparser.xml.ParserException; import org.apache.lucene.queryparser.xml.QueryBuilder; import org.w3c.dom.Element; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/FuzzyLikeThisQueryBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/FuzzyLikeThisQueryBuilder.java index 42eb1f2..fa4976e 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/FuzzyLikeThisQueryBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/FuzzyLikeThisQueryBuilder.java @@ -10,7 +10,7 @@ import org.apache.lucene.search.Query; import org.w3c.dom.Element; import org.w3c.dom.NodeList; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LikeThisQueryBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LikeThisQueryBuilder.java index b0312b8..7fd8058 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LikeThisQueryBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LikeThisQueryBuilder.java @@ -17,7 +17,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.queryparser.xml.DOMUtils; import org.apache.lucene.queryparser.xml.ParserException; import org.w3c.dom.Element; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/MatchAllDocsQueryBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/MatchAllDocsQueryBuilder.java index 4295248..2f25b95 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/MatchAllDocsQueryBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/MatchAllDocsQueryBuilder.java @@ -5,7 +5,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.queryparser.xml.ParserException; import org.apache.lucene.queryparser.xml.QueryBuilder; import org.w3c.dom.Element; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/NumericRangeFilterBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/NumericRangeFilterBuilder.java index bc8484f..c7b46c0 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/NumericRangeFilterBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/NumericRangeFilterBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.xml.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/NumericRangeQueryBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/NumericRangeQueryBuilder.java index 17a1b6c..b3db8be 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/NumericRangeQueryBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/NumericRangeQueryBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.xml.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/RangeFilterBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/RangeFilterBuilder.java index 2744be6..ac5a767 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/RangeFilterBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/RangeFilterBuilder.java @@ -9,7 +9,7 @@ import org.apache.lucene.queryparser.xml.DOMUtils; import org.apache.lucene.queryparser.xml.FilterBuilder; import org.apache.lucene.queryparser.xml.ParserException; import org.w3c.dom.Element; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanBuilderBase.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanBuilderBase.java index 6487709..86eaaad 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanBuilderBase.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanBuilderBase.java @@ -3,7 +3,7 @@ package org.apache.lucene.queryparser.xml.builders; import org.apache.lucene.search.Query; import org.apache.lucene.queryparser.xml.ParserException; import org.w3c.dom.Element; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanFirstBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanFirstBuilder.java index acd604a..96beed9 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanFirstBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanFirstBuilder.java @@ -5,7 +5,7 @@ import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.queryparser.xml.DOMUtils; import org.apache.lucene.queryparser.xml.ParserException; import org.w3c.dom.Element; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanNearBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanNearBuilder.java index 6d15938..a7a20da 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanNearBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanNearBuilder.java @@ -9,7 +9,7 @@ import org.w3c.dom.Node; import java.util.ArrayList; import java.util.List; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanNotBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanNotBuilder.java index 7a76c75..1746036 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanNotBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanNotBuilder.java @@ -5,7 +5,7 @@ import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.queryparser.xml.DOMUtils; import org.apache.lucene.queryparser.xml.ParserException; import org.w3c.dom.Element; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrBuilder.java index 527be75..e70e093 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrBuilder.java @@ -9,7 +9,7 @@ import org.w3c.dom.Node; import java.util.ArrayList; import java.util.List; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrTermsBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrTermsBuilder.java index 87fd8c8..0c5cdca 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrTermsBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrTermsBuilder.java @@ -16,7 +16,7 @@ import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; import java.util.List; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanQueryBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanQueryBuilder.java index fc27e48..3ba23fd 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanQueryBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanQueryBuilder.java @@ -1,5 +1,5 @@ package org.apache.lucene.queryparser.xml.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanQueryBuilderFactory.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanQueryBuilderFactory.java index b37491f..1f2b2cc 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanQueryBuilderFactory.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanQueryBuilderFactory.java @@ -7,7 +7,7 @@ import org.w3c.dom.Element; import java.util.HashMap; import java.util.Map; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanTermBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanTermBuilder.java index b9f97af..2ae6916 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanTermBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanTermBuilder.java @@ -6,7 +6,7 @@ import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.queryparser.xml.DOMUtils; import org.apache.lucene.queryparser.xml.ParserException; import org.w3c.dom.Element; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermQueryBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermQueryBuilder.java index 6b9ad73..bf0c110 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermQueryBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermQueryBuilder.java @@ -7,7 +7,7 @@ import org.apache.lucene.queryparser.xml.DOMUtils; import org.apache.lucene.queryparser.xml.ParserException; import org.apache.lucene.queryparser.xml.QueryBuilder; import org.w3c.dom.Element; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsFilterBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsFilterBuilder.java index ca60541..919c990 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsFilterBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsFilterBuilder.java @@ -15,7 +15,7 @@ import org.w3c.dom.Element; import java.io.IOException; import java.io.StringReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsQueryBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsQueryBuilder.java index 6532d6f..0fcaf82 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsQueryBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsQueryBuilder.java @@ -17,7 +17,7 @@ import org.w3c.dom.Element; import java.io.IOException; import java.io.StringReader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/UserInputQueryBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/UserInputQueryBuilder.java index ae68331..58c3d1b 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/UserInputQueryBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/UserInputQueryBuilder.java @@ -10,7 +10,7 @@ import org.apache.lucene.queryparser.xml.ParserException; import org.apache.lucene.queryparser.xml.QueryBuilder; import org.w3c.dom.Element; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java index d8edb18..a3d41a6 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.analyzing; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,7 +22,16 @@ import java.io.Reader; import org.apache.lucene.analysis.*; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; /** @@ -138,5 +147,28 @@ public class TestAnalyzingQueryParser extends LuceneTestCase { Tokenizer result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true); return new TokenStreamComponents(result, new FoldingFilter(result)); } - } + } + + // LUCENE-4176 + public void testByteTerms() throws Exception { + Directory ramDir = newDirectory(); + Analyzer analyzer = new MockBytesAnalyzer(); + RandomIndexWriter writer = new RandomIndexWriter(random(), ramDir, analyzer); + Document doc = new Document(); + FieldType fieldType = new FieldType(); + fieldType.setIndexed(true); + fieldType.setTokenized(true); + fieldType.setStored(true); + Field field = new Field("content","เข", fieldType); + doc.add(field); + writer.addDocument(doc); + writer.close(); + DirectoryReader ir = DirectoryReader.open(ramDir); + IndexSearcher is = new IndexSearcher(ir); + QueryParser qp = new AnalyzingQueryParser(TEST_VERSION_CURRENT, "content", analyzer); + Query q = qp.parse("[เข TO เข]"); + assertEquals(1, is.search(q, 10).totalHits); + ir.close(); + ramDir.close(); + } } \ No newline at end of file diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiAnalyzer.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiAnalyzer.java index 4451165..cbfabfe 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiAnalyzer.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiAnalyzer.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.classic; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java index ce4b5c9..7ea5696 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.classic; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,7 +23,8 @@ import java.util.Map; import org.apache.lucene.analysis.*; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.search.BooleanClause.Occur; @@ -282,7 +283,7 @@ public class TestMultiFieldQueryParser extends LuceneTestCase { Directory ramDir = newDirectory(); IndexWriter iw = new IndexWriter(ramDir, newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer)); Document doc = new Document(); - doc.add(newField("body", "blah the footest blah", TextField.TYPE_UNSTORED)); + doc.add(newTextField("body", "blah the footest blah", Field.Store.NO)); iw.addDocument(doc); iw.close(); @@ -290,7 +291,7 @@ public class TestMultiFieldQueryParser extends LuceneTestCase { new MultiFieldQueryParser(TEST_VERSION_CURRENT, new String[] {"body"}, analyzer); mfqp.setDefaultOperator(QueryParser.Operator.AND); Query q = mfqp.parse("the footest"); - IndexReader ir = IndexReader.open(ramDir); + IndexReader ir = DirectoryReader.open(ramDir); IndexSearcher is = new IndexSearcher(ir); ScoreDoc[] hits = is.search(q, null, 1000).scoreDocs; assertEquals(1, hits.length); diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiPhraseQueryParsing.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiPhraseQueryParsing.java index d14733d..b8a7644 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiPhraseQueryParsing.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiPhraseQueryParsing.java @@ -18,7 +18,6 @@ package org.apache.lucene.queryparser.classic; */ import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; @@ -67,7 +66,7 @@ public class TestMultiPhraseQueryParsing extends LuceneTestCase { } @Override - public final boolean incrementToken() throws IOException { + public final boolean incrementToken() { clearAttributes(); if (upto < tokens.length) { final TokenAndPos token = tokens[upto++]; diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java index 7d1992e..081d696 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.classic; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/complexPhrase/TestComplexPhraseQuery.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/complexPhrase/TestComplexPhraseQuery.java index 25c0efb..23ec531 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/complexPhrase/TestComplexPhraseQuery.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/complexPhrase/TestComplexPhraseQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.complexPhrase; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,7 +22,8 @@ import java.util.HashSet; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.queryparser.classic.QueryParser; @@ -119,12 +120,12 @@ public class TestComplexPhraseQuery extends LuceneTestCase { IndexWriter w = new IndexWriter(rd, newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer)); for (int i = 0; i < docsContent.length; i++) { Document doc = new Document(); - doc.add(newField("name", docsContent[i].name, TextField.TYPE_STORED)); - doc.add(newField("id", docsContent[i].id, TextField.TYPE_STORED)); + doc.add(newTextField("name", docsContent[i].name, Field.Store.YES)); + doc.add(newTextField("id", docsContent[i].id, Field.Store.YES)); w.addDocument(doc); } w.close(); - reader = IndexReader.open(rd); + reader = DirectoryReader.open(rd); searcher = new IndexSearcher(reader); } diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/ext/ExtensionStub.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/ext/ExtensionStub.java index 7285b08..1fe3eec 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/ext/ExtensionStub.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/ext/ExtensionStub.java @@ -5,7 +5,7 @@ import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/ext/TestExtendableQueryParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/ext/TestExtendableQueryParser.java index cb52c0a..cf1bc56 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/ext/TestExtendableQueryParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/ext/TestExtendableQueryParser.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.ext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/ext/TestExtensions.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/ext/TestExtensions.java index c590ecb..81af93a 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/ext/TestExtensions.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/ext/TestExtensions.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.ext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/core/builders/TestQueryTreeBuilder.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/core/builders/TestQueryTreeBuilder.java index 9a40c70..be49266 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/core/builders/TestQueryTreeBuilder.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/core/builders/TestQueryTreeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/core/nodes/TestQueryNode.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/core/nodes/TestQueryNode.java index 53caba8..f3b2e8d 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/core/nodes/TestQueryNode.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/core/nodes/TestQueryNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.core.nodes; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/messages/MessagesTestBundle.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/messages/MessagesTestBundle.java index e56f4e9..8a6627a 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/messages/MessagesTestBundle.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/messages/MessagesTestBundle.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.messages; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/messages/TestNLS.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/messages/TestNLS.java index 7cf4d9a..2ee16ff 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/messages/TestNLS.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/messages/TestNLS.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.messages; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java index e558ad2..8926a4a 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.precedence; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -446,7 +446,7 @@ public class TestPrecedenceQueryParser extends LuceneTestCase { } /** for testing DateTools support */ - private String getDate(Date d, DateTools.Resolution resolution) throws Exception { + private String getDate(Date d, DateTools.Resolution resolution) { return DateTools.dateToString(d, resolution); } diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/SpanOrQueryNodeBuilder.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/SpanOrQueryNodeBuilder.java index 75fcbf4..9c644a1 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/SpanOrQueryNodeBuilder.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/SpanOrQueryNodeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/SpanTermQueryNodeBuilder.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/SpanTermQueryNodeBuilder.java index 6fb7e10..c1c57fa 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/SpanTermQueryNodeBuilder.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/SpanTermQueryNodeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/SpansQueryConfigHandler.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/SpansQueryConfigHandler.java index 96ccd80..853c429 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/SpansQueryConfigHandler.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/SpansQueryConfigHandler.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/SpansQueryTreeBuilder.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/SpansQueryTreeBuilder.java index e8711a5..e344c86 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/SpansQueryTreeBuilder.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/SpansQueryTreeBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/SpansValidatorQueryNodeProcessor.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/SpansValidatorQueryNodeProcessor.java index 965aaa7..501dc22 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/SpansValidatorQueryNodeProcessor.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/SpansValidatorQueryNodeProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/TestSpanQueryParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/TestSpanQueryParser.java index 5d5da85..57d0534 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/TestSpanQueryParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/TestSpanQueryParser.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/TestSpanQueryParserSimpleSample.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/TestSpanQueryParserSimpleSample.java index 24e7e1c..08706fc 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/TestSpanQueryParserSimpleSample.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/TestSpanQueryParserSimpleSample.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/UniqueFieldAttribute.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/UniqueFieldAttribute.java index c4f873b..45225a7 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/UniqueFieldAttribute.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/UniqueFieldAttribute.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/UniqueFieldAttributeImpl.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/UniqueFieldAttributeImpl.java index 50c7571..021cc1a 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/UniqueFieldAttributeImpl.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/UniqueFieldAttributeImpl.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/UniqueFieldQueryNodeProcessor.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/UniqueFieldQueryNodeProcessor.java index e4ecba3..10563ea 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/UniqueFieldQueryNodeProcessor.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/spans/UniqueFieldQueryNodeProcessor.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.spans; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiAnalyzerQPHelper.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiAnalyzerQPHelper.java index 7b1d49b..dd7d110 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiAnalyzerQPHelper.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiAnalyzerQPHelper.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java index bb99a37..3a6f788 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,15 +17,14 @@ package org.apache.lucene.queryparser.flexible.standard; * limitations under the License. */ -import java.io.IOException; import java.io.Reader; -import java.io.StringReader; import java.util.HashMap; import java.util.Map; import org.apache.lucene.analysis.*; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.queryparser.flexible.core.QueryNodeException; @@ -321,7 +320,7 @@ public class TestMultiFieldQPHelper extends LuceneTestCase { Directory ramDir = newDirectory(); IndexWriter iw = new IndexWriter(ramDir, newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer)); Document doc = new Document(); - doc.add(newField("body", "blah the footest blah", TextField.TYPE_UNSTORED)); + doc.add(newTextField("body", "blah the footest blah", Field.Store.NO)); iw.addDocument(doc); iw.close(); @@ -331,7 +330,7 @@ public class TestMultiFieldQPHelper extends LuceneTestCase { mfqp.setAnalyzer(analyzer); mfqp.setDefaultOperator(StandardQueryConfigHandler.Operator.AND); Query q = mfqp.parse("the footest", null); - IndexReader ir = IndexReader.open(ramDir); + IndexReader ir = DirectoryReader.open(ramDir); IndexSearcher is = new IndexSearcher(ir); ScoreDoc[] hits = is.search(q, null, 1000).scoreDocs; assertEquals(1, hits.length); diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestNumericQueryParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestNumericQueryParser.java index 2e7388d..082db96 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestNumericQueryParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestNumericQueryParser.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -88,10 +88,13 @@ public class TestNumericQueryParser extends LuceneTestCase { private static IndexReader reader = null; private static IndexSearcher searcher = null; - private static boolean checkDateFormatSanity(DateFormat dateFormat, long date) - throws ParseException { - return date == dateFormat.parse(dateFormat.format(new Date(date))) + private static boolean checkDateFormatSanity(DateFormat dateFormat, long date) { + try { + return date == dateFormat.parse(dateFormat.format(new Date(date))) .getTime(); + } catch (ParseException e) { + return false; + } } @BeforeClass @@ -199,7 +202,7 @@ public class TestNumericQueryParser extends LuceneTestCase { numericConfigMap.put(type.name(), new NumericConfig(PRECISION_STEP, NUMBER_FORMAT, type)); - FieldType ft = new FieldType(IntField.TYPE); + FieldType ft = new FieldType(IntField.TYPE_NOT_STORED); ft.setNumericType(type); ft.setStored(true); ft.setNumericPrecisionStep(PRECISION_STEP); @@ -229,7 +232,7 @@ public class TestNumericQueryParser extends LuceneTestCase { numericConfigMap.put(DATE_FIELD_NAME, new NumericConfig(PRECISION_STEP, DATE_FORMAT, NumericType.LONG)); - FieldType ft = new FieldType(LongField.TYPE); + FieldType ft = new FieldType(LongField.TYPE_NOT_STORED); ft.setStored(true); ft.setNumericPrecisionStep(PRECISION_STEP); LongField dateField = new LongField(DATE_FIELD_NAME, 0l, ft); diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java index 91d25f8..4f66aa8 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.flexible.standard; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,7 +19,6 @@ package org.apache.lucene.queryparser.flexible.standard; import java.io.IOException; import java.io.Reader; -import java.io.StringReader; import java.text.DateFormat; import java.util.Calendar; import java.util.Date; @@ -35,7 +34,8 @@ import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.document.DateTools; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; @@ -680,8 +680,7 @@ public class TestQPHelper extends LuceneTestCase { } /** for testing DateTools support */ - private String getDate(Date d, DateTools.Resolution resolution) - throws Exception { + private String getDate(Date d, DateTools.Resolution resolution) { return DateTools.dateToString(d, resolution); } @@ -1288,9 +1287,9 @@ public class TestQPHelper extends LuceneTestCase { Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new CannedAnalyzer())); Document doc = new Document(); - doc.add(newField("field", "", TextField.TYPE_UNSTORED)); + doc.add(newTextField("field", "", Field.Store.NO)); w.addDocument(doc); - IndexReader r = IndexReader.open(w, true); + IndexReader r = DirectoryReader.open(w, true); IndexSearcher s = newSearcher(r); Query q = new StandardQueryParser(new CannedAnalyzer()).parse("\"a\"", "field"); @@ -1301,4 +1300,24 @@ public class TestQPHelper extends LuceneTestCase { dir.close(); } + public void testRegexQueryParsing() throws Exception { + final String[] fields = {"b", "t"}; + + final StandardQueryParser parser = new StandardQueryParser(); + parser.setMultiFields(fields); + parser.setDefaultOperator(StandardQueryConfigHandler.Operator.AND); + parser.setAnalyzer(new MockAnalyzer(random())); + + BooleanQuery exp = new BooleanQuery(); + exp.add(new BooleanClause(new RegexpQuery(new Term("b", "ab.+")), BooleanClause.Occur.MUST)); + exp.add(new BooleanClause(new RegexpQuery(new Term("t", "ab.+")), BooleanClause.Occur.MUST)); + + assertEquals(exp, parser.parse("/ab.+/", null)); + + RegexpQuery regexpQueryexp = new RegexpQuery(new Term("test", "[abc]?[0-9]")); + + assertEquals(regexpQueryexp, parser.parse("test:/[abc]?[0-9]/", null)); + + } + } diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/BooleanQueryTst.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/BooleanQueryTst.java index 81fbf41..35a402d 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/BooleanQueryTst.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/BooleanQueryTst.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,6 +20,7 @@ package org.apache.lucene.queryparser.surround.query; import java.io.IOException; import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Collector; @@ -122,7 +123,7 @@ public class BooleanQueryTst { /* if (verbose) System.out.println("Lucene: " + query.toString()); */ TestCollector tc = new TestCollector(); - IndexReader reader = IndexReader.open(dBase.getDb()); + IndexReader reader = DirectoryReader.open(dBase.getDb()); IndexSearcher searcher = new IndexSearcher(reader); try { searcher.search(query, tc); diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/ExceptionQueryTst.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/ExceptionQueryTst.java index 2d2c695..99e2e0c 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/ExceptionQueryTst.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/ExceptionQueryTst.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/SingleFieldTestDb.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/SingleFieldTestDb.java index 32a573b..586eba1 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/SingleFieldTestDb.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/SingleFieldTestDb.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,6 +19,7 @@ package org.apache.lucene.queryparser.surround.query; import java.util.Random; +import org.apache.lucene.document.Field; import org.apache.lucene.store.Directory; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.store.RAMDirectory; @@ -44,7 +45,7 @@ public class SingleFieldTestDb { new MockAnalyzer(random))); for (int j = 0; j < docs.length; j++) { Document d = new Document(); - d.add(new TextField(fieldName, docs[j])); + d.add(new TextField(fieldName, docs[j], Field.Store.NO)); writer.addDocument(d); } writer.close(); diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/Test01Exceptions.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/Test01Exceptions.java index fe5e3ab..3e79d22 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/Test01Exceptions.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/Test01Exceptions.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/Test02Boolean.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/Test02Boolean.java index a0e4639..5ca4fbf 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/Test02Boolean.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/Test02Boolean.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/Test03Distance.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/Test03Distance.java index cf22ec1..b14e590 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/Test03Distance.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/surround/query/Test03Distance.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.surround.query; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java index bb79d1e..431ebc0 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -31,7 +31,8 @@ import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.document.DateTools; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; @@ -585,7 +586,7 @@ public abstract class QueryParserTestBase extends LuceneTestCase { } /** for testing DateTools support */ - private String getDate(Date d, DateTools.Resolution resolution) throws Exception { + private String getDate(Date d, DateTools.Resolution resolution) { return DateTools.dateToString(d, resolution); } @@ -935,13 +936,13 @@ public abstract class QueryParserTestBase extends LuceneTestCase { final int[] type = new int[1]; QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)) { @Override - protected Query getWildcardQuery(String field, String termStr) throws ParseException { + protected Query getWildcardQuery(String field, String termStr) { // override error checking of superclass type[0]=1; return new TermQuery(new Term(field,termStr)); } @Override - protected Query getPrefixQuery(String field, String termStr) throws ParseException { + protected Query getPrefixQuery(String field, String termStr) { // override error checking of superclass type[0]=2; return new TermQuery(new Term(field,termStr)); @@ -1110,9 +1111,9 @@ public abstract class QueryParserTestBase extends LuceneTestCase { Analyzer a = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, a)); Document doc = new Document(); - doc.add(newField("f", "the wizard of ozzy", TextField.TYPE_UNSTORED)); + doc.add(newTextField("f", "the wizard of ozzy", Field.Store.NO)); w.addDocument(doc); - IndexReader r = IndexReader.open(w, true); + IndexReader r = DirectoryReader.open(w, true); w.close(); IndexSearcher s = newSearcher(r); QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "f", a); diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java index e1b5c16..e087f64 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.xml; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,9 +22,10 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenFilter; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.IntField; -import org.apache.lucene.document.TextField; import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.search.DisjunctionMaxQuery; @@ -43,6 +44,7 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.util.List; public class TestParser extends LuceneTestCase { @@ -59,7 +61,8 @@ public class TestParser extends LuceneTestCase { //initialize the parser builder = new CorePlusExtensionsParser("contents", analyzer); - BufferedReader d = new BufferedReader(new InputStreamReader(TestParser.class.getResourceAsStream("reuters21578.txt"))); + BufferedReader d = new BufferedReader(new InputStreamReader( + TestParser.class.getResourceAsStream("reuters21578.txt"), "US-ASCII")); dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(Version.LUCENE_40, analyzer)); String line = d.readLine(); @@ -68,15 +71,15 @@ public class TestParser extends LuceneTestCase { String date = line.substring(0, endOfDate).trim(); String content = line.substring(endOfDate).trim(); Document doc = new Document(); - doc.add(newField("date", date, TextField.TYPE_STORED)); - doc.add(newField("contents", content, TextField.TYPE_STORED)); - doc.add(new IntField("date2", Integer.valueOf(date))); + doc.add(newTextField("date", date, Field.Store.YES)); + doc.add(newTextField("contents", content, Field.Store.YES)); + doc.add(new IntField("date2", Integer.valueOf(date), Field.Store.NO)); writer.addDocument(doc); line = d.readLine(); } d.close(); writer.close(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); searcher = newSearcher(reader); } @@ -193,8 +196,8 @@ public class TestParser extends LuceneTestCase { } public void testDuplicateFilterQueryXML() throws ParserException, IOException { - AtomicReaderContext leaves[] = searcher.getTopReaderContext().leaves(); - Assume.assumeTrue(leaves == null || leaves.length == 1); + List leaves = searcher.getTopReaderContext().leaves(); + Assume.assumeTrue(leaves.size() == 1); Query q = parse("DuplicateFilterQuery.xml"); int h = searcher.search(q, null, 1000).totalHits; assertEquals("DuplicateFilterQuery should produce 1 result ", 1, h); diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestQueryTemplateManager.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestQueryTemplateManager.java index 868880a..3429c4f 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestQueryTemplateManager.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestQueryTemplateManager.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.xml; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,7 +19,8 @@ package org.apache.lucene.queryparser.xml; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.search.IndexSearcher; @@ -128,7 +129,7 @@ public class TestQueryTemplateManager extends LuceneTestCase { String name = st.nextToken().trim(); if (st.hasMoreTokens()) { String value = st.nextToken().trim(); - result.add(newField(name, value, TextField.TYPE_STORED)); + result.add(newTextField(name, value, Field.Store.YES)); } } return result; @@ -150,7 +151,7 @@ public class TestQueryTemplateManager extends LuceneTestCase { } w.forceMerge(1); w.close(); - reader = IndexReader.open(dir); + reader = DirectoryReader.open(dir); searcher = new IndexSearcher(reader); //initialize the parser diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java index 1c0a59a..61656d2 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.xml.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,6 +18,7 @@ package org.apache.lucene.queryparser.xml.builders; */ import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.SlowCompositeReaderWrapper; @@ -63,7 +64,7 @@ public class TestNumericRangeFilterBuilder extends LuceneTestCase { IndexWriter writer = new IndexWriter(ramDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null)); writer.commit(); try { - AtomicReader reader = new SlowCompositeReaderWrapper(IndexReader.open(ramDir)); + AtomicReader reader = new SlowCompositeReaderWrapper(DirectoryReader.open(ramDir)); try { assertNull(filter.getDocIdSet(reader.getTopReaderContext(), reader.getLiveDocs())); } diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeQueryBuilder.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeQueryBuilder.java index 9935ff8..4ffee9d 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeQueryBuilder.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeQueryBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.queryparser.xml.builders; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -166,7 +166,7 @@ public class TestNumericRangeQueryBuilder extends LuceneTestCase { private static Document getDocumentFromString(String str) throws SAXException, IOException, ParserConfigurationException { - InputStream is = new ByteArrayInputStream(str.getBytes()); + InputStream is = new ByteArrayInputStream(str.getBytes("UTF-8")); DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); factory.setNamespaceAware(true); DocumentBuilder builder = factory.newDocumentBuilder(); diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java index 81384d6..aadfad8 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java @@ -1,5 +1,5 @@ package org.apache.lucene.sandbox.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java index 4b3ea6a..f40b134 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.sandbox.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java index 4bc543b..8bf7d0e 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java @@ -1,6 +1,6 @@ package org.apache.lucene.sandbox.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeFilter.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeFilter.java index 2674d98..d511c89 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeFilter.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.sandbox.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeQuery.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeQuery.java index 8914bea..229267d 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.sandbox.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeTermsEnum.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeTermsEnum.java index 56caec6..6933701 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeTermsEnum.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeTermsEnum.java @@ -1,6 +1,6 @@ package org.apache.lucene.sandbox.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -63,11 +63,9 @@ public class SlowCollatedTermRangeTermsEnum extends FilteredTermsEnum { * The collator to use to collate index Terms, to determine their * membership in the range bounded by lowerTerm and * upperTerm. - * - * @throws IOException */ public SlowCollatedTermRangeTermsEnum(TermsEnum tenum, String lowerTermText, String upperTermText, - boolean includeLower, boolean includeUpper, Collator collator) throws IOException { + boolean includeLower, boolean includeUpper, Collator collator) { super(tenum); this.collator = collator; this.upperTermText = upperTermText; diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyQuery.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyQuery.java index 605b774..b09b045 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.sandbox.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java index f106c91..95721ec 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java @@ -1,6 +1,6 @@ package org.apache.lucene.sandbox.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/JakartaRegexpCapabilities.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/JakartaRegexpCapabilities.java index d2acb4e..bb62e28 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/JakartaRegexpCapabilities.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/JakartaRegexpCapabilities.java @@ -1,6 +1,6 @@ package org.apache.lucene.sandbox.queries.regex; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/JavaUtilRegexCapabilities.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/JavaUtilRegexCapabilities.java index d089361..894a427 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/JavaUtilRegexCapabilities.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/JavaUtilRegexCapabilities.java @@ -1,6 +1,6 @@ package org.apache.lucene.sandbox.queries.regex; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexCapabilities.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexCapabilities.java index 0050288..6875e9d 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexCapabilities.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexCapabilities.java @@ -2,7 +2,7 @@ package org.apache.lucene.sandbox.queries.regex; import org.apache.lucene.util.BytesRef; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexQuery.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexQuery.java index 932e55b..6420447 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.sandbox.queries.regex; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexQueryCapable.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexQueryCapable.java index efb2ddc..1b2c253 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexQueryCapable.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexQueryCapable.java @@ -1,6 +1,6 @@ package org.apache.lucene.sandbox.queries.regex; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexTermsEnum.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexTermsEnum.java index a81cae6..bce7bf8 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexTermsEnum.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexTermsEnum.java @@ -1,6 +1,6 @@ package org.apache.lucene.sandbox.queries.regex; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,8 +23,6 @@ import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.StringHelper; -import java.io.IOException; - /** * Subclass of FilteredTermEnum for enumerating all terms that match the * specified regular expression term using the specified regular expression @@ -39,7 +37,7 @@ public class RegexTermsEnum extends FilteredTermsEnum { private RegexCapabilities.RegexMatcher regexImpl; private final BytesRef prefixRef; - public RegexTermsEnum(TermsEnum tenum, Term term, RegexCapabilities regexCap) throws IOException { + public RegexTermsEnum(TermsEnum tenum, Term term, RegexCapabilities regexCap) { super(tenum); String text = term.text(); this.regexImpl = regexCap.compile(text); diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/DuplicateFilterTest.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/DuplicateFilterTest.java index cc7af8d..9b13f99 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/DuplicateFilterTest.java +++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/DuplicateFilterTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.sandbox.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,8 +22,7 @@ import java.util.HashSet; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.index.*; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; @@ -76,9 +75,9 @@ public class DuplicateFilterTest extends LuceneTestCase { private void addDoc(RandomIndexWriter writer, String url, String text, String date) throws IOException { Document doc = new Document(); - doc.add(newField(KEY_FIELD, url, StringField.TYPE_STORED)); - doc.add(newField("text", text, TextField.TYPE_STORED)); - doc.add(newField("date", date, TextField.TYPE_STORED)); + doc.add(newStringField(KEY_FIELD, url, Field.Store.YES)); + doc.add(newTextField("text", text, Field.Store.YES)); + doc.add(newTextField("date", date, Field.Store.YES)); writer.addDocument(doc); } diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/FuzzyLikeThisQueryTest.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/FuzzyLikeThisQueryTest.java index 4927350..dfedd8a 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/FuzzyLikeThisQueryTest.java +++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/FuzzyLikeThisQueryTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.sandbox.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,7 +20,7 @@ package org.apache.lucene.sandbox.queries; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; @@ -69,8 +69,8 @@ public class FuzzyLikeThisQueryTest extends LuceneTestCase { private void addDoc(RandomIndexWriter writer, String name, String id) throws IOException { Document doc = new Document(); - doc.add(newField("name", name, TextField.TYPE_STORED)); - doc.add(newField("id", id, TextField.TYPE_STORED)); + doc.add(newTextField("name", name, Field.Store.YES)); + doc.add(newTextField("id", id, Field.Store.YES)); writer.addDocument(doc); } diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowCollationMethods.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowCollationMethods.java index cee6b5e..fbe09d9 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowCollationMethods.java +++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowCollationMethods.java @@ -1,12 +1,10 @@ package org.apache.lucene.sandbox.queries; -import java.io.IOException; import java.text.Collator; import java.util.Locale; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.*; @@ -17,7 +15,7 @@ import org.apache.lucene.util._TestUtil; import org.junit.AfterClass; import org.junit.BeforeClass; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -58,7 +56,7 @@ public class TestSlowCollationMethods extends LuceneTestCase { for (int i = 0; i < numDocs; i++) { Document doc = new Document(); String value = _TestUtil.randomUnicodeString(random()); - Field field = newField("field", value, StringField.TYPE_STORED); + Field field = newStringField("field", value, Field.Store.YES); doc.add(field); iw.addDocument(doc); } @@ -91,7 +89,7 @@ public class TestSlowCollationMethods extends LuceneTestCase { public void testSort() throws Exception { SortField sf = new SortField("field", new FieldComparatorSource() { @Override - public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) { return new SlowCollatedStringComparator(numHits, fieldname, collator); } }); diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java index 8557e3d..8e58fd3 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java +++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.sandbox.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,7 +23,7 @@ import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.RandomIndexWriter; @@ -462,7 +462,7 @@ public class TestSlowFuzzyQuery extends LuceneTestCase { private void addDoc(String text, RandomIndexWriter writer) throws IOException { Document doc = new Document(); - doc.add(newField("field", text, TextField.TYPE_STORED)); + doc.add(newTextField("field", text, Field.Store.YES)); writer.addDocument(doc); } } diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery2.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery2.java index ce1a964..c93c14a 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery2.java +++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery2.java @@ -1,6 +1,6 @@ package org.apache.lucene.sandbox.queries; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -25,7 +25,6 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; @@ -93,7 +92,7 @@ public class TestSlowFuzzyQuery2 extends LuceneTestCase { RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)).setMergePolicy(newLogMergePolicy())); Document doc = new Document(); - Field field = newField("field", "", TextField.TYPE_UNSTORED); + Field field = newTextField("field", "", Field.Store.NO); doc.add(field); for (int i = 0; i < terms; i++) { diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestJakartaRegexpCapabilities.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestJakartaRegexpCapabilities.java index 0d40c73..9de008d 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestJakartaRegexpCapabilities.java +++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestJakartaRegexpCapabilities.java @@ -1,6 +1,6 @@ package org.apache.lucene.sandbox.queries.regex; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestRegexQuery.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestRegexQuery.java index 802caed..7ae01e2 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestRegexQuery.java +++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestRegexQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.sandbox.queries.regex; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,6 +17,7 @@ package org.apache.lucene.sandbox.queries.regex; * limitations under the License. */ +import org.apache.lucene.document.Field; import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; import org.apache.lucene.store.Directory; import org.apache.lucene.index.IndexReader; @@ -25,7 +26,6 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.index.TermsEnum; @@ -47,7 +47,7 @@ public class TestRegexQuery extends LuceneTestCase { directory = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), directory); Document doc = new Document(); - doc.add(newField(FN, "the quick brown fox jumps over the lazy dog", TextField.TYPE_UNSTORED)); + doc.add(newTextField(FN, "the quick brown fox jumps over the lazy dog", Field.Store.NO)); writer.addDocument(doc); reader = writer.getReader(); writer.close(); diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java index 058b98e..a508018 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java +++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java @@ -1,6 +1,6 @@ package org.apache.lucene.sandbox.queries.regex; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,23 +17,19 @@ package org.apache.lucene.sandbox.queries.regex; * limitations under the License. */ -import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.spans.SpanFirstQuery; import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.LuceneTestCase; public class TestSpanRegexQuery extends LuceneTestCase { @@ -64,15 +60,15 @@ public class TestSpanRegexQuery extends LuceneTestCase { // Field.Store.NO, Field.Index.ANALYZED)); // writer.addDocument(doc); // doc = new Document(); - doc.add(newField("field", "auto update", TextField.TYPE_UNSTORED)); + doc.add(newTextField("field", "auto update", Field.Store.NO)); writer.addDocument(doc); doc = new Document(); - doc.add(newField("field", "first auto update", TextField.TYPE_UNSTORED)); + doc.add(newTextField("field", "first auto update", Field.Store.NO)); writer.addDocument(doc); writer.forceMerge(1); writer.close(); - IndexReader reader = IndexReader.open(directory); + IndexReader reader = DirectoryReader.open(directory); IndexSearcher searcher = new IndexSearcher(reader); SpanQuery srq = new SpanMultiTermQueryWrapper(new RegexQuery(new Term("field", "aut.*"))); SpanFirstQuery sfq = new SpanFirstQuery(srq, 1); diff --git a/lucene/site/changes/changes2html.pl b/lucene/site/changes/changes2html.pl index de1359c..2ade718 100755 --- a/lucene/site/changes/changes2html.pl +++ b/lucene/site/changes/changes2html.pl @@ -24,7 +24,8 @@ use strict; use warnings; -my $project_info_url = 'https://issues.apache.org/jira/rest/api/2.0.alpha1/project/LUCENE'; +# JIRA REST API documentation: +my $project_info_url = 'https://issues.apache.org/jira/rest/api/2/project/LUCENE'; my $jira_url_prefix = 'http://issues.apache.org/jira/browse/'; my $bugzilla_url_prefix = 'http://issues.apache.org/bugzilla/show_bug.cgi?id='; my %release_dates = &setup_release_dates; @@ -64,7 +65,7 @@ for (my $line_num = 0 ; $line_num <= $#lines ; ++$line_num) { if (/\s*===+\s*(.*?)\s*===+\s*/) { # New-style release headings $release = $1; $release =~ s/^(?:release|lucene)\s*//i; # Trim "Release " or "Lucene " prefix - ($release, $relinfo) = ($release =~ /^(\d+(?:\.(?:\d+|[xyz]))*|Trunk)\s*(.*)/i); + ($release, $relinfo) = ($release =~ /^(\d+(?:\.(?:\d+))*(?:-(?:ALPHA|BETA))?|Trunk)\s*(.*)/i); $relinfo =~ s/\s*:\s*$//; # Trim trailing colon $relinfo =~ s/^\s*,\s*//; # Trim leading comma ($reldate, $relinfo) = get_release_date($release, $relinfo); @@ -155,7 +156,7 @@ for (my $line_num = 0 ; $line_num <= $#lines ; ++$line_num) { $item .= "\n"; while ($line_num < $#lines - and ($line = $lines[++$line_num]) !~ /^(?:\S|\s*\Q$type\E)/) { + and ($line = $lines[++$line_num]) !~ /^(?:\S|\s*\Q$type\E\s+)/) { $line =~ s/^\s{$leading_ws_width}//; # Trim leading whitespace $line =~ s/\s+$//; # Trim trailing whitespace $item .= "$line\n"; @@ -508,6 +509,9 @@ for my $rel (@releases) { $leading_whitespace . $issue1 . $interlude . $issue2; ~gex; + # Linkify URLs, except Bugzilla links, which don't work anymore + $item =~ s~(?])(https?://(?!(?:nagoya|issues)\.apache\.org/bugzilla)\S+)~$1~g; + print "
    • $item
      • \n"; } print " \n" unless ($has_release_sections and not $heading); diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/SimpleSpatialFieldInfo.java b/lucene/spatial/src/java/org/apache/lucene/spatial/SimpleSpatialFieldInfo.java deleted file mode 100644 index 04255ac..0000000 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/SimpleSpatialFieldInfo.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.lucene.spatial; - - -public class SimpleSpatialFieldInfo implements SpatialFieldInfo { - - private final String fieldName; - - public SimpleSpatialFieldInfo(String fieldName) { - this.fieldName = fieldName; - } - - public String getFieldName() { - return fieldName; - } -} diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialFieldInfo.java b/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialFieldInfo.java deleted file mode 100644 index 3370f00..0000000 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialFieldInfo.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.lucene.spatial; - -/** - * Information the strategy needs for the lucene fields - */ -public interface SpatialFieldInfo { -} diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java b/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java index 2681611..59df15e 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java @@ -1,3 +1,5 @@ +package org.apache.lucene.spatial; + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -15,26 +17,37 @@ * limitations under the License. */ -package org.apache.lucene.spatial; - import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.query.SpatialArgs; import com.spatial4j.core.shape.Shape; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.queries.function.FunctionQuery; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.Filter; +import org.apache.lucene.search.FilteredQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.spatial.query.SpatialArgs; /** * must be thread safe + * + * @lucene.experimental */ -public abstract class SpatialStrategy { +public abstract class SpatialStrategy { protected boolean ignoreIncompatibleGeometry = false; protected final SpatialContext ctx; + private final String fieldName; - public SpatialStrategy(SpatialContext ctx) { + /** + * Constructs the spatial strategy with its mandatory arguments. + */ + public SpatialStrategy(SpatialContext ctx, String fieldName) { + if (ctx == null) + throw new IllegalArgumentException("ctx is required"); this.ctx = ctx; + if (fieldName == null || fieldName.length() == 0) + throw new IllegalArgumentException("fieldName is required"); + this.fieldName = fieldName; } public SpatialContext getSpatialContext() { @@ -47,30 +60,48 @@ public abstract class SpatialStrategy { } /** + * The name of the field or the prefix of them if there are multiple + * fields needed internally. + * @return Not null. + */ + public String getFieldName() { + return fieldName; + } + + /** * Corresponds with Solr's FieldType.createField(). * * This may return a null field if it does not want to make anything. * This is reasonable behavior if 'ignoreIncompatibleGeometry=true' and the * geometry is incompatible */ - public abstract IndexableField createField(T fieldInfo, Shape shape, boolean index, boolean store); + public abstract IndexableField createField(Shape shape, boolean index, boolean store); /** Corresponds with Solr's FieldType.createFields(). */ - public IndexableField[] createFields(T fieldInfo, Shape shape, boolean index, boolean store) { - return new IndexableField[] { createField(fieldInfo, shape, index, store) }; + public IndexableField[] createFields(Shape shape, boolean index, boolean store) { + return new IndexableField[] { createField(shape, index, store) }; } - public abstract ValueSource makeValueSource(SpatialArgs args, T fieldInfo); - /** - * Make a query + * The value source yields a number that is proportional to the distance between the query shape and indexed data. */ - public abstract Query makeQuery(SpatialArgs args, T fieldInfo); + public abstract ValueSource makeValueSource(SpatialArgs args); /** + * Make a query which has a score based on the distance from the data to the query shape. + * The default implementation constructs a {@link FilteredQuery} based on + * {@link #makeFilter(org.apache.lucene.spatial.query.SpatialArgs)} and + * {@link #makeValueSource(org.apache.lucene.spatial.query.SpatialArgs)}. + */ + public Query makeQuery(SpatialArgs args) { + Filter filter = makeFilter(args); + ValueSource vs = makeValueSource(args); + return new FilteredQuery(new FunctionQuery(vs), filter); + } + /** * Make a Filter */ - public abstract Filter makeFilter(SpatialArgs args, T fieldInfo); + public abstract Filter makeFilter(SpatialArgs args); public boolean isIgnoreIncompatibleGeometry() { return ignoreIncompatibleGeometry; @@ -79,4 +110,9 @@ public abstract class SpatialStrategy { public void setIgnoreIncompatibleGeometry(boolean ignoreIncompatibleGeometry) { this.ignoreIncompatibleGeometry = ignoreIncompatibleGeometry; } + + @Override + public String toString() { + return getClass().getSimpleName()+" field:"+fieldName+" ctx="+ctx; + } } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/AreaSimilarity.java b/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/AreaSimilarity.java new file mode 100644 index 0000000..d7ad91e --- /dev/null +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/AreaSimilarity.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.spatial.bbox; + +import org.apache.lucene.search.Explanation; + +import com.spatial4j.core.shape.Rectangle; + +/** + * The algorithm is implemented as envelope on envelope overlays rather than + * complex polygon on complex polygon overlays. + *

      + *

      + * Spatial relevance scoring algorithm: + *

      + *
      queryArea = the area of the input query envelope + *
      targetArea = the area of the target envelope (per Lucene document) + *
      intersectionArea = the area of the intersection for the query/target envelopes + *
      queryPower = the weighting power associated with the query envelope (default = 1.0) + *
      targetPower = the weighting power associated with the target envelope (default = 1.0) + *

      + *
      queryRatio = intersectionArea / queryArea; + *
      targetRatio = intersectionArea / targetArea; + *
      queryFactor = Math.pow(queryRatio,queryPower); + *
      targetFactor = Math.pow(targetRatio,targetPower); + *
      score = queryFactor * targetFactor; + *

      + * Based on Geoportal's + * + * SpatialRankingValueSource. + * + * @lucene.experimental + */ +public class AreaSimilarity implements BBoxSimilarity { + /** + * Properties associated with the query envelope + */ + private final Rectangle queryExtent; + private final double queryArea; + + private final double targetPower; + private final double queryPower; + + public AreaSimilarity(Rectangle queryExtent, double queryPower, double targetPower) { + this.queryExtent = queryExtent; + this.queryArea = queryExtent.getArea(); + + this.queryPower = queryPower; + this.targetPower = targetPower; + +// if (this.qryMinX > queryExtent.getMaxX()) { +// this.qryCrossedDateline = true; +// this.qryArea = Math.abs(qryMaxX + 360.0 - qryMinX) * Math.abs(qryMaxY - qryMinY); +// } else { +// this.qryArea = Math.abs(qryMaxX - qryMinX) * Math.abs(qryMaxY - qryMinY); +// } + } + + public AreaSimilarity(Rectangle queryExtent) { + this(queryExtent, 2.0, 0.5); + } + + + public String getDelimiterQueryParameters() { + return queryExtent.toString() + ";" + queryPower + ";" + targetPower; + } + + @Override + public double score(Rectangle target, Explanation exp) { + if (target == null || queryArea <= 0) { + return 0; + } + double targetArea = target.getArea(); + if (targetArea <= 0) { + return 0; + } + double score = 0; + + double top = Math.min(queryExtent.getMaxY(), target.getMaxY()); + double bottom = Math.max(queryExtent.getMinY(), target.getMinY()); + double height = top - bottom; + double width = 0; + + // queries that cross the date line + if (queryExtent.getCrossesDateLine()) { + // documents that cross the date line + if (target.getCrossesDateLine()) { + double left = Math.max(queryExtent.getMinX(), target.getMinX()); + double right = Math.min(queryExtent.getMaxX(), target.getMaxX()); + width = right + 360.0 - left; + } else { + double qryWestLeft = Math.max(queryExtent.getMinX(), target.getMaxX()); + double qryWestRight = Math.min(target.getMaxX(), 180.0); + double qryWestWidth = qryWestRight - qryWestLeft; + if (qryWestWidth > 0) { + width = qryWestWidth; + } else { + double qryEastLeft = Math.max(target.getMaxX(), -180.0); + double qryEastRight = Math.min(queryExtent.getMaxX(), target.getMaxX()); + double qryEastWidth = qryEastRight - qryEastLeft; + if (qryEastWidth > 0) { + width = qryEastWidth; + } + } + } + } else { // queries that do not cross the date line + + if (target.getCrossesDateLine()) { + double tgtWestLeft = Math.max(queryExtent.getMinX(), target.getMinX()); + double tgtWestRight = Math.min(queryExtent.getMaxX(), 180.0); + double tgtWestWidth = tgtWestRight - tgtWestLeft; + if (tgtWestWidth > 0) { + width = tgtWestWidth; + } else { + double tgtEastLeft = Math.max(queryExtent.getMinX(), -180.0); + double tgtEastRight = Math.min(queryExtent.getMaxX(), target.getMaxX()); + double tgtEastWidth = tgtEastRight - tgtEastLeft; + if (tgtEastWidth > 0) { + width = tgtEastWidth; + } + } + } else { + double left = Math.max(queryExtent.getMinX(), target.getMinX()); + double right = Math.min(queryExtent.getMaxX(), target.getMaxX()); + width = right - left; + } + } + + + // calculate the score + if ((width > 0) && (height > 0)) { + double intersectionArea = width * height; + double queryRatio = intersectionArea / queryArea; + double targetRatio = intersectionArea / targetArea; + double queryFactor = Math.pow(queryRatio, queryPower); + double targetFactor = Math.pow(targetRatio, targetPower); + score = queryFactor * targetFactor * 10000.0; + + if (exp!=null) { +// StringBuilder sb = new StringBuilder(); +// sb.append("\nscore=").append(score); +// sb.append("\n query=").append(); +// sb.append("\n target=").append(target.toString()); +// sb.append("\n intersectionArea=").append(intersectionArea); +// +// sb.append(" queryArea=").append(queryArea).append(" targetArea=").append(targetArea); +// sb.append("\n queryRatio=").append(queryRatio).append(" targetRatio=").append(targetRatio); +// sb.append("\n queryFactor=").append(queryFactor).append(" targetFactor=").append(targetFactor); +// sb.append(" (queryPower=").append(queryPower).append(" targetPower=").append(targetPower).append(")"); + + exp.setValue((float)score); + exp.setDescription(this.getClass().getSimpleName()); + + Explanation e = null; + + exp.addDetail( e = new Explanation((float)intersectionArea, "IntersectionArea") ); + e.addDetail(new Explanation((float)width, "width; Query: "+queryExtent.toString())); + e.addDetail(new Explanation((float)height, "height; Target: "+target.toString())); + + exp.addDetail( e = new Explanation((float)queryFactor, "Query") ); + e.addDetail(new Explanation((float)queryArea, "area")); + e.addDetail(new Explanation((float)queryRatio, "ratio")); + e.addDetail(new Explanation((float)queryPower, "power")); + + exp.addDetail( e = new Explanation((float)targetFactor, "Target") ); + e.addDetail(new Explanation((float)targetArea, "area")); + e.addDetail(new Explanation((float)targetRatio, "ratio")); + e.addDetail(new Explanation((float)targetPower, "power")); + } + } + else if(exp !=null) { + exp.setValue(0); + exp.setDescription("Shape does not intersect"); + } + return score; + } + + + /** + * Determines if this ValueSource is equal to another. + * + * @param o the ValueSource to compare + * @return true if the two objects are based upon the same query envelope + */ + @Override + public boolean equals(Object o) { + if (o.getClass() != AreaSimilarity.class) + return false; + + AreaSimilarity other = (AreaSimilarity) o; + return getDelimiterQueryParameters().equals(other.getDelimiterQueryParameters()); + } + + /** + * Returns the ValueSource hash code. + * + * @return the hash code + */ + @Override + public int hashCode() { + return getDelimiterQueryParameters().hashCode(); + } +} diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarity.java b/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarity.java new file mode 100644 index 0000000..d077920 --- /dev/null +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarity.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.spatial.bbox; + +import org.apache.lucene.search.Explanation; + +import com.spatial4j.core.shape.Rectangle; + +/** + * @lucene.experimental + */ +public interface BBoxSimilarity { + + public double score(Rectangle extent, Explanation exp); +} diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java b/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java new file mode 100644 index 0000000..3008072 --- /dev/null +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java @@ -0,0 +1,129 @@ +package org.apache.lucene.spatial.bbox; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.spatial4j.core.shape.Rectangle; +import com.spatial4j.core.shape.simple.RectangleImpl; +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.util.Bits; + +import java.io.IOException; +import java.util.Map; + +/** + * An implementation of the Lucene ValueSource model to support spatial relevance ranking. + * + * @lucene.experimental + */ +public class BBoxSimilarityValueSource extends ValueSource { + + private final BBoxStrategy strategy; + private final BBoxSimilarity similarity; + + public BBoxSimilarityValueSource(BBoxStrategy strategy, BBoxSimilarity similarity) { + this.strategy = strategy; + this.similarity = similarity; + } + + /** + * Returns the ValueSource description. + * + * @return the description + */ + @Override + public String description() { + return "BBoxSimilarityValueSource(" + similarity + ")"; + } + + + /** + * Returns the DocValues used by the function query. + * + * @param readerContext the AtomicReaderContext which holds an AtomicReader + * @return the values + */ + @Override + public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException { + AtomicReader reader = readerContext.reader(); + final double[] minX = FieldCache.DEFAULT.getDoubles(reader, strategy.field_minX, true); + final double[] minY = FieldCache.DEFAULT.getDoubles(reader, strategy.field_minY, true); + final double[] maxX = FieldCache.DEFAULT.getDoubles(reader, strategy.field_maxX, true); + final double[] maxY = FieldCache.DEFAULT.getDoubles(reader, strategy.field_maxY, true); + + final Bits validMinX = FieldCache.DEFAULT.getDocsWithField(reader, strategy.field_minX); + final Bits validMaxX = FieldCache.DEFAULT.getDocsWithField(reader, strategy.field_maxX); + + return new FunctionValues() { + @Override + public float floatVal(int doc) { + // make sure it has minX and area + if (validMinX.get(doc) && validMaxX.get(doc)) { + Rectangle rect = new RectangleImpl( + minX[doc], maxX[doc], + minY[doc], maxY[doc]); + return (float) similarity.score(rect, null); + } + return 0; + } + + public Explanation explain(int doc) { + // make sure it has minX and area + if (validMinX.get(doc) && validMaxX.get(doc)) { + Rectangle rect = new RectangleImpl( + minX[doc], maxX[doc], + minY[doc], maxY[doc]); + Explanation exp = new Explanation(); + similarity.score(rect, exp); + return exp; + } + return new Explanation(0, "No BBox"); + } + + @Override + public String toString(int doc) { + return description() + "=" + floatVal(doc); + } + }; + } + + /** + * Determines if this ValueSource is equal to another. + * + * @param o the ValueSource to compare + * @return true if the two objects are based upon the same query envelope + */ + @Override + public boolean equals(Object o) { + if (o.getClass() != BBoxSimilarityValueSource.class) { + return false; + } + + BBoxSimilarityValueSource other = (BBoxSimilarityValueSource) o; + return similarity.equals(other.similarity); + } + + @Override + public int hashCode() { + return BBoxSimilarityValueSource.class.hashCode() + similarity.hashCode(); + } +} diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java b/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java new file mode 100644 index 0000000..1631d92 --- /dev/null +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java @@ -0,0 +1,516 @@ +package org.apache.lucene.spatial.bbox; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.spatial4j.core.context.SpatialContext; +import com.spatial4j.core.shape.Rectangle; +import com.spatial4j.core.shape.Shape; +import org.apache.lucene.document.DoubleField; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.queries.function.FunctionQuery; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.Filter; +import org.apache.lucene.search.NumericRangeQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryWrapperFilter; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.spatial.SpatialStrategy; +import org.apache.lucene.spatial.query.SpatialArgs; +import org.apache.lucene.spatial.query.SpatialOperation; +import org.apache.lucene.spatial.query.UnsupportedSpatialOperation; + +import java.text.NumberFormat; +import java.util.Locale; + + +/** + * Based on GeoPortal's + * SpatialClauseAdapter. + * + * @lucene.experimental + */ +public class BBoxStrategy extends SpatialStrategy { + + public static final String SUFFIX_MINX = "__minX"; + public static final String SUFFIX_MAXX = "__maxX"; + public static final String SUFFIX_MINY = "__minY"; + public static final String SUFFIX_MAXY = "__maxY"; + public static final String SUFFIX_XDL = "__xdl"; + + /* + * The Bounding Box gets stored as four fields for x/y min/max and a flag + * that says if the box crosses the dateline (xdl). + */ + public final String field_bbox; + public final String field_minX; + public final String field_minY; + public final String field_maxX; + public final String field_maxY; + public final String field_xdl; // crosses dateline + + public double queryPower = 1.0; + public double targetPower = 1.0f; + public int precisionStep = 8; // same as solr default + + public BBoxStrategy(SpatialContext ctx, String fieldNamePrefix) { + super(ctx, fieldNamePrefix); + field_bbox = fieldNamePrefix; + field_minX = fieldNamePrefix + SUFFIX_MINX; + field_maxX = fieldNamePrefix + SUFFIX_MAXX; + field_minY = fieldNamePrefix + SUFFIX_MINY; + field_maxY = fieldNamePrefix + SUFFIX_MAXY; + field_xdl = fieldNamePrefix + SUFFIX_XDL; + } + + public void setPrecisionStep( int p ) { + precisionStep = p; + if (precisionStep<=0 || precisionStep>=64) + precisionStep=Integer.MAX_VALUE; + } + + //--------------------------------- + // Indexing + //--------------------------------- + + @Override + public IndexableField[] createFields(Shape shape, boolean index, boolean store) { + + Rectangle bbox = shape.getBoundingBox(); + IndexableField[] fields = new IndexableField[store?6:5]; + fields[0] = createDouble(field_minX, bbox.getMinX(), index, store); + fields[1] = createDouble(field_maxX, bbox.getMaxX(), index, store); + fields[2] = createDouble(field_minY, bbox.getMinY(), index, store); + fields[3] = createDouble(field_maxY, bbox.getMaxY(), index, store); + + FieldType ft = new FieldType(); + ft.setIndexed(index); + ft.setStored(store); + ft.setTokenized(false); + ft.setOmitNorms(true); + ft.setIndexOptions(IndexOptions.DOCS_ONLY); + ft.freeze(); + + Field xdl = new Field( field_xdl, bbox.getCrossesDateLine()?"T":"F", ft ); + fields[4] = xdl; + if( store ) { + FieldType ff = new FieldType(); + ff.setIndexed(false); + ff.setStored(true); + ff.setOmitNorms(true); + ff.setIndexOptions(IndexOptions.DOCS_ONLY); + ff.freeze(); + + NumberFormat nf = NumberFormat.getInstance( Locale.US ); + nf.setMaximumFractionDigits( 5 ); + nf.setMinimumFractionDigits( 5 ); + nf.setGroupingUsed(false); + String ext = + nf.format( bbox.getMinX() ) + ' ' + + nf.format( bbox.getMinY() ) + ' ' + + nf.format( bbox.getMaxX() ) + ' ' + + nf.format( bbox.getMaxY() ) + ' '; + fields[5] = new Field( field_bbox, ext, ff ); + } + return fields; + } + + private IndexableField createDouble(String name, double v, boolean index, boolean store) { + if (!store && !index) + throw new IllegalArgumentException("field must be indexed or stored"); + + FieldType fieldType = new FieldType(DoubleField.TYPE_NOT_STORED); + fieldType.setStored(store); + fieldType.setIndexed(index); + fieldType.setNumericPrecisionStep(precisionStep); + return new DoubleField(name,v,fieldType); + } + + @Override + public IndexableField createField(Shape shape, + boolean index, boolean store) { + throw new UnsupportedOperationException("BBOX is poly field"); + } + + @Override + public boolean isPolyField() { + return true; + } + + //--------------------------------- + // Query Builder + //--------------------------------- + + @Override + public ValueSource makeValueSource(SpatialArgs args) { + return new BBoxSimilarityValueSource( + this, new AreaSimilarity(args.getShape().getBoundingBox(), queryPower, targetPower)); + } + + + @Override + public Filter makeFilter(SpatialArgs args) { + Query spatial = makeSpatialQuery(args); + return new QueryWrapperFilter( spatial ); + } + + @Override + public Query makeQuery(SpatialArgs args) { + BooleanQuery bq = new BooleanQuery(); + Query spatial = makeSpatialQuery(args); + bq.add(new ConstantScoreQuery(spatial), BooleanClause.Occur.MUST); + + // This part does the scoring + Query spatialRankingQuery = new FunctionQuery(makeValueSource(args)); + bq.add(spatialRankingQuery, BooleanClause.Occur.MUST); + return bq; + } + + + private Query makeSpatialQuery(SpatialArgs args) { + Rectangle bbox = args.getShape().getBoundingBox(); + Query spatial = null; + + // Useful for understanding Relations: + // http://edndoc.esri.com/arcsde/9.1/general_topics/understand_spatial_relations.htm + SpatialOperation op = args.getOperation(); + if( op == SpatialOperation.BBoxIntersects ) spatial = makeIntersects(bbox); + else if( op == SpatialOperation.BBoxWithin ) spatial = makeWithin(bbox); + else if( op == SpatialOperation.Contains ) spatial = makeContains(bbox); + else if( op == SpatialOperation.Intersects ) spatial = makeIntersects(bbox); + else if( op == SpatialOperation.IsEqualTo ) spatial = makeEquals(bbox); + else if( op == SpatialOperation.IsDisjointTo ) spatial = makeDisjoint(bbox); + else if( op == SpatialOperation.IsWithin ) spatial = makeWithin(bbox); + else if( op == SpatialOperation.Overlaps ) spatial = makeIntersects(bbox); + else { + throw new UnsupportedSpatialOperation(op); + } + return spatial; + } + + + //------------------------------------------------------------------------------- + // + //------------------------------------------------------------------------------- + + /** + * Constructs a query to retrieve documents that fully contain the input envelope. + * + * @return the spatial query + */ + Query makeContains(Rectangle bbox) { + + // general case + // docMinX <= queryExtent.getMinX() AND docMinY <= queryExtent.getMinY() AND docMaxX >= queryExtent.getMaxX() AND docMaxY >= queryExtent.getMaxY() + + // Y conditions + // docMinY <= queryExtent.getMinY() AND docMaxY >= queryExtent.getMaxY() + Query qMinY = NumericRangeQuery.newDoubleRange(field_minY, precisionStep, null, bbox.getMinY(), false, true); + Query qMaxY = NumericRangeQuery.newDoubleRange(field_maxY, precisionStep, bbox.getMaxY(), null, true, false); + Query yConditions = this.makeQuery(new Query[]{qMinY, qMaxY}, BooleanClause.Occur.MUST); + + // X conditions + Query xConditions = null; + + // queries that do not cross the date line + if (!bbox.getCrossesDateLine()) { + + // X Conditions for documents that do not cross the date line, + // documents that contain the min X and max X of the query envelope, + // docMinX <= queryExtent.getMinX() AND docMaxX >= queryExtent.getMaxX() + Query qMinX = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, null, bbox.getMinX(), false, true); + Query qMaxX = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, bbox.getMaxX(), null, true, false); + Query qMinMax = this.makeQuery(new Query[]{qMinX, qMaxX}, BooleanClause.Occur.MUST); + Query qNonXDL = this.makeXDL(false, qMinMax); + + // X Conditions for documents that cross the date line, + // the left portion of the document contains the min X of the query + // OR the right portion of the document contains the max X of the query, + // docMinXLeft <= queryExtent.getMinX() OR docMaxXRight >= queryExtent.getMaxX() + Query qXDLLeft = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, null, bbox.getMinX(), false, true); + Query qXDLRight = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, bbox.getMaxX(), null, true, false); + Query qXDLLeftRight = this.makeQuery(new Query[]{qXDLLeft, qXDLRight}, BooleanClause.Occur.SHOULD); + Query qXDL = this.makeXDL(true, qXDLLeftRight); + + // apply the non-XDL and XDL conditions + xConditions = this.makeQuery(new Query[]{qNonXDL, qXDL}, BooleanClause.Occur.SHOULD); + + // queries that cross the date line + } else { + + // No need to search for documents that do not cross the date line + + // X Conditions for documents that cross the date line, + // the left portion of the document contains the min X of the query + // AND the right portion of the document contains the max X of the query, + // docMinXLeft <= queryExtent.getMinX() AND docMaxXRight >= queryExtent.getMaxX() + Query qXDLLeft = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, null, bbox.getMinX(), false, true); + Query qXDLRight = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, bbox.getMaxX(), null, true, false); + Query qXDLLeftRight = this.makeQuery(new Query[]{qXDLLeft, qXDLRight}, BooleanClause.Occur.MUST); + + xConditions = this.makeXDL(true, qXDLLeftRight); + } + + // both X and Y conditions must occur + return this.makeQuery(new Query[]{xConditions, yConditions}, BooleanClause.Occur.MUST); + } + + /** + * Constructs a query to retrieve documents that are disjoint to the input envelope. + * + * @return the spatial query + */ + Query makeDisjoint(Rectangle bbox) { + + // general case + // docMinX > queryExtent.getMaxX() OR docMaxX < queryExtent.getMinX() OR docMinY > queryExtent.getMaxY() OR docMaxY < queryExtent.getMinY() + + // Y conditions + // docMinY > queryExtent.getMaxY() OR docMaxY < queryExtent.getMinY() + Query qMinY = NumericRangeQuery.newDoubleRange(field_minY, precisionStep, bbox.getMaxY(), null, false, false); + Query qMaxY = NumericRangeQuery.newDoubleRange(field_maxY, precisionStep, null, bbox.getMinY(), false, false); + Query yConditions = this.makeQuery(new Query[]{qMinY, qMaxY}, BooleanClause.Occur.SHOULD); + + // X conditions + Query xConditions = null; + + // queries that do not cross the date line + if (!bbox.getCrossesDateLine()) { + + // X Conditions for documents that do not cross the date line, + // docMinX > queryExtent.getMaxX() OR docMaxX < queryExtent.getMinX() + Query qMinX = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, bbox.getMaxX(), null, false, false); + Query qMaxX = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, null, bbox.getMinX(), false, false); + Query qMinMax = this.makeQuery(new Query[]{qMinX, qMaxX}, BooleanClause.Occur.SHOULD); + Query qNonXDL = this.makeXDL(false, qMinMax); + + // X Conditions for documents that cross the date line, + // both the left and right portions of the document must be disjoint to the query + // (docMinXLeft > queryExtent.getMaxX() OR docMaxXLeft < queryExtent.getMinX()) AND + // (docMinXRight > queryExtent.getMaxX() OR docMaxXRight < queryExtent.getMinX()) + // where: docMaxXLeft = 180.0, docMinXRight = -180.0 + // (docMaxXLeft < queryExtent.getMinX()) equates to (180.0 < queryExtent.getMinX()) and is ignored + // (docMinXRight > queryExtent.getMaxX()) equates to (-180.0 > queryExtent.getMaxX()) and is ignored + Query qMinXLeft = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, bbox.getMaxX(), null, false, false); + Query qMaxXRight = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, null, bbox.getMinX(), false, false); + Query qLeftRight = this.makeQuery(new Query[]{qMinXLeft, qMaxXRight}, BooleanClause.Occur.MUST); + Query qXDL = this.makeXDL(true, qLeftRight); + + // apply the non-XDL and XDL conditions + xConditions = this.makeQuery(new Query[]{qNonXDL, qXDL}, BooleanClause.Occur.SHOULD); + + // queries that cross the date line + } else { + + // X Conditions for documents that do not cross the date line, + // the document must be disjoint to both the left and right query portions + // (docMinX > queryExtent.getMaxX()Left OR docMaxX < queryExtent.getMinX()) AND (docMinX > queryExtent.getMaxX() OR docMaxX < queryExtent.getMinX()Left) + // where: queryExtent.getMaxX()Left = 180.0, queryExtent.getMinX()Left = -180.0 + Query qMinXLeft = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, 180.0, null, false, false); + Query qMaxXLeft = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, null, bbox.getMinX(), false, false); + Query qMinXRight = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, bbox.getMaxX(), null, false, false); + Query qMaxXRight = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, null, -180.0, false, false); + Query qLeft = this.makeQuery(new Query[]{qMinXLeft, qMaxXLeft}, BooleanClause.Occur.SHOULD); + Query qRight = this.makeQuery(new Query[]{qMinXRight, qMaxXRight}, BooleanClause.Occur.SHOULD); + Query qLeftRight = this.makeQuery(new Query[]{qLeft, qRight}, BooleanClause.Occur.MUST); + + // No need to search for documents that do not cross the date line + + xConditions = this.makeXDL(false, qLeftRight); + } + + // either X or Y conditions should occur + return this.makeQuery(new Query[]{xConditions, yConditions}, BooleanClause.Occur.SHOULD); + } + + /** + * Constructs a query to retrieve documents that equal the input envelope. + * + * @return the spatial query + */ + Query makeEquals(Rectangle bbox) { + + // docMinX = queryExtent.getMinX() AND docMinY = queryExtent.getMinY() AND docMaxX = queryExtent.getMaxX() AND docMaxY = queryExtent.getMaxY() + Query qMinX = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, bbox.getMinX(), bbox.getMinX(), true, true); + Query qMinY = NumericRangeQuery.newDoubleRange(field_minY, precisionStep, bbox.getMinY(), bbox.getMinY(), true, true); + Query qMaxX = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, bbox.getMaxX(), bbox.getMaxX(), true, true); + Query qMaxY = NumericRangeQuery.newDoubleRange(field_maxY, precisionStep, bbox.getMaxY(), bbox.getMaxY(), true, true); + BooleanQuery bq = new BooleanQuery(); + bq.add(qMinX, BooleanClause.Occur.MUST); + bq.add(qMinY, BooleanClause.Occur.MUST); + bq.add(qMaxX, BooleanClause.Occur.MUST); + bq.add(qMaxY, BooleanClause.Occur.MUST); + return bq; + } + + /** + * Constructs a query to retrieve documents that intersect the input envelope. + * + * @return the spatial query + */ + Query makeIntersects(Rectangle bbox) { + + // the original intersects query does not work for envelopes that cross the date line, + // switch to a NOT Disjoint query + + // MUST_NOT causes a problem when it's the only clause type within a BooleanQuery, + // to get round it we add all documents as a SHOULD + + // there must be an envelope, it must not be disjoint + Query qDisjoint = makeDisjoint(bbox); + Query qIsNonXDL = this.makeXDL(false); + Query qIsXDL = this.makeXDL(true); + Query qHasEnv = this.makeQuery(new Query[]{qIsNonXDL, qIsXDL}, BooleanClause.Occur.SHOULD); + BooleanQuery qNotDisjoint = new BooleanQuery(); + qNotDisjoint.add(qHasEnv, BooleanClause.Occur.MUST); + qNotDisjoint.add(qDisjoint, BooleanClause.Occur.MUST_NOT); + + //Query qDisjoint = makeDisjoint(); + //BooleanQuery qNotDisjoint = new BooleanQuery(); + //qNotDisjoint.add(new MatchAllDocsQuery(),BooleanClause.Occur.SHOULD); + //qNotDisjoint.add(qDisjoint,BooleanClause.Occur.MUST_NOT); + return qNotDisjoint; + } + + /** + * Makes a boolean query based upon a collection of queries and a logical operator. + * + * @param queries the query collection + * @param occur the logical operator + * @return the query + */ + BooleanQuery makeQuery(Query[] queries, BooleanClause.Occur occur) { + BooleanQuery bq = new BooleanQuery(); + for (Query query : queries) { + bq.add(query, occur); + } + return bq; + } + + /** + * Constructs a query to retrieve documents are fully within the input envelope. + * + * @return the spatial query + */ + Query makeWithin(Rectangle bbox) { + + // general case + // docMinX >= queryExtent.getMinX() AND docMinY >= queryExtent.getMinY() AND docMaxX <= queryExtent.getMaxX() AND docMaxY <= queryExtent.getMaxY() + + // Y conditions + // docMinY >= queryExtent.getMinY() AND docMaxY <= queryExtent.getMaxY() + Query qMinY = NumericRangeQuery.newDoubleRange(field_minY, precisionStep, bbox.getMinY(), null, true, false); + Query qMaxY = NumericRangeQuery.newDoubleRange(field_maxY, precisionStep, null, bbox.getMaxY(), false, true); + Query yConditions = this.makeQuery(new Query[]{qMinY, qMaxY}, BooleanClause.Occur.MUST); + + // X conditions + Query xConditions = null; + + // X Conditions for documents that cross the date line, + // the left portion of the document must be within the left portion of the query, + // AND the right portion of the document must be within the right portion of the query + // docMinXLeft >= queryExtent.getMinX() AND docMaxXLeft <= 180.0 + // AND docMinXRight >= -180.0 AND docMaxXRight <= queryExtent.getMaxX() + Query qXDLLeft = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, bbox.getMinX(), null, true, false); + Query qXDLRight = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, null, bbox.getMaxX(), false, true); + Query qXDLLeftRight = this.makeQuery(new Query[]{qXDLLeft, qXDLRight}, BooleanClause.Occur.MUST); + Query qXDL = this.makeXDL(true, qXDLLeftRight); + + // queries that do not cross the date line + if (!bbox.getCrossesDateLine()) { + + // X Conditions for documents that do not cross the date line, + // docMinX >= queryExtent.getMinX() AND docMaxX <= queryExtent.getMaxX() + Query qMinX = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, bbox.getMinX(), null, true, false); + Query qMaxX = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, null, bbox.getMaxX(), false, true); + Query qMinMax = this.makeQuery(new Query[]{qMinX, qMaxX}, BooleanClause.Occur.MUST); + Query qNonXDL = this.makeXDL(false, qMinMax); + + // apply the non-XDL or XDL X conditions + if ((bbox.getMinX() <= -180.0) && bbox.getMaxX() >= 180.0) { + xConditions = this.makeQuery(new Query[]{qNonXDL, qXDL}, BooleanClause.Occur.SHOULD); + } else { + xConditions = qNonXDL; + } + + // queries that cross the date line + } else { + + // X Conditions for documents that do not cross the date line + + // the document should be within the left portion of the query + // docMinX >= queryExtent.getMinX() AND docMaxX <= 180.0 + Query qMinXLeft = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, bbox.getMinX(), null, true, false); + Query qMaxXLeft = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, null, 180.0, false, true); + Query qLeft = this.makeQuery(new Query[]{qMinXLeft, qMaxXLeft}, BooleanClause.Occur.MUST); + + // the document should be within the right portion of the query + // docMinX >= -180.0 AND docMaxX <= queryExtent.getMaxX() + Query qMinXRight = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, -180.0, null, true, false); + Query qMaxXRight = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, null, bbox.getMaxX(), false, true); + Query qRight = this.makeQuery(new Query[]{qMinXRight, qMaxXRight}, BooleanClause.Occur.MUST); + + // either left or right conditions should occur, + // apply the left and right conditions to documents that do not cross the date line + Query qLeftRight = this.makeQuery(new Query[]{qLeft, qRight}, BooleanClause.Occur.SHOULD); + Query qNonXDL = this.makeXDL(false, qLeftRight); + + // apply the non-XDL and XDL conditions + xConditions = this.makeQuery(new Query[]{qNonXDL, qXDL}, BooleanClause.Occur.SHOULD); + } + + // both X and Y conditions must occur + return this.makeQuery(new Query[]{xConditions, yConditions}, BooleanClause.Occur.MUST); + } + + /** + * Constructs a query to retrieve documents that do or do not cross the date line. + * + * + * @param crossedDateLine true for documents that cross the date line + * @return the query + */ + Query makeXDL(boolean crossedDateLine) { + // The 'T' and 'F' values match solr fields + return new TermQuery(new Term(field_xdl, crossedDateLine ? "T" : "F")); + } + + /** + * Constructs a query to retrieve documents that do or do not cross the date line + * and match the supplied spatial query. + * + * @param crossedDateLine true for documents that cross the date line + * @param query the spatial query + * @return the query + */ + Query makeXDL(boolean crossedDateLine, Query query) { + BooleanQuery bq = new BooleanQuery(); + bq.add(this.makeXDL(crossedDateLine), BooleanClause.Occur.MUST); + bq.add(query, BooleanClause.Occur.MUST); + return bq; + } +} + + + diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/package-info.java b/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/package-info.java new file mode 100644 index 0000000..312ad0d --- /dev/null +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/package-info.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Bounding Box Spatial Strategy + * + * Index a shape extent using 4 numeric fields and a flag to say if it crosses the dateline + */ +package org.apache.lucene.spatial.bbox; + diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PointPrefixTreeFieldCacheProvider.java b/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PointPrefixTreeFieldCacheProvider.java index 3088d3d..5a17741 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PointPrefixTreeFieldCacheProvider.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PointPrefixTreeFieldCacheProvider.java @@ -23,6 +23,9 @@ import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; import org.apache.lucene.spatial.util.ShapeFieldCacheProvider; import org.apache.lucene.util.BytesRef; +/** + * @lucene.internal + */ public class PointPrefixTreeFieldCacheProvider extends ShapeFieldCacheProvider { final SpatialPrefixTree grid; // diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixCellsTokenizer.java b/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixCellsTokenizer.java index a552804..ca59042 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixCellsTokenizer.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixCellsTokenizer.java @@ -25,7 +25,7 @@ import java.io.Reader; /** - * + * @lucene.internal */ class PrefixCellsTokenizer extends Tokenizer { public PrefixCellsTokenizer(Reader input) { diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java b/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java index f854a07..2a96102 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java @@ -1,3 +1,5 @@ +package org.apache.lucene.spatial.prefix; + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -15,10 +17,7 @@ * limitations under the License. */ -package org.apache.lucene.spatial.prefix; - import com.spatial4j.core.distance.DistanceCalculator; -import com.spatial4j.core.query.SpatialArgs; import com.spatial4j.core.shape.Point; import com.spatial4j.core.shape.Shape; import org.apache.lucene.analysis.TokenStream; @@ -28,26 +27,28 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.queries.function.ValueSource; -import org.apache.lucene.spatial.SimpleSpatialFieldInfo; import org.apache.lucene.spatial.SpatialStrategy; import org.apache.lucene.spatial.prefix.tree.Node; import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; +import org.apache.lucene.spatial.query.SpatialArgs; import org.apache.lucene.spatial.util.CachedDistanceValueSource; -import java.io.IOException; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -public abstract class PrefixTreeStrategy extends SpatialStrategy { +/** + * @lucene.internal + */ +public abstract class PrefixTreeStrategy extends SpatialStrategy { protected final SpatialPrefixTree grid; private final Map provider = new ConcurrentHashMap(); protected int defaultFieldValuesArrayLen = 2; protected double distErrPct = SpatialArgs.DEFAULT_DIST_PRECISION; - public PrefixTreeStrategy(SpatialPrefixTree grid) { - super(grid.getSpatialContext()); + public PrefixTreeStrategy(SpatialPrefixTree grid, String fieldName) { + super(grid.getSpatialContext(), fieldName); this.grid = grid; } @@ -62,7 +63,7 @@ public abstract class PrefixTreeStrategy extends SpatialStrategy cells = grid.getNodes(shape, detailLevel, true);//true=intermediates cells //If shape isn't a point, add a full-resolution center-point so that @@ -74,7 +75,10 @@ public abstract class PrefixTreeStrategy extends SpatialStrategy cells = grid.getNodes(qshape, detailLevel, false); - - BooleanQuery booleanQuery = new BooleanQuery(); + public Filter makeFilter(SpatialArgs args) { + final SpatialOperation op = args.getOperation(); + if (! SpatialOperation.is(op, SpatialOperation.IsWithin, SpatialOperation.Intersects, SpatialOperation.BBoxWithin, SpatialOperation.BBoxIntersects)) + throw new UnsupportedSpatialOperation(op); + + Shape shape = args.getShape(); + int detailLevel = grid.getMaxLevelForPrecision(shape, args.getDistPrecision()); + List cells = grid.getNodes(shape, detailLevel, false); + TermsFilter filter = new TermsFilter(); for (Node cell : cells) { - booleanQuery.add(new TermQuery(new Term(fieldInfo.getFieldName(), cell.getTokenString())), BooleanClause.Occur.SHOULD); + filter.addTerm(new Term(getFieldName(), cell.getTokenString())); } - return booleanQuery; + return filter; } } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java b/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java index 2c576c0..518f598 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java @@ -30,6 +30,8 @@ import java.util.List; /** * A SpatialPrefixGrid based on Geohashes. Uses {@link GeohashUtils} to do all the geohash work. + * + * @lucene.experimental */ public class GeohashPrefixTree extends SpatialPrefixTree { diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/Node.java b/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/Node.java index 62c7616..9fe63e6 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/Node.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/Node.java @@ -28,6 +28,8 @@ import java.util.List; /** * Represents a grid cell. These are not necessarily threadsafe, although new Cell("") (world cell) must be. + * + * @lucene.experimental */ public abstract class Node implements Comparable { public static final byte LEAF_BYTE = '+';//NOTE: must sort before letters & numbers diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java b/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java index e203b4a..2ed1c0a 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java @@ -29,7 +29,9 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; - +/** + * @lucene.experimental + */ public class QuadPrefixTree extends SpatialPrefixTree { public static class Factory extends SpatialPrefixTreeFactory { diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java b/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java index 11699e5..e7b6bf7 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java @@ -1,3 +1,5 @@ +package org.apache.lucene.spatial.prefix.tree; + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -15,8 +17,6 @@ * limitations under the License. */ -package org.apache.lucene.spatial.prefix.tree; - import com.spatial4j.core.context.SpatialContext; import com.spatial4j.core.shape.Point; import com.spatial4j.core.shape.Shape; @@ -32,6 +32,8 @@ import java.util.List; * variable precision. Each string corresponds to a spatial region. * * Implementations of this class should be thread-safe and immutable once initialized. + * + * @lucene.experimental */ public abstract class SpatialPrefixTree { @@ -61,10 +63,9 @@ public abstract class SpatialPrefixTree { } /** - * See {@link com.spatial4j.core.query.SpatialArgs#getDistPrecision()}. + * See {@link org.apache.lucene.spatial.query.SpatialArgs#getDistPrecision()}. * A grid level looked up via {@link #getLevelForDistance(double)} is returned. * - * @param shape * @param precision 0-0.5 * @return 1-maxLevels */ diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeFactory.java b/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeFactory.java index 88a1fb6..bb84782 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeFactory.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeFactory.java @@ -24,6 +24,7 @@ import com.spatial4j.core.distance.DistanceUtils; import java.util.Map; /** + * @lucene.experimental */ public abstract class SpatialPrefixTreeFactory { diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgs.java b/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgs.java new file mode 100644 index 0000000..dd4c154 --- /dev/null +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgs.java @@ -0,0 +1,133 @@ +package org.apache.lucene.spatial.query; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.spatial4j.core.context.SpatialContext; +import com.spatial4j.core.context.simple.SimpleSpatialContext; +import com.spatial4j.core.exception.InvalidSpatialArgument; +import com.spatial4j.core.shape.Shape; + +/** + * Principally holds the query {@link Shape} and the {@link SpatialOperation}. + * + * @lucene.experimental + */ +public class SpatialArgs { + + public static final double DEFAULT_DIST_PRECISION = 0.025d; + + private SpatialOperation operation; + private Shape shape; + private double distPrecision = DEFAULT_DIST_PRECISION; + + // Useful for 'distance' calculations + private Double min; + private Double max; + + public SpatialArgs(SpatialOperation operation) { + this.operation = operation; + } + + public SpatialArgs(SpatialOperation operation, Shape shape) { + this.operation = operation; + this.shape = shape; + } + + /** Check if the arguments make sense -- throw an exception if not */ + public void validate() throws InvalidSpatialArgument { + if (operation.isTargetNeedsArea() && !shape.hasArea()) { + throw new InvalidSpatialArgument(operation + " only supports geometry with area"); + } + } + + public String toString(SpatialContext context) { + StringBuilder str = new StringBuilder(); + str.append(operation.getName()).append('('); + str.append(context.toString(shape)); + if (min != null) { + str.append(" min=").append(min); + } + if (max != null) { + str.append(" max=").append(max); + } + str.append(" distPrec=").append(String.format("%.2f%%", distPrecision / 100d)); + str.append(')'); + return str.toString(); + } + + @Override + public String toString() { + return toString(SimpleSpatialContext.GEO_KM); + } + + //------------------------------------------------ + // Getters & Setters + //------------------------------------------------ + + public SpatialOperation getOperation() { + return operation; + } + + public void setOperation(SpatialOperation operation) { + this.operation = operation; + } + + /** Considers {@link SpatialOperation#BBoxWithin} in returning the shape. */ + public Shape getShape() { + if (shape != null && (operation == SpatialOperation.BBoxWithin || operation == SpatialOperation.BBoxIntersects)) + return shape.getBoundingBox(); + return shape; + } + + public void setShape(Shape shape) { + this.shape = shape; + } + + /** + * The fraction of the distance from the center of the query shape to its nearest edge + * that is considered acceptable error. The algorithm for computing the distance to the + * nearest edge is actually a little different. It normalizes the shape to a square + * given it's bounding box area: + * 

      sqrt(shape.bbox.area)/2
      + * And the error distance is beyond the shape such that the shape is a minimum shape. + */ + public Double getDistPrecision() { + return distPrecision; + } + + public void setDistPrecision(Double distPrecision) { + if (distPrecision != null) + this.distPrecision = distPrecision; + } + + public Double getMin() { + return min; + } + + public void setMin(Double min) { + this.min = min; + } + + public Double getMax() { + return max; + } + + public void setMax(Double max) { + this.max = max; + } +} diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgsParser.java b/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgsParser.java new file mode 100644 index 0000000..a521eb3 --- /dev/null +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgsParser.java @@ -0,0 +1,113 @@ +package org.apache.lucene.spatial.query; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.spatial4j.core.context.SpatialContext; +import com.spatial4j.core.exception.InvalidShapeException; +import com.spatial4j.core.exception.InvalidSpatialArgument; +import com.spatial4j.core.shape.Shape; + +import java.util.HashMap; +import java.util.Map; +import java.util.StringTokenizer; + +/** + * Parses a string that usually looks like "OPERATION(SHAPE)" into a {@link SpatialArgs} + * object. The set of operations supported are defined in {@link SpatialOperation}, such + * as "Intersects" being a common one. The shape portion is defined by {@link + * SpatialContext#readShape(String)}. There are some optional name-value pair parameters + * that follow the closing parenthesis. Example: + * 
      + *   Intersects(-10,20,-8,22) distPec=0.025
+ *
      + *

      + * In the future it would be good to support something at least semi-standardized like a + * variant of + * [E]CQL. + * + * @lucene.experimental + */ +public class SpatialArgsParser { + + /** + * Parses a string such as "Intersects(-10,20,-8,22) distPec=0.025". + * + * @param v The string to parse. Mandatory. + * @param ctx The spatial context. Mandatory. + * @return Not null. + * @throws InvalidSpatialArgument If there is a problem parsing the string. + * @throws InvalidShapeException Thrown from {@link SpatialContext#readShape(String)} + */ + public SpatialArgs parse(String v, SpatialContext ctx) throws InvalidSpatialArgument, InvalidShapeException { + int idx = v.indexOf('('); + int edx = v.lastIndexOf(')'); + + if (idx < 0 || idx > edx) { + throw new InvalidSpatialArgument("missing parens: " + v, null); + } + + SpatialOperation op = SpatialOperation.get(v.substring(0, idx).trim()); + + String body = v.substring(idx + 1, edx).trim(); + if (body.length() < 1) { + throw new InvalidSpatialArgument("missing body : " + v, null); + } + + Shape shape = ctx.readShape(body); + SpatialArgs args = new SpatialArgs(op, shape); + + if (v.length() > (edx + 1)) { + body = v.substring(edx + 1).trim(); + if (body.length() > 0) { + Map aa = parseMap(body); + args.setMin(readDouble(aa.remove("min"))); + args.setMax(readDouble(aa.remove("max"))); + args.setDistPrecision(readDouble(aa.remove("distPrec"))); + if (!aa.isEmpty()) { + throw new InvalidSpatialArgument("unused parameters: " + aa, null); + } + } + } + return args; + } + + protected static Double readDouble(String v) { + return v == null ? null : Double.valueOf(v); + } + + protected static boolean readBool(String v, boolean defaultValue) { + return v == null ? defaultValue : Boolean.parseBoolean(v); + } + + protected static Map parseMap(String body) { + Map map = new HashMap(); + StringTokenizer st = new StringTokenizer(body, " \n\t"); + while (st.hasMoreTokens()) { + String a = st.nextToken(); + int idx = a.indexOf('='); + if (idx > 0) { + String k = a.substring(0, idx); + String v = a.substring(idx + 1); + map.put(k, v); + } else { + map.put(a, a); + } + } + return map; + } +} diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialOperation.java b/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialOperation.java new file mode 100644 index 0000000..9a37321 --- /dev/null +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialOperation.java @@ -0,0 +1,115 @@ +package org.apache.lucene.spatial.query; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.spatial4j.core.exception.InvalidSpatialArgument; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * A clause that compares a stored geometry to a supplied geometry. + * + * @see + * ESRIs docs on spatial relations + * + * @lucene.experimental + */ +public class SpatialOperation implements Serializable { + // Private registry + private static final Map registry = new HashMap(); + private static final List list = new ArrayList(); + + // Geometry Operations + public static final SpatialOperation BBoxIntersects = new SpatialOperation("BBoxIntersects", true, false, false); + public static final SpatialOperation BBoxWithin = new SpatialOperation("BBoxWithin", true, false, false); + public static final SpatialOperation Contains = new SpatialOperation("Contains", true, true, false); + public static final SpatialOperation Intersects = new SpatialOperation("Intersects", true, false, false); + public static final SpatialOperation IsEqualTo = new SpatialOperation("IsEqualTo", false, false, false); + public static final SpatialOperation IsDisjointTo = new SpatialOperation("IsDisjointTo", false, false, false); + public static final SpatialOperation IsWithin = new SpatialOperation("IsWithin", true, false, true); + public static final SpatialOperation Overlaps = new SpatialOperation("Overlaps", true, false, true); + + // Member variables + private final boolean scoreIsMeaningful; + private final boolean sourceNeedsArea; + private final boolean targetNeedsArea; + private final String name; + + protected SpatialOperation(String name, boolean scoreIsMeaningful, boolean sourceNeedsArea, boolean targetNeedsArea) { + this.name = name; + this.scoreIsMeaningful = scoreIsMeaningful; + this.sourceNeedsArea = sourceNeedsArea; + this.targetNeedsArea = targetNeedsArea; + registry.put(name, this); + registry.put(name.toUpperCase(Locale.US), this); + list.add( this ); + } + + public static SpatialOperation get( String v ) { + SpatialOperation op = registry.get( v ); + if( op == null ) { + op = registry.get(v.toUpperCase(Locale.US)); + } + if( op == null ) { + throw new InvalidSpatialArgument("Unknown Operation: " + v ); + } + return op; + } + + public static List values() { + return list; + } + + public static boolean is( SpatialOperation op, SpatialOperation ... tst ) { + for( SpatialOperation t : tst ) { + if( op == t ) { + return true; + } + } + return false; + } + + + // ================================================= Getters / Setters ============================================= + + public boolean isScoreIsMeaningful() { + return scoreIsMeaningful; + } + + public boolean isSourceNeedsArea() { + return sourceNeedsArea; + } + + public boolean isTargetNeedsArea() { + return targetNeedsArea; + } + + public String getName() { + return name; + } + + @Override + public String toString() { + return name; + } +} diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/query/UnsupportedSpatialOperation.java b/lucene/spatial/src/java/org/apache/lucene/spatial/query/UnsupportedSpatialOperation.java new file mode 100644 index 0000000..531d59f --- /dev/null +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/query/UnsupportedSpatialOperation.java @@ -0,0 +1,28 @@ +package org.apache.lucene.spatial.query; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @lucene.experimental + */ +public class UnsupportedSpatialOperation extends UnsupportedOperationException { + + public UnsupportedSpatialOperation(SpatialOperation op) { + super(op.getName()); + } +} diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/query/package-info.java b/lucene/spatial/src/java/org/apache/lucene/spatial/query/package-info.java new file mode 100644 index 0000000..120004d --- /dev/null +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/query/package-info.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Spatial Query options + *

        + *
      • useful for client side requests
        • + *
      + */ +package org.apache.lucene.spatial.query; + diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/CachedDistanceValueSource.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/CachedDistanceValueSource.java index 49e9ec3..49bb4de 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/CachedDistanceValueSource.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/CachedDistanceValueSource.java @@ -28,9 +28,9 @@ import java.util.List; import java.util.Map; /** - * * An implementation of the Lucene ValueSource model to support spatial relevance ranking. * + * @lucene.internal */ public class CachedDistanceValueSource extends ValueSource { diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/CachingDoubleValueSource.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/CachingDoubleValueSource.java index 3842b07..678e2a5 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/CachingDoubleValueSource.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/CachingDoubleValueSource.java @@ -25,6 +25,9 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +/** + * @lucene.internal + */ public class CachingDoubleValueSource extends ValueSource { final ValueSource source; diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/NumericFieldInfo.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/NumericFieldInfo.java index 1300aaf..c4d137e 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/NumericFieldInfo.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/NumericFieldInfo.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.IndexableField; /** * Hold some of the parameters used by solr... + * @lucene.experimental */ public class NumericFieldInfo { public int precisionStep = 8; // same as solr default @@ -39,7 +40,7 @@ public class NumericFieldInfo { if (!store && !index) throw new IllegalArgumentException("field must be indexed or stored"); - FieldType fieldType = new FieldType(DoubleField.TYPE); + FieldType fieldType = new FieldType(DoubleField.TYPE_NOT_STORED); fieldType.setStored(store); fieldType.setIndexed(index); fieldType.setNumericPrecisionStep(precisionStep); diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/ShapeFieldCache.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/ShapeFieldCache.java index 45b21ec..93a829f 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/ShapeFieldCache.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/ShapeFieldCache.java @@ -22,6 +22,9 @@ import com.spatial4j.core.shape.Shape; import java.util.ArrayList; import java.util.List; +/** + * @lucene.internal + */ public class ShapeFieldCache { private List[] cache; public int defaultLength; diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/ShapeFieldCacheProvider.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/ShapeFieldCacheProvider.java index 1eff3c3..4d1b906 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/ShapeFieldCacheProvider.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/ShapeFieldCacheProvider.java @@ -26,6 +26,9 @@ import java.io.IOException; import java.util.WeakHashMap; import java.util.logging.Logger; +/** + * @lucene.internal + */ public abstract class ShapeFieldCacheProvider { private Logger log = Logger.getLogger(getClass().getName()); diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/StringListTokenizer.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/StringListTokenizer.java index 6c2ab43..eda4e86 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/StringListTokenizer.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/StringListTokenizer.java @@ -24,7 +24,8 @@ import java.io.IOException; import java.util.Iterator; /** - * Put a list of strings directly into the token stream + * Put a list of strings directly into the token stream. + * @lucene.internal */ public final class StringListTokenizer extends TokenStream { diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/TruncateFilter.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/TruncateFilter.java index bc01443..5c3553c 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/TruncateFilter.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/TruncateFilter.java @@ -24,6 +24,9 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import java.io.IOException; +/** + * @lucene.internal + */ public class TruncateFilter extends TokenFilter { private final int maxTokenLength; diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/ValueSourceFilter.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/ValueSourceFilter.java index 57130de..1a430b7 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/ValueSourceFilter.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/ValueSourceFilter.java @@ -27,6 +27,9 @@ import org.apache.lucene.util.Bits; import java.io.IOException; +/** + * @lucene.internal + */ public class ValueSourceFilter extends Filter { final Filter startingFilter; diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java b/lucene/spatial/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java index fd977ad..99a84ca 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java @@ -1,3 +1,5 @@ +package org.apache.lucene.spatial.vector; + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -15,42 +17,36 @@ * limitations under the License. */ -package org.apache.lucene.spatial.vector; - import com.spatial4j.core.distance.DistanceCalculator; import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.simple.PointImpl; import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.FieldCache; -import org.apache.lucene.search.FieldCache.DoubleParser; import org.apache.lucene.util.Bits; import java.io.IOException; import java.util.Map; /** - * * An implementation of the Lucene ValueSource model to support spatial relevance ranking. * + * @lucene.internal */ public class DistanceValueSource extends ValueSource { - private final TwoDoublesFieldInfo fields; - private final DistanceCalculator calculator; + private TwoDoublesStrategy strategy; private final Point from; - private final DoubleParser parser; + private final DistanceCalculator calculator; /** * Constructor. */ - public DistanceValueSource(Point from, DistanceCalculator calc, TwoDoublesFieldInfo fields, DoubleParser parser) { + public DistanceValueSource(TwoDoublesStrategy strategy, Point from, DistanceCalculator calc) { + this.strategy = strategy; this.from = from; - this.fields = fields; this.calculator = calc; - this.parser = parser; } /** @@ -69,10 +65,10 @@ public class DistanceValueSource extends ValueSource { public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException { AtomicReader reader = readerContext.reader(); - final double[] ptX = FieldCache.DEFAULT.getDoubles(reader, fields.getFieldNameX(), true); - final double[] ptY = FieldCache.DEFAULT.getDoubles(reader, fields.getFieldNameY(), true); - final Bits validX = FieldCache.DEFAULT.getDocsWithField(reader, fields.getFieldNameX()); - final Bits validY = FieldCache.DEFAULT.getDocsWithField(reader, fields.getFieldNameY()); + final double[] ptX = FieldCache.DEFAULT.getDoubles(reader, strategy.getFieldNameX(), true); + final double[] ptY = FieldCache.DEFAULT.getDoubles(reader, strategy.getFieldNameY(), true); + final Bits validX = FieldCache.DEFAULT.getDocsWithField(reader, strategy.getFieldNameX()); + final Bits validY = FieldCache.DEFAULT.getDocsWithField(reader, strategy.getFieldNameY()); return new FunctionValues() { @Override @@ -84,8 +80,7 @@ public class DistanceValueSource extends ValueSource { public double doubleVal(int doc) { // make sure it has minX and area if (validX.get(doc) && validY.get(doc)) { - PointImpl pt = new PointImpl( ptX[doc], ptY[doc] ); - return calculator.distance(from, pt); + return calculator.distance(from, ptX[doc], ptY[doc]); } return 0; } @@ -110,7 +105,7 @@ public class DistanceValueSource extends ValueSource { DistanceValueSource that = (DistanceValueSource) o; if (calculator != null ? !calculator.equals(that.calculator) : that.calculator != null) return false; - if (fields != null ? !fields.equals(that.fields) : that.fields != null) return false; + if (strategy != null ? !strategy.equals(that.strategy) : that.strategy != null) return false; if (from != null ? !from.equals(that.from) : that.from != null) return false; return true; @@ -118,7 +113,7 @@ public class DistanceValueSource extends ValueSource { @Override public int hashCode() { - int result = fields != null ? fields.hashCode() : 0; + int result = strategy != null ? strategy.hashCode() : 0; result = 31 * result + (calculator != null ? calculator.hashCode() : 0); result = 31 * result + (from != null ? from.hashCode() : 0); return result; diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/vector/TwoDoublesFieldInfo.java b/lucene/spatial/src/java/org/apache/lucene/spatial/vector/TwoDoublesFieldInfo.java deleted file mode 100644 index 7801d5a..0000000 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/vector/TwoDoublesFieldInfo.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.lucene.spatial.vector; - -import org.apache.lucene.spatial.SpatialFieldInfo; - -public class TwoDoublesFieldInfo implements SpatialFieldInfo { - - public static final String SUFFIX_X = "__x"; - public static final String SUFFIX_Y = "__y"; - - private final String fieldName; - private final String fieldNameX; - private final String fieldNameY; - - public TwoDoublesFieldInfo(String fieldNamePrefix) { - fieldName = fieldNamePrefix; - fieldNameX = fieldNamePrefix + SUFFIX_X; - fieldNameY = fieldNamePrefix + SUFFIX_Y; - } - - public String getFieldName() { - return fieldName; - } - - public String getFieldNameX() { - return fieldNameX; - } - - public String getFieldNameY() { - return fieldNameY; - } -} diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/vector/TwoDoublesStrategy.java b/lucene/spatial/src/java/org/apache/lucene/spatial/vector/TwoDoublesStrategy.java index 559b840..2a69f7a 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/vector/TwoDoublesStrategy.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/vector/TwoDoublesStrategy.java @@ -1,3 +1,5 @@ +package org.apache.lucene.spatial.vector; + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -15,38 +17,64 @@ * limitations under the License. */ -package org.apache.lucene.spatial.vector; - import com.spatial4j.core.context.SpatialContext; import com.spatial4j.core.exception.InvalidShapeException; -import com.spatial4j.core.exception.UnsupportedSpatialOperation; -import com.spatial4j.core.query.SpatialArgs; -import com.spatial4j.core.query.SpatialOperation; import com.spatial4j.core.shape.Circle; import com.spatial4j.core.shape.Point; import com.spatial4j.core.shape.Rectangle; import com.spatial4j.core.shape.Shape; +import org.apache.lucene.document.DoubleField; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexableField; import org.apache.lucene.queries.function.FunctionQuery; import org.apache.lucene.queries.function.ValueSource; -import org.apache.lucene.search.*; -import org.apache.lucene.search.FieldCache.DoubleParser; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Filter; +import org.apache.lucene.search.FilteredQuery; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.NumericRangeQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryWrapperFilter; import org.apache.lucene.spatial.SpatialStrategy; +import org.apache.lucene.spatial.query.SpatialArgs; +import org.apache.lucene.spatial.query.SpatialOperation; +import org.apache.lucene.spatial.query.UnsupportedSpatialOperation; import org.apache.lucene.spatial.util.CachingDoubleValueSource; -import org.apache.lucene.spatial.util.NumericFieldInfo; import org.apache.lucene.spatial.util.ValueSourceFilter; -public class TwoDoublesStrategy extends SpatialStrategy { +/** + * @lucene.experimental + */ +public class TwoDoublesStrategy extends SpatialStrategy { + + public static final String SUFFIX_X = "__x"; + public static final String SUFFIX_Y = "__y"; + + private final String fieldNameX; + private final String fieldNameY; - private final NumericFieldInfo finfo; - private final DoubleParser parser; + public int precisionStep = 8; // same as solr default - public TwoDoublesStrategy(SpatialContext ctx, NumericFieldInfo finfo, DoubleParser parser) { - super(ctx); - this.finfo = finfo; - this.parser = parser; + public TwoDoublesStrategy(SpatialContext ctx, String fieldNamePrefix) { + super(ctx, fieldNamePrefix); + this.fieldNameX = fieldNamePrefix+SUFFIX_X; + this.fieldNameY = fieldNamePrefix+SUFFIX_Y; + } + + public void setPrecisionStep( int p ) { + precisionStep = p; + if (precisionStep<=0 || precisionStep>=64) + precisionStep=Integer.MAX_VALUE; + } + + String getFieldNameX() { + return fieldNameX; + } + + String getFieldNameY() { + return fieldNameY; } @Override @@ -55,20 +83,19 @@ public class TwoDoublesStrategy extends SpatialStrategy { } @Override - public IndexableField[] createFields(TwoDoublesFieldInfo fieldInfo, - Shape shape, boolean index, boolean store) { + public IndexableField[] createFields(Shape shape, boolean index, boolean store) { if( shape instanceof Point ) { Point point = (Point)shape; IndexableField[] f = new IndexableField[(index ? 2 : 0) + (store ? 1 : 0)]; if (index) { - f[0] = finfo.createDouble( fieldInfo.getFieldNameX(), point.getX() ); - f[1] = finfo.createDouble( fieldInfo.getFieldNameY(), point.getY() ); + f[0] = createDouble(fieldNameX, point.getX(), index, store); + f[1] = createDouble(fieldNameY, point.getY(), index, store); } if(store) { FieldType customType = new FieldType(); customType.setStored(true); - f[f.length-1] = new Field( fieldInfo.getFieldName(), ctx.toString( shape ), customType ); + f[f.length-1] = new Field( getFieldName(), ctx.toString( shape ), customType ); } return f; } @@ -78,45 +105,59 @@ public class TwoDoublesStrategy extends SpatialStrategy { return new IndexableField[0]; // nothing (solr does not support null) } + private IndexableField createDouble(String name, double v, boolean index, boolean store) { + if (!store && !index) + throw new IllegalArgumentException("field must be indexed or stored"); + + FieldType fieldType = new FieldType(DoubleField.TYPE_NOT_STORED); + fieldType.setStored(store); + fieldType.setIndexed(index); + fieldType.setNumericPrecisionStep(precisionStep); + return new DoubleField(name,v,fieldType); + } + @Override - public IndexableField createField(TwoDoublesFieldInfo indexInfo, Shape shape, - boolean index, boolean store) { + public IndexableField createField(Shape shape, + boolean index, boolean store) { throw new UnsupportedOperationException("Point is poly field"); } @Override - public ValueSource makeValueSource(SpatialArgs args, TwoDoublesFieldInfo fieldInfo) { + public ValueSource makeValueSource(SpatialArgs args) { Point p = args.getShape().getCenter(); - return new DistanceValueSource(p, ctx.getDistCalc(), fieldInfo, parser); + return new DistanceValueSource(this, p, ctx.getDistCalc()); } @Override - public Filter makeFilter(SpatialArgs args, TwoDoublesFieldInfo fieldInfo) { + public Filter makeFilter(SpatialArgs args) { if( args.getShape() instanceof Circle) { if( SpatialOperation.is( args.getOperation(), SpatialOperation.Intersects, SpatialOperation.IsWithin )) { Circle circle = (Circle)args.getShape(); - Query bbox = makeWithin(circle.getBoundingBox(), fieldInfo); + Query bbox = makeWithin(circle.getBoundingBox()); // Make the ValueSource - ValueSource valueSource = makeValueSource(args, fieldInfo); + ValueSource valueSource = makeValueSource(args); return new ValueSourceFilter( new QueryWrapperFilter( bbox ), valueSource, 0, circle.getDistance() ); } } - return new QueryWrapperFilter( makeQuery(args, fieldInfo) ); + return new QueryWrapperFilter( makeQuery(args) ); } @Override - public Query makeQuery(SpatialArgs args, TwoDoublesFieldInfo fieldInfo) { + public Query makeQuery(SpatialArgs args) { // For starters, just limit the bbox Shape shape = args.getShape(); - if (!(shape instanceof Rectangle)) { - throw new InvalidShapeException("A rectangle is the only supported shape (so far), not "+shape.getClass());//TODO + if (!(shape instanceof Rectangle || shape instanceof Circle)) { + throw new InvalidShapeException("Only Rectangles and Circles are currently supported, " + + "found [" + shape.getClass() + "]");//TODO } - Rectangle bbox = (Rectangle) shape; + + Rectangle bbox = shape.getBoundingBox(); + if (bbox.getCrossesDateLine()) { throw new UnsupportedOperationException( "Crossing dateline not yet supported" ); } @@ -129,17 +170,17 @@ public class TwoDoublesStrategy extends SpatialStrategy { if( SpatialOperation.is( op, SpatialOperation.BBoxWithin, SpatialOperation.BBoxIntersects ) ) { - spatial = makeWithin(bbox, fieldInfo); + spatial = makeWithin(bbox); } else if( SpatialOperation.is( op, SpatialOperation.Intersects, SpatialOperation.IsWithin ) ) { - spatial = makeWithin(bbox, fieldInfo); + spatial = makeWithin(bbox); if( args.getShape() instanceof Circle) { Circle circle = (Circle)args.getShape(); // Make the ValueSource - valueSource = makeValueSource(args, fieldInfo); + valueSource = makeValueSource(args); ValueSourceFilter vsf = new ValueSourceFilter( new QueryWrapperFilter( spatial ), valueSource, 0, circle.getDistance() ); @@ -148,7 +189,7 @@ public class TwoDoublesStrategy extends SpatialStrategy { } } else if( op == SpatialOperation.IsDisjointTo ) { - spatial = makeDisjoint(bbox, fieldInfo); + spatial = makeDisjoint(bbox); } if( spatial == null ) { @@ -159,7 +200,7 @@ public class TwoDoublesStrategy extends SpatialStrategy { valueSource = new CachingDoubleValueSource(valueSource); } else { - valueSource = makeValueSource(args, fieldInfo); + valueSource = makeValueSource(args); } Query spatialRankingQuery = new FunctionQuery(valueSource); BooleanQuery bq = new BooleanQuery(); @@ -172,17 +213,17 @@ public class TwoDoublesStrategy extends SpatialStrategy { * Constructs a query to retrieve documents that fully contain the input envelope. * @return the spatial query */ - private Query makeWithin(Rectangle bbox, TwoDoublesFieldInfo fieldInfo) { + private Query makeWithin(Rectangle bbox) { Query qX = NumericRangeQuery.newDoubleRange( - fieldInfo.getFieldNameX(), - finfo.precisionStep, + fieldNameX, + precisionStep, bbox.getMinX(), bbox.getMaxX(), true, true); Query qY = NumericRangeQuery.newDoubleRange( - fieldInfo.getFieldNameY(), - finfo.precisionStep, + fieldNameY, + precisionStep, bbox.getMinY(), bbox.getMaxY(), true, @@ -198,17 +239,17 @@ public class TwoDoublesStrategy extends SpatialStrategy { * Constructs a query to retrieve documents that fully contain the input envelope. * @return the spatial query */ - Query makeDisjoint(Rectangle bbox, TwoDoublesFieldInfo fieldInfo) { + Query makeDisjoint(Rectangle bbox) { Query qX = NumericRangeQuery.newDoubleRange( - fieldInfo.getFieldNameX(), - finfo.precisionStep, + fieldNameX, + precisionStep, bbox.getMinX(), bbox.getMaxX(), true, true); Query qY = NumericRangeQuery.newDoubleRange( - fieldInfo.getFieldNameY(), - finfo.precisionStep, + fieldNameY, + precisionStep, bbox.getMinY(), bbox.getMaxY(), true, @@ -219,6 +260,7 @@ public class TwoDoublesStrategy extends SpatialStrategy { bq.add(qY,BooleanClause.Occur.MUST_NOT); return bq; } + } diff --git a/lucene/spatial/src/test-files/data/simple-bbox.txt b/lucene/spatial/src/test-files/data/simple-bbox.txt new file mode 100644 index 0000000..dc097a6 --- /dev/null +++ b/lucene/spatial/src/test-files/data/simple-bbox.txt @@ -0,0 +1,5 @@ +#id name shape +C5 CenterAt5 -5 -5 5 5 +C10 CenterAt10 -10 -10 10 10 +NW15 NorthWest 15 15 20 20 + diff --git a/lucene/spatial/src/test-files/simple-Queries-BBox.txt b/lucene/spatial/src/test-files/simple-Queries-BBox.txt new file mode 100644 index 0000000..3cd323b --- /dev/null +++ b/lucene/spatial/src/test-files/simple-Queries-BBox.txt @@ -0,0 +1,13 @@ +C5 @ IsWithin(-6 -6 6 6) +C5 @ BBoxWithin(-6 -6 6 6) +C10 @ Contains(-6 -6 6 6) +C10 @ IsEqualTo(-10 -10 10 10) +C5 C10 @ Intersects(-2 -2 2 2) +C5 C10 @ Overlaps(-2 -2 2 2) +C5 C10 @ BBoxIntersects(-2 -2 2 2) +NW15 @ IsDisjointTo(-10 -10 10 10) + + + + + diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/PortedSolr3Test.java b/lucene/spatial/src/test/org/apache/lucene/spatial/PortedSolr3Test.java new file mode 100644 index 0000000..92b9741 --- /dev/null +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/PortedSolr3Test.java @@ -0,0 +1,217 @@ +package org.apache.lucene.spatial; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.spatial4j.core.context.SpatialContext; +import com.spatial4j.core.context.simple.SimpleSpatialContext; +import com.spatial4j.core.shape.Point; +import com.spatial4j.core.shape.Shape; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.FilteredQuery; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.TermQueryPrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; +import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; +import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; +import org.apache.lucene.spatial.query.SpatialArgs; +import org.apache.lucene.spatial.query.SpatialOperation; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Based off of Solr 3's SpatialFilterTest. + */ +public class PortedSolr3Test extends StrategyTestCase { + + @ParametersFactory + public static Iterable parameters() { + List ctorArgs = new ArrayList(); + + SpatialContext ctx = SimpleSpatialContext.GEO_KM; + SpatialPrefixTree grid; + SpatialStrategy strategy; + + grid = new GeohashPrefixTree(ctx,12); + strategy = new RecursivePrefixTreeStrategy(grid, "recursive_geohash"); + ctorArgs.add(new Object[]{strategy}); + + grid = new QuadPrefixTree(ctx,25); + strategy = new RecursivePrefixTreeStrategy(grid, "recursive_quad"); + ctorArgs.add(new Object[]{strategy}); + + grid = new GeohashPrefixTree(ctx,12); + strategy = new TermQueryPrefixTreeStrategy(grid, "termquery_geohash"); + ctorArgs.add(new Object[]{strategy}); + + return ctorArgs; + } + +// private String fieldName; + + public PortedSolr3Test(SpatialStrategy strategy) { + this.ctx = strategy.getSpatialContext(); + this.strategy = strategy; + } + + private void setupDocs() throws IOException { + super.deleteAll(); + adoc("1", "32.7693246, -79.9289094"); + adoc("2", "33.7693246, -80.9289094"); + adoc("3", "-32.7693246, 50.9289094"); + adoc("4", "-50.7693246, 60.9289094"); + adoc("5", "0,0"); + adoc("6", "0.1,0.1"); + adoc("7", "-0.1,-0.1"); + adoc("8", "0,179.9"); + adoc("9", "0,-179.9"); + adoc("10", "89.9,50"); + adoc("11", "89.9,-130"); + adoc("12", "-89.9,50"); + adoc("13", "-89.9,-130"); + commit(); + } + + + @Test + public void testIntersections() throws Exception { + setupDocs(); + //Try some edge cases + checkHitsCircle("1,1", 175, 3, 5, 6, 7); + checkHitsCircle("0,179.8", 200, 2, 8, 9); + checkHitsCircle("89.8, 50", 200, 2, 10, 11);//this goes over the north pole + checkHitsCircle("-89.8, 50", 200, 2, 12, 13);//this goes over the south pole + //try some normal cases + checkHitsCircle("33.0,-80.0", 300, 2); + //large distance + checkHitsCircle("1,1", 5000, 3, 5, 6, 7); + //Because we are generating a box based on the west/east longitudes and the south/north latitudes, which then + //translates to a range query, which is slightly more inclusive. Thus, even though 0.0 is 15.725 kms away, + //it will be included, b/c of the box calculation. + checkHitsBBox("0.1,0.1", 15, 2, 5, 6); + //try some more + deleteAll(); + adoc("14", "0,5"); + adoc("15", "0,15"); + //3000KM from 0,0, see http://www.movable-type.co.uk/scripts/latlong.html + adoc("16", "18.71111,19.79750"); + adoc("17", "44.043900,-95.436643"); + commit(); + + checkHitsCircle("0,0", 1000, 1, 14); + checkHitsCircle("0,0", 2000, 2, 14, 15); + checkHitsBBox("0,0", 3000, 3, 14, 15, 16); + checkHitsCircle("0,0", 3001, 3, 14, 15, 16); + checkHitsCircle("0,0", 3000.1, 3, 14, 15, 16); + + //really fine grained distance and reflects some of the vagaries of how we are calculating the box + checkHitsCircle("43.517030,-96.789603", 109, 0); + + // falls outside of the real distance, but inside the bounding box + checkHitsCircle("43.517030,-96.789603", 110, 0); + checkHitsBBox("43.517030,-96.789603", 110, 1, 17); + } + + /** + * This test is similar to a Solr 3 spatial test. + */ + @Test + public void testDistanceOrder() throws IOException { + adoc("100","1,2"); + adoc("101","4,-1"); + commit(); + + //query closer to #100 + checkHitsOrdered("Intersects(Circle(3,4 d=1000))", "101", "100"); + //query closer to #101 + checkHitsOrdered("Intersects(Circle(4,0 d=1000))", "100", "101"); + } + + private void checkHitsOrdered(String spatialQ, String... ids) { + SpatialArgs args = this.argsParser.parse(spatialQ,ctx); + Query query = strategy.makeQuery(args); + SearchResults results = executeQuery(query, 100); + String[] resultIds = new String[results.numFound]; + int i = 0; + for (SearchResult result : results.results) { + resultIds[i++] = result.document.get("id"); + } + assertArrayEquals("order matters",ids, resultIds); + } + + //---- these are similar to Solr test methods + + private void adoc(String idStr, String shapeStr) throws IOException { + Shape shape = ctx.readShape(shapeStr); + addDocument(newDoc(idStr,shape)); + } + + @SuppressWarnings("unchecked") + private Document newDoc(String id, Shape shape) { + Document doc = new Document(); + doc.add(new StringField("id", id, Field.Store.YES)); + for (IndexableField f : strategy.createFields(shape, true, storeShape)) { + doc.add(f); + } + return doc; + } + + private void checkHitsCircle(String ptStr, double dist, int assertNumFound, int... assertIds) { + _checkHits(SpatialOperation.Intersects, ptStr, dist, assertNumFound, assertIds); + } + private void checkHitsBBox(String ptStr, double dist, int assertNumFound, int... assertIds) { + _checkHits(SpatialOperation.BBoxIntersects, ptStr, dist, assertNumFound, assertIds); + } + + @SuppressWarnings("unchecked") + private void _checkHits(SpatialOperation op, String ptStr, double dist, int assertNumFound, int... assertIds) { + Point pt = (Point) ctx.readShape(ptStr); + Shape shape = ctx.makeCircle(pt,dist); + + SpatialArgs args = new SpatialArgs(op,shape); + //args.setDistPrecision(0.025); + Query query; + if (random().nextBoolean()) { + query = strategy.makeQuery(args); + } else { + query = new FilteredQuery(new MatchAllDocsQuery(),strategy.makeFilter(args)); + } + SearchResults results = executeQuery(query, 100); + assertEquals(""+shape,assertNumFound,results.numFound); + if (assertIds != null) { + Set resultIds = new HashSet(); + for (SearchResult result : results.results) { + resultIds.add(Integer.valueOf(result.document.get("id"))); + } + for (int assertId : assertIds) { + assertTrue("has " + assertId, resultIds.contains(assertId)); + } + } + } + +} diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestCase.java index 3bc278c..8d1741d 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestCase.java @@ -17,16 +17,15 @@ package org.apache.lucene.spatial; -import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.junit.After; import org.junit.Before; @@ -37,7 +36,7 @@ import java.util.*; public abstract class SpatialTestCase extends LuceneTestCase { private DirectoryReader indexReader; - private IndexWriter indexWriter; + private RandomIndexWriter indexWriter; private Directory directory; private IndexSearcher indexSearcher; @@ -45,26 +44,15 @@ public abstract class SpatialTestCase extends LuceneTestCase { @Before public void setUp() throws Exception { super.setUp(); - Random random = random(); directory = newDirectory(); - - IndexWriterConfig writerConfig = newIndexWriterConfig(random, TEST_VERSION_CURRENT, new MockAnalyzer(random)); - indexWriter = new IndexWriter(directory, writerConfig); + indexWriter = new RandomIndexWriter(random(),directory); } @Override @After public void tearDown() throws Exception { - if (indexWriter != null) { - indexWriter.close(); - } - if (indexReader != null) { - indexReader.close(); - } - if (directory != null) { - directory.close(); - } + IOUtils.close(indexWriter,indexReader,directory); super.tearDown(); } @@ -87,11 +75,8 @@ public abstract class SpatialTestCase extends LuceneTestCase { protected void commit() throws IOException { indexWriter.commit(); - if (indexReader == null) { - indexReader = DirectoryReader.open(directory); - } else { - indexReader = DirectoryReader.openIfChanged(indexReader); - } + IOUtils.close(indexReader); + indexReader = indexWriter.getReader(); indexSearcher = newSearcher(indexReader); } @@ -124,6 +109,22 @@ public abstract class SpatialTestCase extends LuceneTestCase { this.numFound = numFound; this.results = results; } + + public StringBuilder toDebugString() { + StringBuilder str = new StringBuilder(); + str.append("found: ").append(numFound).append('['); + for(SearchResult r : results) { + String id = r.document.get("id"); + str.append(id).append(", "); + } + str.append(']'); + return str; + } + + @Override + public String toString() { + return "[found:"+numFound+" "+results+"]"; + } } protected static class SearchResult { @@ -135,6 +136,11 @@ public abstract class SpatialTestCase extends LuceneTestCase { this.score = score; this.document = document; } + + @Override + public String toString() { + return "["+score+"="+document+"]"; + } } } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestQuery.java b/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestQuery.java index f512399..e165a7c 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestQuery.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestQuery.java @@ -1,3 +1,5 @@ +package org.apache.lucene.spatial; + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -15,12 +17,10 @@ * limitations under the License. */ -package org.apache.lucene.spatial; - import com.spatial4j.core.context.SpatialContext; import com.spatial4j.core.io.LineReader; -import com.spatial4j.core.query.SpatialArgs; -import com.spatial4j.core.query.SpatialArgsParser; +import org.apache.lucene.spatial.query.SpatialArgs; +import org.apache.lucene.spatial.query.SpatialArgsParser; import java.io.IOException; import java.io.InputStream; diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java index 6b6833b..9e6fe82 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java @@ -1,3 +1,6 @@ +package org.apache.lucene.spatial; + + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -15,27 +18,30 @@ * limitations under the License. */ -package org.apache.lucene.spatial; - - import com.spatial4j.core.context.SpatialContext; import com.spatial4j.core.io.sample.SampleData; import com.spatial4j.core.io.sample.SampleDataReader; -import com.spatial4j.core.query.SpatialArgsParser; import com.spatial4j.core.shape.Shape; -import org.junit.Assert; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.spatial.query.SpatialArgsParser; +import org.junit.Assert; import java.io.IOException; import java.io.InputStream; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; import java.util.logging.Logger; -public abstract class StrategyTestCase extends SpatialTestCase { +public abstract class StrategyTestCase extends SpatialTestCase { + public static final String DATA_SIMPLE_BBOX = "simple-bbox.txt"; public static final String DATA_STATES_POLY = "states-poly.txt"; public static final String DATA_STATES_BBOX = "states-bbox.txt"; public static final String DATA_COUNTRIES_POLY = "countries-poly.txt"; @@ -44,16 +50,15 @@ public abstract class StrategyTestCase extends Spati public static final String QTEST_States_IsWithin_BBox = "states-IsWithin-BBox.txt"; public static final String QTEST_States_Intersects_BBox = "states-Intersects-BBox.txt"; - public static final String QTEST_Cities_IsWithin_BBox = "cities-IsWithin-BBox.txt"; + public static final String QTEST_Simple_Queries_BBox = "simple-Queries-BBox.txt"; private Logger log = Logger.getLogger(getClass().getName()); protected final SpatialArgsParser argsParser = new SpatialArgsParser(); - protected SpatialStrategy strategy; + protected SpatialStrategy strategy; protected SpatialContext ctx; - protected T fieldInfo; protected boolean storeShape = true; protected void executeQueries(SpatialMatchConcern concern, String... testQueryFile) throws IOException { @@ -76,10 +81,10 @@ public abstract class StrategyTestCase extends Spati while (sampleData.hasNext()) { SampleData data = sampleData.next(); Document document = new Document(); - document.add(new Field("id", data.id, StringField.TYPE_STORED)); - document.add(new Field("name", data.name, StringField.TYPE_STORED)); + document.add(new StringField("id", data.id, Field.Store.YES)); + document.add(new StringField("name", data.name, Field.Store.YES)); Shape shape = ctx.readShape(data.shape); - for (IndexableField f : strategy.createFields(fieldInfo, shape, true, storeShape)) { + for (IndexableField f : strategy.createFields(shape, true, storeShape)) { if( f != null ) { // null if incompatibleGeometry && ignore document.add(f); } @@ -107,13 +112,17 @@ public abstract class StrategyTestCase extends Spati SpatialTestQuery q = queries.next(); String msg = q.line; //"Query: " + q.args.toString(ctx); - SearchResults got = executeQuery(strategy.makeQuery(q.args, fieldInfo), 100); + SearchResults got = executeQuery(strategy.makeQuery(q.args), 100); if (concern.orderIsImportant) { Iterator ids = q.ids.iterator(); for (SearchResult r : got.results) { String id = r.document.get("id"); + if(!ids.hasNext()) { + Assert.fail(msg + " :: Did not get enough results. Expect" + q.ids+", got: "+got.toDebugString()); + } Assert.assertEquals( "out of order: " + msg, ids.next(), id); } + if (ids.hasNext()) { Assert.fail(msg + " :: expect more results then we got: " + ids.next()); } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/TestTestFramework.java b/lucene/spatial/src/test/org/apache/lucene/spatial/TestTestFramework.java index e35d848..b84d91f 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/TestTestFramework.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/TestTestFramework.java @@ -1,3 +1,5 @@ +package org.apache.lucene.spatial; + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -15,14 +17,11 @@ * limitations under the License. */ -package org.apache.lucene.spatial; - import com.spatial4j.core.context.SpatialContext; import com.spatial4j.core.context.simple.SimpleSpatialContext; -import com.spatial4j.core.query.SpatialArgsParser; -import com.spatial4j.core.query.SpatialOperation; import com.spatial4j.core.shape.Rectangle; - +import org.apache.lucene.spatial.query.SpatialArgsParser; +import org.apache.lucene.spatial.query.SpatialOperation; import org.apache.lucene.util.LuceneTestCase; import org.junit.Assert; import org.junit.Test; diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java b/lucene/spatial/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java new file mode 100644 index 0000000..a75930d --- /dev/null +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java @@ -0,0 +1,60 @@ +package org.apache.lucene.spatial.bbox; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.spatial4j.core.context.simple.SimpleSpatialContext; +import org.apache.lucene.spatial.SpatialMatchConcern; +import org.apache.lucene.spatial.StrategyTestCase; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +public class TestBBoxStrategy extends StrategyTestCase { + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + this.ctx = SimpleSpatialContext.GEO_KM; + this.strategy = new BBoxStrategy(ctx, "bbox"); + } + + @Test + public void testBasicOperaions() throws IOException { + getAddAndVerifyIndexedDocuments(DATA_SIMPLE_BBOX); + + executeQueries(SpatialMatchConcern.EXACT, QTEST_Simple_Queries_BBox); + } + + @Test + public void testStatesBBox() throws IOException { + getAddAndVerifyIndexedDocuments(DATA_STATES_BBOX); + + executeQueries(SpatialMatchConcern.FILTER, QTEST_States_IsWithin_BBox); + executeQueries(SpatialMatchConcern.FILTER, QTEST_States_Intersects_BBox); + } + + @Test + public void testCitiesWithinBBox() throws IOException { + getAddAndVerifyIndexedDocuments(DATA_WORLD_CITIES_POINTS); + + executeQueries(SpatialMatchConcern.FILTER, QTEST_Cities_IsWithin_BBox); + } + +} diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/BaseRecursivePrefixTreeStrategyTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/BaseRecursivePrefixTreeStrategyTestCase.java deleted file mode 100644 index bf989e6..0000000 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/BaseRecursivePrefixTreeStrategyTestCase.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.lucene.spatial.prefix; - -import com.spatial4j.core.context.SpatialContext; -import org.apache.lucene.spatial.SimpleSpatialFieldInfo; -import org.apache.lucene.spatial.SpatialMatchConcern; -import org.apache.lucene.spatial.StrategyTestCase; -import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; -import org.junit.Test; - -import java.io.IOException; - - -public abstract class BaseRecursivePrefixTreeStrategyTestCase extends StrategyTestCase { - - private int maxLength; - - protected abstract SpatialContext getSpatialContext(); - - @Override - public void setUp() throws Exception { - super.setUp(); - maxLength = GeohashPrefixTree.getMaxLevelsPossible(); - // SimpleIO - this.ctx = getSpatialContext(); - this.strategy = new RecursivePrefixTreeStrategy(new GeohashPrefixTree( - ctx, maxLength )); - this.fieldInfo = new SimpleSpatialFieldInfo( getClass().getSimpleName() ); - } - - @Test - public void testFilterWithVariableScanLevel() throws IOException { - getAddAndVerifyIndexedDocuments(DATA_WORLD_CITIES_POINTS); - - //execute queries for each prefix grid scan level - for(int i = 0; i <= maxLength; i++) { - ((RecursivePrefixTreeStrategy)strategy).setPrefixGridScanLevel(i); - executeQueries(SpatialMatchConcern.FILTER, QTEST_Cities_IsWithin_BBox); - } - } -} diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestRecursivePrefixTreeStrategy.java b/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestRecursivePrefixTreeStrategy.java index 78903d0..3e7fbf1 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestRecursivePrefixTreeStrategy.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestRecursivePrefixTreeStrategy.java @@ -1,3 +1,5 @@ +package org.apache.lucene.spatial.prefix; + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -15,22 +17,172 @@ * limitations under the License. */ -package org.apache.lucene.spatial.prefix; - -import com.spatial4j.core.context.SpatialContext; import com.spatial4j.core.context.simple.SimpleSpatialContext; -import org.junit.Before; +import com.spatial4j.core.distance.DistanceUtils; +import com.spatial4j.core.shape.Point; +import com.spatial4j.core.shape.Rectangle; +import com.spatial4j.core.shape.Shape; +import com.spatial4j.core.shape.simple.PointImpl; +import com.spatial4j.core.util.GeohashUtils; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.spatial.SpatialMatchConcern; +import org.apache.lucene.spatial.StrategyTestCase; +import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; +import org.apache.lucene.spatial.query.SpatialArgs; +import org.apache.lucene.spatial.query.SpatialOperation; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static java.lang.Math.toRadians; + +public class TestRecursivePrefixTreeStrategy extends StrategyTestCase { + + private int maxLength; + + //Tests should call this first. + private void init(int maxLength) { + this.maxLength = maxLength; + this.ctx = SimpleSpatialContext.GEO_KM; + GeohashPrefixTree grid = new GeohashPrefixTree(ctx, maxLength); + this.strategy = new RecursivePrefixTreeStrategy(grid, getClass().getSimpleName()); + } + + @Test + public void testFilterWithVariableScanLevel() throws IOException { + init(GeohashPrefixTree.getMaxLevelsPossible()); + getAddAndVerifyIndexedDocuments(DATA_WORLD_CITIES_POINTS); + + //execute queries for each prefix grid scan level + for(int i = 0; i <= maxLength; i++) { + ((RecursivePrefixTreeStrategy)strategy).setPrefixGridScanLevel(i); + executeQueries(SpatialMatchConcern.FILTER, QTEST_Cities_IsWithin_BBox); + } + } + + @Test + public void geohashRecursiveRandom() throws IOException { + init(12); + + //1. Iterate test with the cluster at some worldly point of interest + Point[] clusterCenters = new Point[]{new PointImpl(0,0), new PointImpl(0,90),new PointImpl(0,-90)}; + for (Point clusterCenter : clusterCenters) { + //2. Iterate on size of cluster (a really small one and a large one) + String hashCenter = GeohashUtils.encodeLatLon(clusterCenter.getY(), clusterCenter.getX(), maxLength); + //calculate the number of degrees in the smallest grid box size (use for both lat & lon) + String smallBox = hashCenter.substring(0,hashCenter.length()-1);//chop off leaf precision + Rectangle clusterDims = GeohashUtils.decodeBoundary(smallBox,ctx); + double smallDegrees = Math.max(clusterDims.getMaxX()-clusterDims.getMinX(),clusterDims.getMaxY()-clusterDims.getMinY()); + assert smallDegrees < 1; + double largeDegrees = 20d;//good large size; don't use >=45 for this test code to work + double[] sideDegrees = {largeDegrees,smallDegrees}; + for (double sideDegree : sideDegrees) { + //3. Index random points in this cluster box + deleteAll(); + List points = new ArrayList(); + for(int i = 0; i < 20; i++) { + double x = random().nextDouble()*sideDegree - sideDegree/2 + clusterCenter.getX(); + double y = random().nextDouble()*sideDegree - sideDegree/2 + clusterCenter.getY(); + final Point pt = normPointXY(x, y); + points.add(pt); + addDocument(newDoc("" + i, pt)); + } + commit(); + + //3. Use 4 query centers. Each is radially out from each corner of cluster box by twice distance to box edge. + for(double qcXoff : new double[]{sideDegree,-sideDegree}) {//query-center X offset from cluster center + for(double qcYoff : new double[]{sideDegree,-sideDegree}) {//query-center Y offset from cluster center + Point queryCenter = normPointXY(qcXoff + clusterCenter.getX(), + qcYoff + clusterCenter.getY()); + double[] distRange = calcDistRange(queryCenter,clusterCenter,sideDegree); + //4.1 query a small box getting nothing + checkHits(queryCenter, distRange[0]*0.99, 0, null); + //4.2 Query a large box enclosing the cluster, getting everything + checkHits(queryCenter, distRange[1]*1.01, points.size(), null); + //4.3 Query a medium box getting some (calculate the correct solution and verify) + double queryDist = distRange[0] + (distRange[1]-distRange[0])/2;//average + + //Find matching points. Put into int[] of doc ids which is the same thing as the index into points list. + int[] ids = new int[points.size()]; + int ids_sz = 0; + for (int i = 0; i < points.size(); i++) { + Point point = points.get(i); + if (ctx.getDistCalc().distance(queryCenter, point) <= queryDist) + ids[ids_sz++] = i; + } + ids = Arrays.copyOf(ids, ids_sz); + //assert ids_sz > 0 (can't because randomness keeps us from being able to) + + checkHits(queryCenter, queryDist, ids.length, ids); + } + } + }//for sideDegree -public class TestRecursivePrefixTreeStrategy extends BaseRecursivePrefixTreeStrategyTestCase { + }//for clusterCenter - @Before - public void setUp() throws Exception { - super.setUp(); + }//randomTest() + + //TODO can we use super.runTestQueries() ? + private void checkHits(Point pt, double dist, int assertNumFound, int[] assertIds) { + Shape shape = ctx.makeCircle(pt,dist); + SpatialArgs args = new SpatialArgs(SpatialOperation.Intersects,shape); + args.setDistPrecision(0.0); + SearchResults got = executeQuery(strategy.makeQuery(args), 100); + assertEquals(""+shape,assertNumFound,got.numFound); + if (assertIds != null) { + Set gotIds = new HashSet(); + for (SearchResult result : got.results) { + gotIds.add(Integer.valueOf(result.document.get("id"))); + } + for (int assertId : assertIds) { + assertTrue("has "+assertId,gotIds.contains(assertId)); + } + } + } + + // + private Document newDoc(String id, Shape shape) { + Document doc = new Document(); + doc.add(new StringField("id", id, Field.Store.YES)); + for (IndexableField f : strategy.createFields(shape, true, storeShape)) { + doc.add(f); + } + return doc; } - @Override - protected SpatialContext getSpatialContext() { - return SimpleSpatialContext.GEO_KM; + private double[] calcDistRange(Point startPoint, Point targetCenter, double targetSideDegrees) { + double min = Double.MAX_VALUE; + double max = Double.MIN_VALUE; + for(double xLen : new double[]{targetSideDegrees,-targetSideDegrees}) { + for(double yLen : new double[]{targetSideDegrees,-targetSideDegrees}) { + Point p2 = normPointXY(targetCenter.getX() + xLen / 2, targetCenter.getY() + yLen / 2); + double d = ctx.getDistCalc().distance(startPoint, p2); + min = Math.min(min,d); + max = Math.max(max,d); + } + } + return new double[]{min,max}; + } + + /** Normalize x & y (put in lon-lat ranges) & ensure geohash round-trip for given precision. */ + private Point normPointXY(double x, double y) { + //put x,y as degrees into double[] as radians + double[] latLon = {y*DistanceUtils.DEG_180_AS_RADS, toRadians(x)}; + DistanceUtils.normLatRAD(latLon); + DistanceUtils.normLatRAD(latLon); + double x2 = Math.toDegrees(latLon[1]); + double y2 = Math.toDegrees(latLon[0]); + //overwrite latLon, units is now degrees + + return GeohashUtils.decode(GeohashUtils.encodeLatLon(y2, x2, maxLength),ctx); } } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestSpatialPrefixField.java b/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestSpatialPrefixField.java deleted file mode 100644 index cbeae7b..0000000 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestSpatialPrefixField.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.lucene.spatial.prefix; - -import org.apache.lucene.util.LuceneTestCase; -import org.junit.Test; - -import java.util.Arrays; -import java.util.List; - - -/** - * This is just a quick idea for *simple* tests - */ -public class TestSpatialPrefixField extends LuceneTestCase { - - @Test - public void testRawTokens() { - // Ignoring geometry for now, and focus on what tokens need to match - - List docA = Arrays.asList( - "AAAAAA*", - "AAAAAB+" - ); - - List docB = Arrays.asList( - "A*", - "BB*" - ); - - // Assumptions: - checkQuery("AAAAA", "docA", "docB"); - checkQuery("AAAAA*", "docA", "docB"); // for now * and + are essentially identical - checkQuery("AAAAA+", "docA", "docB"); // down the road, there may be a difference between 'covers' and an edge - - checkQuery("AA*", "docB", "docA"); // Bigger input query - - checkQuery("AAAAAAAAAAAA*", "docA", "docB"); // small - - checkQuery("BC"); // nothing - checkQuery("XX"); // nothing - - // match only B - checkQuery("B", "docB"); - checkQuery("BBBB", "docB"); - checkQuery("B*", "docB"); - checkQuery("BBBB*", "docB"); - } - - void checkQuery(String query, String... expect) { - // TODO, check that the query returns the docs in order - } - -} diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestTermQueryPrefixGridStrategy.java b/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestTermQueryPrefixGridStrategy.java index b9363f3..d58b83b 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestTermQueryPrefixGridStrategy.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestTermQueryPrefixGridStrategy.java @@ -1,3 +1,5 @@ +package org.apache.lucene.spatial.prefix; + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -15,19 +17,16 @@ * limitations under the License. */ -package org.apache.lucene.spatial.prefix; - import com.spatial4j.core.context.SpatialContext; import com.spatial4j.core.context.simple.SimpleSpatialContext; -import com.spatial4j.core.query.SpatialArgsParser; import com.spatial4j.core.shape.Shape; import com.spatial4j.core.shape.simple.PointImpl; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; -import org.apache.lucene.spatial.SimpleSpatialFieldInfo; import org.apache.lucene.spatial.SpatialTestCase; import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; +import org.apache.lucene.spatial.query.SpatialArgsParser; import org.junit.Test; import java.io.IOException; @@ -38,15 +37,14 @@ public class TestTermQueryPrefixGridStrategy extends SpatialTestCase { @Test public void testNGramPrefixGridLosAngeles() throws IOException { - SimpleSpatialFieldInfo fieldInfo = new SimpleSpatialFieldInfo("geo"); SpatialContext ctx = SimpleSpatialContext.GEO_KM; - TermQueryPrefixTreeStrategy prefixGridStrategy = new TermQueryPrefixTreeStrategy(new QuadPrefixTree(ctx)); + TermQueryPrefixTreeStrategy prefixGridStrategy = new TermQueryPrefixTreeStrategy(new QuadPrefixTree(ctx), "geo"); Shape point = new PointImpl(-118.243680, 34.052230); Document losAngeles = new Document(); - losAngeles.add(new Field("name", "Los Angeles", StringField.TYPE_STORED)); - losAngeles.add(prefixGridStrategy.createField(fieldInfo, point, true, true)); + losAngeles.add(new StringField("name", "Los Angeles", Field.Store.YES)); + losAngeles.add(prefixGridStrategy.createField(point, true, true)); addDocumentsAndCommit(Arrays.asList(losAngeles)); diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/query/SpatialArgsParserTest.java b/lucene/spatial/src/test/org/apache/lucene/spatial/query/SpatialArgsParserTest.java new file mode 100644 index 0000000..67c22df --- /dev/null +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/query/SpatialArgsParserTest.java @@ -0,0 +1,65 @@ +package org.apache.lucene.spatial.query; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.spatial4j.core.context.SpatialContext; +import com.spatial4j.core.context.simple.SimpleSpatialContext; +import com.spatial4j.core.shape.Rectangle; +import org.apache.lucene.util.LuceneTestCase; +import org.junit.Test; + + +public class SpatialArgsParserTest extends LuceneTestCase { + + private SpatialContext ctx = SimpleSpatialContext.GEO_KM; + + //The args parser is only dependent on the ctx for IO so I don't care to test + // with other implementations. + + @Test + public void testArgsParser() throws Exception { + SpatialArgsParser parser = new SpatialArgsParser(); + + String arg = SpatialOperation.IsWithin + "(-10 -20 10 20)"; + SpatialArgs out = parser.parse(arg, ctx); + assertEquals(SpatialOperation.IsWithin, out.getOperation()); + Rectangle bounds = (Rectangle) out.getShape(); + assertEquals(-10.0, bounds.getMinX(), 0D); + assertEquals(10.0, bounds.getMaxX(), 0D); + + // Disjoint should not be scored + arg = SpatialOperation.IsDisjointTo + " (-10 10 -20 20)"; + out = parser.parse(arg, ctx); + assertEquals(SpatialOperation.IsDisjointTo, out.getOperation()); + + try { + parser.parse(SpatialOperation.IsDisjointTo + "[ ]", ctx); + fail("spatial operations need args"); + } + catch (Exception ex) {//expected + } + + try { + parser.parse("XXXX(-10 10 -20 20)", ctx); + fail("unknown operation!"); + } + catch (Exception ex) {//expected + } + } + +} diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/vector/BaseTwoDoublesStrategyTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/vector/BaseTwoDoublesStrategyTestCase.java deleted file mode 100644 index 9c15d2e..0000000 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/vector/BaseTwoDoublesStrategyTestCase.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.lucene.spatial.vector; - -import com.spatial4j.core.context.SpatialContext; -import org.apache.lucene.search.FieldCache; -import org.apache.lucene.spatial.SpatialMatchConcern; -import org.apache.lucene.spatial.StrategyTestCase; -import org.apache.lucene.spatial.util.NumericFieldInfo; -import org.junit.Test; - -import java.io.IOException; - - -public abstract class BaseTwoDoublesStrategyTestCase extends StrategyTestCase { - - protected abstract SpatialContext getSpatialContext(); - - @Override - public void setUp() throws Exception { - super.setUp(); - this.ctx = getSpatialContext(); - this.strategy = new TwoDoublesStrategy(ctx, - new NumericFieldInfo(), FieldCache.NUMERIC_UTILS_DOUBLE_PARSER); - this.fieldInfo = new TwoDoublesFieldInfo(getClass().getSimpleName()); - } - - @Test - public void testCitiesWithinBBox() throws IOException { - getAddAndVerifyIndexedDocuments(DATA_WORLD_CITIES_POINTS); - executeQueries(SpatialMatchConcern.FILTER, QTEST_Cities_IsWithin_BBox); - } -} diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/vector/TestTwoDoublesStrategy.java b/lucene/spatial/src/test/org/apache/lucene/spatial/vector/TestTwoDoublesStrategy.java index a68ff5f..53c32db 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/vector/TestTwoDoublesStrategy.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/vector/TestTwoDoublesStrategy.java @@ -1,3 +1,5 @@ +package org.apache.lucene.spatial.vector; + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -15,15 +17,51 @@ * limitations under the License. */ -package org.apache.lucene.spatial.vector; - -import com.spatial4j.core.context.SpatialContext; import com.spatial4j.core.context.simple.SimpleSpatialContext; +import com.spatial4j.core.exception.InvalidShapeException; +import com.spatial4j.core.shape.Circle; +import com.spatial4j.core.shape.Point; +import com.spatial4j.core.shape.simple.CircleImpl; +import com.spatial4j.core.shape.simple.PointImpl; +import org.apache.lucene.search.Query; +import org.apache.lucene.spatial.SpatialMatchConcern; +import org.apache.lucene.spatial.StrategyTestCase; +import org.apache.lucene.spatial.query.SpatialArgs; +import org.apache.lucene.spatial.query.SpatialOperation; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; -public class TestTwoDoublesStrategy extends BaseTwoDoublesStrategyTestCase { +public class TestTwoDoublesStrategy extends StrategyTestCase { + @Before @Override - protected SpatialContext getSpatialContext() { - return SimpleSpatialContext.GEO_KM; + public void setUp() throws Exception { + super.setUp(); + this.ctx = SimpleSpatialContext.GEO_KM; + this.strategy = new TwoDoublesStrategy(ctx, getClass().getSimpleName()); + } + + @Test + public void testCircleShapeSupport() { + Circle circle = new CircleImpl(new PointImpl(0, 0), 10, this.ctx); + SpatialArgs args = new SpatialArgs(SpatialOperation.Intersects, circle); + Query query = this.strategy.makeQuery(args); + + assertNotNull(query); + } + + @Test(expected = InvalidShapeException.class) + public void testInvalidQueryShape() { + Point point = new PointImpl(0, 0); + SpatialArgs args = new SpatialArgs(SpatialOperation.Intersects, point); + this.strategy.makeQuery(args); + } + + @Test + public void testCitiesWithinBBox() throws IOException { + getAddAndVerifyIndexedDocuments(DATA_WORLD_CITIES_POINTS); + executeQueries(SpatialMatchConcern.FILTER, QTEST_Cities_IsWithin_BBox); } } diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/CombineSuggestion.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/CombineSuggestion.java new file mode 100644 index 0000000..7e7b502 --- /dev/null +++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/CombineSuggestion.java @@ -0,0 +1,37 @@ +package org.apache.lucene.search.spell; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + *

      A suggestion generated by combining one or more original query terms

      + */ +public class CombineSuggestion { + /** + *

      The indexes from the passed-in array of terms used to make this word combination

      + */ + public final int[] originalTermIndexes; + /** + *

      The word combination suggestion

      + */ + public final SuggestWord suggestion; + + public CombineSuggestion (SuggestWord suggestion, int[] originalTermIndexes) { + this.suggestion = suggestion; + this.originalTermIndexes = originalTermIndexes; + } +} diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/Dictionary.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/Dictionary.java index 4602e24..d1ed4e7 100755 --- a/lucene/suggest/src/java/org/apache/lucene/search/spell/Dictionary.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/Dictionary.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.spell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java index 368908d..a360419 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java index 3f4833e..5a95f78 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/JaroWinklerDistance.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/JaroWinklerDistance.java index 9bcde79..f47cd14 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/spell/JaroWinklerDistance.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/JaroWinklerDistance.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -114,4 +114,25 @@ public class JaroWinklerDistance implements StringDistance { public float getThreshold() { return threshold; } + + @Override + public int hashCode() { + return 113 * Float.floatToIntBits(threshold) * getClass().hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (null == obj || getClass() != obj.getClass()) return false; + + JaroWinklerDistance o = (JaroWinklerDistance)obj; + return (Float.floatToIntBits(o.threshold) + == Float.floatToIntBits(this.threshold)); + } + + @Override + public String toString() { + return "jarowinkler(" + threshold + ")"; + } + } diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/LevensteinDistance.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/LevensteinDistance.java index c599925..f26fd7d 100755 --- a/lucene/suggest/src/java/org/apache/lucene/search/spell/LevensteinDistance.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/LevensteinDistance.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -106,4 +106,20 @@ public final class LevensteinDistance implements StringDistance { return 1.0f - ((float) p[n] / Math.max(other.length(), sa.length)); } + @Override + public int hashCode() { + return 163 * getClass().hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (null == obj) return false; + return (getClass() == obj.getClass()); + } + + @Override + public String toString() { + return "levenstein"; + } } diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/LuceneDictionary.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/LuceneDictionary.java index 4bd7f30..9bfbda7 100755 --- a/lucene/suggest/src/java/org/apache/lucene/search/spell/LuceneDictionary.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/LuceneDictionary.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/LuceneLevenshteinDistance.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/LuceneLevenshteinDistance.java index 151ae7c..d2326eb 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/spell/LuceneLevenshteinDistance.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/LuceneLevenshteinDistance.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/NGramDistance.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/NGramDistance.java index b6d8c5e..7e6d0b5 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/spell/NGramDistance.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/NGramDistance.java @@ -141,4 +141,22 @@ public class NGramDistance implements StringDistance { return 1.0f - (p[sl] / Math.max(tl, sl)); } + @Override + public int hashCode() { + return 1427 * n * getClass().hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (null == obj || getClass() != obj.getClass()) return false; + + NGramDistance o = (NGramDistance)obj; + return o.n == this.n; + } + + @Override + public String toString() { + return "ngram(" + n + ")"; + } } diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java index 2be996f..d47ebc7 100755 --- a/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java index 858804d..52139b1 100755 --- a/lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,7 +20,6 @@ package org.apache.lucene.search.spell; import java.io.IOException; import java.util.ArrayList; import java.util.Comparator; -import java.util.Iterator; import java.util.List; import org.apache.lucene.document.Document; @@ -29,10 +28,12 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.document.StringField; import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.Terms; @@ -47,7 +48,6 @@ import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; -import org.apache.lucene.util.ReaderUtil; import org.apache.lucene.util.Version; /** @@ -498,14 +498,11 @@ public class SpellChecker implements java.io.Closeable { final IndexReader reader = searcher.getIndexReader(); if (reader.maxDoc() > 0) { - new ReaderUtil.Gather(reader) { - @Override - protected void add(int base, AtomicReader r) throws IOException { - Terms terms = r.terms(F_WORD); - if (terms != null) - termsEnums.add(terms.iterator(null)); - } - }.run(); + for (final AtomicReaderContext ctx : reader.getTopReaderContext().leaves()) { + Terms terms = ctx.reader().terms(F_WORD); + if (terms != null) + termsEnums.add(terms.iterator(null)); + } } boolean isEmpty = termsEnums.isEmpty(); @@ -575,7 +572,7 @@ public class SpellChecker implements java.io.Closeable { Document doc = new Document(); // the word field is never queried on... its indexed so it can be quickly // checked for rebuild (and stored for retrieval). Doesn't need norms or TF/pos - Field f = new Field(F_WORD, text, StringField.TYPE_STORED); + Field f = new StringField(F_WORD, text, Field.Store.YES); doc.add(f); // orig term addGram(text, doc, ng1, ng2); return doc; @@ -588,7 +585,7 @@ public class SpellChecker implements java.io.Closeable { String end = null; for (int i = 0; i < len - ng + 1; i++) { String gram = text.substring(i, i + ng); - FieldType ft = new FieldType(StringField.TYPE_UNSTORED); + FieldType ft = new FieldType(StringField.TYPE_NOT_STORED); ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS); Field ngramField = new Field(key, gram, ft); // spellchecker does not use positional queries, but we want freqs @@ -596,14 +593,14 @@ public class SpellChecker implements java.io.Closeable { doc.add(ngramField); if (i == 0) { // only one term possible in the startXXField, TF/pos and norms aren't needed. - Field startField = new StringField("start" + ng, gram); + Field startField = new StringField("start" + ng, gram, Field.Store.NO); doc.add(startField); } end = gram; } if (end != null) { // may not be present if len==ng1 // only one term possible in the endXXField, TF/pos and norms aren't needed. - Field endField = new StringField("end" + ng, end); + Field endField = new StringField("end" + ng, end, Field.Store.NO); doc.add(endField); } } @@ -674,7 +671,7 @@ public class SpellChecker implements java.io.Closeable { */ // for testing purposes IndexSearcher createSearcher(final Directory dir) throws IOException{ - return new IndexSearcher(IndexReader.open(dir)); + return new IndexSearcher(DirectoryReader.open(dir)); } /** diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/StringDistance.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/StringDistance.java index 14899fe..2731ba0 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/spell/StringDistance.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/StringDistance.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestMode.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestMode.java index 252d61a..7f89906 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestMode.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestMode.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWord.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWord.java index 3e436d7..da17f8f 100755 --- a/lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWord.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWord.java @@ -1,7 +1,7 @@ package org.apache.lucene.search.spell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordFrequencyComparator.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordFrequencyComparator.java index c425b82..1abb5c6 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordFrequencyComparator.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordFrequencyComparator.java @@ -1,7 +1,7 @@ package org.apache.lucene.search.spell; import java.util.Comparator; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordQueue.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordQueue.java index 7d8de51..b908f1d 100755 --- a/lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordQueue.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordQueue.java @@ -1,7 +1,7 @@ package org.apache.lucene.search.spell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordScoreComparator.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordScoreComparator.java index 19038e0..6cecf2b 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordScoreComparator.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordScoreComparator.java @@ -1,5 +1,5 @@ package org.apache.lucene.search.spell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqIterator.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqIterator.java index ebc19c0..19027ea 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqIterator.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqIterator.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/WordBreakSpellChecker.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/WordBreakSpellChecker.java new file mode 100644 index 0000000..40f4d63 --- /dev/null +++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/WordBreakSpellChecker.java @@ -0,0 +1,502 @@ +package org.apache.lucene.search.spell; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Comparator; +import java.util.PriorityQueue; +import java.util.Queue; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.spell.SuggestMode; +import org.apache.lucene.util.BytesRef; + +/** + *

      + * A spell checker whose sole function is to offer suggestions by combining + * multiple terms into one word and/or breaking terms into multiple words. + *

      + */ +public class WordBreakSpellChecker { + private int minSuggestionFrequency = 1; + private int minBreakWordLength = 1; + private int maxCombineWordLength = 20; + private int maxChanges = 1; + private int maxEvaluations = 1000; + + public static final Term SEPARATOR_TERM = new Term("", ""); + + /** + *

      + * Determines the order to list word break suggestions + *

      + */ + public enum BreakSuggestionSortMethod { + /** + *

      + * Sort by Number of word breaks, then by the Sum of all the component + * term's frequencies + *

      + */ + NUM_CHANGES_THEN_SUMMED_FREQUENCY, + /** + *

      + * Sort by Number of word breaks, then by the Maximum of all the component + * term's frequencies + *

      + */ + NUM_CHANGES_THEN_MAX_FREQUENCY + } + + /** + *

      + * Generate suggestions by breaking the passed-in term into multiple words. + * The scores returned are equal to the number of word breaks needed so a + * lower score is generally preferred over a higher score. + *

      + * + * @param term + * @param maxSuggestions + * @param ir + * @param suggestMode + * - default = {@link SuggestMode#SUGGEST_WHEN_NOT_IN_INDEX} + * @param sortMethod + * - default = + * {@link BreakSuggestionSortMethod#NUM_CHANGES_THEN_MAX_FREQUENCY} + * @return one or more arrays of words formed by breaking up the original term + * @throws IOException + */ + public SuggestWord[][] suggestWordBreaks(Term term, int maxSuggestions, + IndexReader ir, SuggestMode suggestMode, + BreakSuggestionSortMethod sortMethod) throws IOException { + if (maxSuggestions < 1) { + return new SuggestWord[0][0]; + } + if (suggestMode == null) { + suggestMode = SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX; + } + if (sortMethod == null) { + sortMethod = BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY; + } + + int queueInitialCapacity = maxSuggestions > 10 ? 10 : maxSuggestions; + Comparator queueComparator = sortMethod == BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY ? new LengthThenMaxFreqComparator() + : new LengthThenSumFreqComparator(); + Queue suggestions = new PriorityQueue( + queueInitialCapacity, queueComparator); + + int origFreq = ir.docFreq(term); + if (origFreq > 0 && suggestMode == SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX) { + return new SuggestWord[0][]; + } + + int useMinSuggestionFrequency = minSuggestionFrequency; + if (suggestMode == SuggestMode.SUGGEST_MORE_POPULAR) { + useMinSuggestionFrequency = (origFreq == 0 ? 1 : origFreq); + } + + generateBreakUpSuggestions(term, ir, 1, maxSuggestions, + useMinSuggestionFrequency, new SuggestWord[0], suggestions, 0, + sortMethod); + + SuggestWord[][] suggestionArray = new SuggestWord[suggestions.size()][]; + for (int i = suggestions.size() - 1; i >= 0; i--) { + suggestionArray[i] = suggestions.remove().suggestWords; + } + + return suggestionArray; + } + + /** + *

      + * Generate suggestions by combining one or more of the passed-in terms into + * single words. The returned {@link CombineSuggestion} contains both a + * {@link SuggestWord} and also an array detailing which passed-in terms were + * involved in creating this combination. The scores returned are equal to the + * number of word combinations needed, also one less than the length of the + * array {@link CombineSuggestion#originalTermIndexes}. Generally, a + * suggestion with a lower score is preferred over a higher score. + *

      + *

      + * To prevent two adjacent terms from being combined (for instance, if one is + * mandatory and the other is prohibited), separate the two terms with + * {@link WordBreakSpellChecker#SEPARATOR_TERM} + *

      + *

      + * When suggestMode equals {@link SuggestMode#SUGGEST_WHEN_NOT_IN_INDEX}, each + * suggestion will include at least one term not in the index. + *

      + *

      + * When suggestMode equals {@link SuggestMode#SUGGEST_MORE_POPULAR}, each + * suggestion will have the same, or better frequency than the most-popular + * included term. + *

      + * + * @param terms + * @param maxSuggestions + * @param ir + * @param suggestMode + * @return an array of words generated by combining original terms + * @throws IOException + */ + public CombineSuggestion[] suggestWordCombinations(Term[] terms, + int maxSuggestions, IndexReader ir, SuggestMode suggestMode) + throws IOException { + if (maxSuggestions < 1) { + return new CombineSuggestion[0]; + } + + int[] origFreqs = null; + if (suggestMode != SuggestMode.SUGGEST_ALWAYS) { + origFreqs = new int[terms.length]; + for (int i = 0; i < terms.length; i++) { + origFreqs[i] = ir.docFreq(terms[i]); + } + } + + int queueInitialCapacity = maxSuggestions > 10 ? 10 : maxSuggestions; + Comparator queueComparator = new CombinationsThenFreqComparator(); + Queue suggestions = new PriorityQueue( + queueInitialCapacity, queueComparator); + + int thisTimeEvaluations = 0; + BytesRef reuse = new BytesRef(); + for (int i = 0; i < terms.length - 1; i++) { + if (terms[i].equals(SEPARATOR_TERM)) { + continue; + } + + int byteLength = terms[i].bytes().length; + if (byteLength > maxCombineWordLength) { + continue; + } + + reuse.grow(byteLength); + reuse.length = byteLength; + System.arraycopy(terms[i].bytes().bytes, terms[i].bytes().offset, + reuse.bytes, 0, byteLength); + + int maxFreq = 0; + int minFreq = Integer.MAX_VALUE; + if (origFreqs != null) { + maxFreq = origFreqs[i]; + minFreq = origFreqs[i]; + } + + for (int j = i + 1; j < terms.length && j - i <= maxChanges; j++) { + if (terms[j].equals(SEPARATOR_TERM)) { + break; + } + byteLength += terms[j].bytes().length; + if (byteLength > maxCombineWordLength) { + break; + } + + if (origFreqs != null) { + maxFreq = Math.max(maxFreq, origFreqs[j]); + minFreq = Math.min(minFreq, origFreqs[j]); + } + + reuse.grow(byteLength); + System.arraycopy(terms[j].bytes().bytes, terms[j].bytes().offset, + reuse.bytes, reuse.length, terms[j].bytes().length); + reuse.length = byteLength; + + Term combinedTerm = new Term(terms[0].field(), reuse); + int combinedTermFreq = ir.docFreq(combinedTerm); + + if (suggestMode != SuggestMode.SUGGEST_MORE_POPULAR + || combinedTermFreq >= maxFreq) { + if (suggestMode != SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX + || minFreq == 0) { + if (combinedTermFreq >= minSuggestionFrequency) { + int[] origIndexes = new int[j - i + 1]; + origIndexes[0] = i; + for (int k = 1; k < origIndexes.length; k++) { + origIndexes[k] = i + k; + } + SuggestWord word = new SuggestWord(); + word.freq = combinedTermFreq; + word.score = origIndexes.length - 1; + word.string = combinedTerm.text(); + CombineSuggestionWrapper suggestion = new CombineSuggestionWrapper( + new CombineSuggestion(word, origIndexes), + (origIndexes.length - 1)); + suggestions.offer(suggestion); + if (suggestions.size() > maxSuggestions) { + suggestions.poll(); + } + } + } + } + thisTimeEvaluations++; + if (thisTimeEvaluations == maxEvaluations) { + break; + } + } + } + CombineSuggestion[] combineSuggestions = new CombineSuggestion[suggestions + .size()]; + for (int i = suggestions.size() - 1; i >= 0; i--) { + combineSuggestions[i] = suggestions.remove().combineSuggestion; + } + return combineSuggestions; + } + + private int generateBreakUpSuggestions(Term term, IndexReader ir, + int numberBreaks, int maxSuggestions, int useMinSuggestionFrequency, + SuggestWord[] prefix, Queue suggestions, + int totalEvaluations, BreakSuggestionSortMethod sortMethod) + throws IOException { + int termLength = term.bytes().length; + int useMinBreakWordLength = minBreakWordLength; + if (useMinBreakWordLength < 1) { + useMinBreakWordLength = 1; + } + if (termLength <= (useMinBreakWordLength * 2)) { + return 0; + } + + int thisTimeEvaluations = 0; + BytesRef termBytes = term.bytes().clone(); + for (int i = useMinBreakWordLength; i < (termLength - useMinBreakWordLength); i++) { + SuggestWord leftWord = generateSuggestWord(ir, termBytes, 0, i, term + .field()); + + if (leftWord.freq >= useMinSuggestionFrequency) { + SuggestWord rightWord = generateSuggestWord(ir, termBytes, i, + termLength - i, term.field()); + if (rightWord.freq >= useMinSuggestionFrequency) { + SuggestWordArrayWrapper suggestion = new SuggestWordArrayWrapper( + newSuggestion(prefix, leftWord, rightWord)); + suggestions.offer(suggestion); + if (suggestions.size() > maxSuggestions) { + suggestions.poll(); + } + } + + int newNumberBreaks = numberBreaks + 1; + if (newNumberBreaks <= maxChanges) { + int evaluations = generateBreakUpSuggestions(new Term(term.field(), + rightWord.string), ir, newNumberBreaks, maxSuggestions, + useMinSuggestionFrequency, newPrefix(prefix, leftWord), + suggestions, totalEvaluations, sortMethod); + totalEvaluations += evaluations; + } + } + thisTimeEvaluations++; + totalEvaluations++; + if (totalEvaluations >= maxEvaluations) { + break; + } + } + return thisTimeEvaluations; + } + + private SuggestWord[] newPrefix(SuggestWord[] oldPrefix, SuggestWord append) { + SuggestWord[] newPrefix = new SuggestWord[oldPrefix.length + 1]; + System.arraycopy(oldPrefix, 0, newPrefix, 0, oldPrefix.length); + newPrefix[newPrefix.length - 1] = append; + return newPrefix; + } + + private SuggestWord[] newSuggestion(SuggestWord[] prefix, + SuggestWord append1, SuggestWord append2) { + SuggestWord[] newSuggestion = new SuggestWord[prefix.length + 2]; + int score = prefix.length + 1; + for (int i = 0; i < prefix.length; i++) { + SuggestWord word = new SuggestWord(); + word.string = prefix[i].string; + word.freq = prefix[i].freq; + word.score = score; + newSuggestion[i] = word; + } + append1.score = score; + append2.score = score; + newSuggestion[newSuggestion.length - 2] = append1; + newSuggestion[newSuggestion.length - 1] = append2; + return newSuggestion; + } + + private SuggestWord generateSuggestWord(IndexReader ir, BytesRef bytes, + int offset, int length, String fieldname) throws IOException { + bytes.offset = offset; + bytes.length = length; + Term term = new Term(fieldname, bytes); + int freq = ir.docFreq(term); + SuggestWord word = new SuggestWord(); + word.freq = freq; + word.score = 1; + word.string = term.text(); + return word; + } + + public int getMinSuggestionFrequency() { + return minSuggestionFrequency; + } + + public int getMaxCombineWordLength() { + return maxCombineWordLength; + } + + public int getMinBreakWordLength() { + return minBreakWordLength; + } + + public int getMaxChanges() { + return maxChanges; + } + + public int getMaxEvaluations() { + return maxEvaluations; + } + + /** + *

      + * The minimum frequency a term must have to be included as part of a + * suggestion. Default=1 Not applicable when used with + * {@link SuggestMode#SUGGEST_MORE_POPULAR} + *

      + * + * @param minSuggestionFrequency + */ + public void setMinSuggestionFrequency(int minSuggestionFrequency) { + this.minSuggestionFrequency = minSuggestionFrequency; + } + + /** + *

      + * The maximum length of a suggestion made by combining 1 or more original + * terms. Default=20 + *

      + * + * @param maxCombineWordLength + */ + public void setMaxCombineWordLength(int maxCombineWordLength) { + this.maxCombineWordLength = maxCombineWordLength; + } + + /** + *

      + * The minimum length to break words down to. Default=1 + *

      + * + * @param minBreakWordLength + */ + public void setMinBreakWordLength(int minBreakWordLength) { + this.minBreakWordLength = minBreakWordLength; + } + + /** + *

      + * The maximum numbers of changes (word breaks or combinations) to make on the + * original term(s). Default=1 + *

      + * + * @param maxChanges + */ + public void setMaxChanges(int maxChanges) { + this.maxChanges = maxChanges; + } + + /** + *

      + * The maximum number of word combinations to evaluate. Default=1000. A higher + * value might improve result quality. A lower value might improve + * performance. + *

      + * + * @param maxEvaluations + */ + public void setMaxEvaluations(int maxEvaluations) { + this.maxEvaluations = maxEvaluations; + } + + private class LengthThenMaxFreqComparator implements + Comparator { + @Override + public int compare(SuggestWordArrayWrapper o1, SuggestWordArrayWrapper o2) { + if (o1.suggestWords.length != o2.suggestWords.length) { + return o2.suggestWords.length - o1.suggestWords.length; + } + if (o1.freqMax != o2.freqMax) { + return o1.freqMax - o2.freqMax; + } + return 0; + } + } + + private class LengthThenSumFreqComparator implements + Comparator { + @Override + public int compare(SuggestWordArrayWrapper o1, SuggestWordArrayWrapper o2) { + if (o1.suggestWords.length != o2.suggestWords.length) { + return o2.suggestWords.length - o1.suggestWords.length; + } + if (o1.freqSum != o2.freqSum) { + return o1.freqSum - o2.freqSum; + } + return 0; + } + } + + private class CombinationsThenFreqComparator implements + Comparator { + @Override + public int compare(CombineSuggestionWrapper o1, CombineSuggestionWrapper o2) { + if (o1.numCombinations != o2.numCombinations) { + return o2.numCombinations - o1.numCombinations; + } + if (o1.combineSuggestion.suggestion.freq != o2.combineSuggestion.suggestion.freq) { + return o1.combineSuggestion.suggestion.freq + - o2.combineSuggestion.suggestion.freq; + } + return 0; + } + } + + private class SuggestWordArrayWrapper { + final SuggestWord[] suggestWords; + final int freqMax; + final int freqSum; + + SuggestWordArrayWrapper(SuggestWord[] suggestWords) { + this.suggestWords = suggestWords; + int aFreqSum = 0; + int aFreqMax = 0; + for (SuggestWord sw : suggestWords) { + aFreqSum += sw.freq; + aFreqMax = Math.max(aFreqMax, sw.freq); + } + this.freqSum = aFreqSum; + this.freqMax = aFreqMax; + } + } + + private class CombineSuggestionWrapper { + final CombineSuggestion combineSuggestion; + final int numCombinations; + + CombineSuggestionWrapper(CombineSuggestion combineSuggestion, + int numCombinations) { + this.combineSuggestion = combineSuggestion; + this.numCombinations = numCombinations; + } + } +} diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqIteratorWrapper.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqIteratorWrapper.java index dd6a86b..723468f 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqIteratorWrapper.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqIteratorWrapper.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/BytesRefList.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/BytesRefList.java index 9fea152..ff8c400 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/BytesRefList.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/BytesRefList.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF @@ -17,7 +17,6 @@ package org.apache.lucene.search.suggest; * the License. */ -import java.io.IOException; import java.util.Arrays; import java.util.Comparator; @@ -190,7 +189,7 @@ public final class BytesRefList { int pos = 0; @Override - public BytesRef next() throws IOException { + public BytesRef next() { if (pos < size) { return get(spare, ords == null ? pos++ : ords[pos++]); } diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java index 49f9f76..96b7f87 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java index 5783eea..5ff7ce3 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java index 0206181..40b1c27 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedTermFreqIteratorWrapper.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedTermFreqIteratorWrapper.java index a97b170..b52921f 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedTermFreqIteratorWrapper.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedTermFreqIteratorWrapper.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/BytesRefSorter.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/BytesRefSorter.java index 3d14102..48c5f6a 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/BytesRefSorter.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/BytesRefSorter.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java index 77995c1..e23508b 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletion.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletion.java index 9e49b1e..be73bc4 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletion.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletion.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -135,11 +135,12 @@ public class FSTCompletion { try { List> rootArcs = new ArrayList>(); Arc arc = automaton.getFirstArc(new Arc()); - automaton.readFirstTargetArc(arc, arc); + FST.BytesReader fstReader = automaton.getBytesReader(0); + automaton.readFirstTargetArc(arc, arc, fstReader); while (true) { rootArcs.add(new Arc().copyFrom(arc)); if (arc.isLast()) break; - automaton.readNextArc(arc); + automaton.readNextArc(arc, fstReader); } Collections.reverse(rootArcs); // we want highest weights first. @@ -168,13 +169,14 @@ public class FSTCompletion { // Get the UTF-8 bytes representation of the input key. try { final FST.Arc scratch = new FST.Arc(); + FST.BytesReader fstReader = automaton.getBytesReader(0); for (; rootArcIndex < rootArcs.length; rootArcIndex++) { final FST.Arc rootArc = rootArcs[rootArcIndex]; final FST.Arc arc = scratch.copyFrom(rootArc); // Descend into the automaton using the key as prefix. if (descendWithPrefix(arc, utf8)) { - automaton.readFirstTargetArc(arc, arc); + automaton.readFirstTargetArc(arc, arc, fstReader); if (arc.label == FST.END_LABEL) { // Normalize prefix-encoded weight. return rootArc.label; @@ -356,8 +358,8 @@ public class FSTCompletion { } assert output.offset == 0; output.bytes[output.length++] = (byte) arc.label; - - automaton.readFirstTargetArc(arc, arc); + FST.BytesReader fstReader = automaton.getBytesReader(0); + automaton.readFirstTargetArc(arc, arc, fstReader); while (true) { if (arc.label == FST.END_LABEL) { res.add(new Completion(output, bucket)); @@ -373,7 +375,7 @@ public class FSTCompletion { if (arc.isLast()) { break; } - automaton.readNextArc(arc); + automaton.readNextArc(arc, fstReader); } return false; } diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionBuilder.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionBuilder.java index ba4c5c7..179d44b 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionBuilder.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionBuilder.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java index dd0b58a..84bdacb 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/InMemorySorter.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/InMemorySorter.java index ce6a17d..a87bf22 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/InMemorySorter.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/InMemorySorter.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java index a373f92..9c16401 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java index 3baf9a1..fe0a030 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest.jaspell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTAutocomplete.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTAutocomplete.java index 48b570c..3f4b056 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTAutocomplete.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTAutocomplete.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest.tst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java index 130fc71..6d15032 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest.tst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TernaryTreeNode.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TernaryTreeNode.java index 39cd4cc..0ebe010 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TernaryTreeNode.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TernaryTreeNode.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest.tst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestDirectSpellChecker.java b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestDirectSpellChecker.java index a0c5a26..647e298 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestDirectSpellChecker.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestDirectSpellChecker.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,7 +20,7 @@ package org.apache.lucene.search.spell; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; @@ -39,7 +39,7 @@ public class TestDirectSpellChecker extends LuceneTestCase { String[] termsToAdd = { "metanoia", "metanoian", "metanoiai", "metanoias", "metanoið‘" }; for (int i = 0; i < termsToAdd.length; i++) { Document doc = new Document(); - doc.add(newField("repentance", termsToAdd[i], TextField.TYPE_UNSTORED)); + doc.add(newTextField("repentance", termsToAdd[i], Field.Store.NO)); writer.addDocument(doc); } @@ -68,7 +68,7 @@ public class TestDirectSpellChecker extends LuceneTestCase { for (int i = 0; i < 20; i++) { Document doc = new Document(); - doc.add(newField("numbers", English.intToEnglish(i), TextField.TYPE_UNSTORED)); + doc.add(newTextField("numbers", English.intToEnglish(i), Field.Store.NO)); writer.addDocument(doc); } @@ -108,7 +108,7 @@ public class TestDirectSpellChecker extends LuceneTestCase { // add some more documents for (int i = 1000; i < 1100; i++) { Document doc = new Document(); - doc.add(newField("numbers", English.intToEnglish(i), TextField.TYPE_UNSTORED)); + doc.add(newTextField("numbers", English.intToEnglish(i), Field.Store.NO)); writer.addDocument(doc); } @@ -132,13 +132,13 @@ public class TestDirectSpellChecker extends LuceneTestCase { new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)); Document doc = new Document(); - doc.add(newField("text", "foobar", TextField.TYPE_UNSTORED)); + doc.add(newTextField("text", "foobar", Field.Store.NO)); writer.addDocument(doc); - doc.add(newField("text", "foobar", TextField.TYPE_UNSTORED)); + doc.add(newTextField("text", "foobar", Field.Store.NO)); writer.addDocument(doc); - doc.add(newField("text", "foobaz", TextField.TYPE_UNSTORED)); + doc.add(newTextField("text", "foobaz", Field.Store.NO)); writer.addDocument(doc); - doc.add(newField("text", "fobar", TextField.TYPE_UNSTORED)); + doc.add(newTextField("text", "fobar", Field.Store.NO)); writer.addDocument(doc); IndexReader ir = writer.getReader(); @@ -200,7 +200,7 @@ public class TestDirectSpellChecker extends LuceneTestCase { for (int i = 0; i < 20; i++) { Document doc = new Document(); - doc.add(newField("numbers", English.intToEnglish(i), TextField.TYPE_UNSTORED)); + doc.add(newTextField("numbers", English.intToEnglish(i), Field.Store.NO)); writer.addDocument(doc); } @@ -224,7 +224,7 @@ public class TestDirectSpellChecker extends LuceneTestCase { for (int i = 0; i < 20; i++) { Document doc = new Document(); - doc.add(newField("numbers", English.intToEnglish(i), TextField.TYPE_UNSTORED)); + doc.add(newTextField("numbers", English.intToEnglish(i), Field.Store.NO)); writer.addDocument(doc); } @@ -249,7 +249,7 @@ public class TestDirectSpellChecker extends LuceneTestCase { for (int i = 0; i < 20; i++) { Document doc = new Document(); - doc.add(newField("numbers", English.intToEnglish(i), TextField.TYPE_UNSTORED)); + doc.add(newTextField("numbers", English.intToEnglish(i), Field.Store.NO)); writer.addDocument(doc); } diff --git a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestJaroWinklerDistance.java b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestJaroWinklerDistance.java index 01aa3d1..b980d4d 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestJaroWinklerDistance.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestJaroWinklerDistance.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestLevenshteinDistance.java b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestLevenshteinDistance.java index 1a54d37..9343b1f 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestLevenshteinDistance.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestLevenshteinDistance.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java index 40ac84a..35ed90c 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,7 +22,7 @@ import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; +import org.apache.lucene.document.Field; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; @@ -54,23 +54,23 @@ public class TestLuceneDictionary extends LuceneTestCase { Document doc; doc = new Document(); - doc.add(newField("aaa", "foo", TextField.TYPE_STORED)); + doc.add(newTextField("aaa", "foo", Field.Store.YES)); writer.addDocument(doc); doc = new Document(); - doc.add(newField("aaa", "foo", TextField.TYPE_STORED)); + doc.add(newTextField("aaa", "foo", Field.Store.YES)); writer.addDocument(doc); doc = new Document(); - doc.add(newField("contents", "Tom", TextField.TYPE_STORED)); + doc.add(newTextField("contents", "Tom", Field.Store.YES)); writer.addDocument(doc); doc = new Document(); - doc.add(newField("contents", "Jerry", TextField.TYPE_STORED)); + doc.add(newTextField("contents", "Jerry", Field.Store.YES)); writer.addDocument(doc); doc = new Document(); - doc.add(newField("zzz", "bar", TextField.TYPE_STORED)); + doc.add(newTextField("zzz", "bar", Field.Store.YES)); writer.addDocument(doc); writer.forceMerge(1); diff --git a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestNGramDistance.java b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestNGramDistance.java index 9ca3b86..e70ca9f 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestNGramDistance.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestNGramDistance.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestPlainTextDictionary.java b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestPlainTextDictionary.java index ae33c6e..5c735aa 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestPlainTextDictionary.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestPlainTextDictionary.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java index 9dd9e01..579a5f4 100755 --- a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.spell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -28,8 +28,8 @@ import java.util.concurrent.TimeUnit; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -58,14 +58,14 @@ public class TestSpellChecker extends LuceneTestCase { for (int i = 0; i < 1000; i++) { Document doc = new Document(); - doc.add(newField("field1", English.intToEnglish(i), TextField.TYPE_STORED)); - doc.add(newField("field2", English.intToEnglish(i + 1), TextField.TYPE_STORED)); // + word thousand - doc.add(newField("field3", "fvei" + (i % 2 == 0 ? " five" : ""), TextField.TYPE_STORED)); // + word thousand + doc.add(newTextField("field1", English.intToEnglish(i), Field.Store.YES)); + doc.add(newTextField("field2", English.intToEnglish(i + 1), Field.Store.YES)); // + word thousand + doc.add(newTextField("field3", "fvei" + (i % 2 == 0 ? " five" : ""), Field.Store.YES)); // + word thousand writer.addDocument(doc); } { Document doc = new Document(); - doc.add(newField("field1", "eight", TextField.TYPE_STORED)); // "eight" in + doc.add(newTextField("field1", "eight", Field.Store.YES)); // "eight" in // the index // twice writer.addDocument(doc); @@ -73,13 +73,12 @@ public class TestSpellChecker extends LuceneTestCase { { Document doc = new Document(); doc - .add(newField("field1", "twenty-one twenty-one", - TextField.TYPE_STORED)); // "twenty-one" in the index thrice + .add(newTextField("field1", "twenty-one twenty-one", Field.Store.YES)); // "twenty-one" in the index thrice writer.addDocument(doc); } { Document doc = new Document(); - doc.add(newField("field1", "twenty", TextField.TYPE_STORED)); // "twenty" + doc.add(newTextField("field1", "twenty", Field.Store.YES)); // "twenty" // in the // index // twice @@ -103,8 +102,8 @@ public class TestSpellChecker extends LuceneTestCase { } - public void testBuild() throws CorruptIndexException, IOException { - IndexReader r = IndexReader.open(userindex); + public void testBuild() throws IOException { + IndexReader r = DirectoryReader.open(userindex); spellChecker.clearIndex(); @@ -144,7 +143,7 @@ public class TestSpellChecker extends LuceneTestCase { } public void testComparator() throws Exception { - IndexReader r = IndexReader.open(userindex); + IndexReader r = DirectoryReader.open(userindex); Directory compIdx = newDirectory(); SpellChecker compareSP = new SpellCheckerMock(compIdx, new LevensteinDistance(), new SuggestWordFrequencyComparator()); addwords(r, compareSP, "field3"); @@ -162,7 +161,7 @@ public class TestSpellChecker extends LuceneTestCase { } public void testBogusField() throws Exception { - IndexReader r = IndexReader.open(userindex); + IndexReader r = DirectoryReader.open(userindex); Directory compIdx = newDirectory(); SpellChecker compareSP = new SpellCheckerMock(compIdx, new LevensteinDistance(), new SuggestWordFrequencyComparator()); addwords(r, compareSP, "field3"); @@ -177,7 +176,7 @@ public class TestSpellChecker extends LuceneTestCase { } public void testSuggestModes() throws Exception { - IndexReader r = IndexReader.open(userindex); + IndexReader r = DirectoryReader.open(userindex); spellChecker.clearIndex(); addwords(r, spellChecker, "field1"); @@ -337,7 +336,7 @@ public class TestSpellChecker extends LuceneTestCase { } private int numdoc() throws IOException { - IndexReader rs = IndexReader.open(spellindex); + IndexReader rs = DirectoryReader.open(spellindex); int num = rs.numDocs(); assertTrue(num != 0); //System.out.println("num docs: " + num); @@ -346,7 +345,7 @@ public class TestSpellChecker extends LuceneTestCase { } public void testClose() throws IOException { - IndexReader r = IndexReader.open(userindex); + IndexReader r = DirectoryReader.open(userindex); spellChecker.clearIndex(); String field = "field1"; addwords(r, spellChecker, "field1"); @@ -402,7 +401,7 @@ public class TestSpellChecker extends LuceneTestCase { */ public void testConcurrentAccess() throws IOException, InterruptedException { assertEquals(1, searchers.size()); - final IndexReader r = IndexReader.open(userindex); + final IndexReader r = DirectoryReader.open(userindex); spellChecker.clearIndex(); assertEquals(2, searchers.size()); addwords(r, spellChecker, "field1"); diff --git a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestWordBreakSpellChecker.java b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestWordBreakSpellChecker.java new file mode 100644 index 0000000..36e9411 --- /dev/null +++ b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestWordBreakSpellChecker.java @@ -0,0 +1,251 @@ +package org.apache.lucene.search.spell; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import junit.framework.Assert; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.MockTokenizer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.spell.WordBreakSpellChecker.BreakSuggestionSortMethod; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.English; +import org.apache.lucene.util.LuceneTestCase; + +public class TestWordBreakSpellChecker extends LuceneTestCase { + private Directory dir = null; + + @Override + public void setUp() throws Exception { + super.setUp(); + dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true)); + + for (int i = 900; i < 1112; i++) { + Document doc = new Document(); + String num = English.intToEnglish(i).replaceAll("[-]", " ").replaceAll("[,]", ""); + doc.add(newTextField("numbers", num, Field.Store.NO)); + writer.addDocument(doc); + } + + { + Document doc = new Document(); + doc.add(newTextField("numbers", "thou hast sand betwixt thy toes", Field.Store.NO)); + writer.addDocument(doc); + } + { + Document doc = new Document(); + doc.add(newTextField("numbers", "hundredeight eightyeight yeight", Field.Store.NO)); + writer.addDocument(doc); + } + { + Document doc = new Document(); + doc.add(newTextField("numbers", "tres y cinco", Field.Store.NO)); + writer.addDocument(doc); + } + + writer.commit(); + writer.close(); + } + + @Override + public void tearDown() throws Exception { + if(dir!=null) { + dir.close(); + dir = null; + } + super.tearDown(); + } + public void testCombiningWords() throws Exception { + IndexReader ir = null; + try { + ir = DirectoryReader.open(dir); + WordBreakSpellChecker wbsp = new WordBreakSpellChecker(); + + { + Term[] terms = { + new Term("numbers", "one"), + new Term("numbers", "hun"), + new Term("numbers", "dred"), + new Term("numbers", "eight"), + new Term("numbers", "y"), + new Term("numbers", "eight"), + }; + wbsp.setMaxChanges(3); + wbsp.setMaxCombineWordLength(20); + wbsp.setMinSuggestionFrequency(1); + CombineSuggestion[] cs = wbsp.suggestWordCombinations(terms, 10, ir, SuggestMode.SUGGEST_ALWAYS); + Assert.assertTrue(cs.length==5); + + Assert.assertTrue(cs[0].originalTermIndexes.length==2); + Assert.assertTrue(cs[0].originalTermIndexes[0]==1); + Assert.assertTrue(cs[0].originalTermIndexes[1]==2); + Assert.assertTrue(cs[0].suggestion.string.equals("hundred")); + Assert.assertTrue(cs[0].suggestion.score==1); + + Assert.assertTrue(cs[1].originalTermIndexes.length==2); + Assert.assertTrue(cs[1].originalTermIndexes[0]==3); + Assert.assertTrue(cs[1].originalTermIndexes[1]==4); + Assert.assertTrue(cs[1].suggestion.string.equals("eighty")); + Assert.assertTrue(cs[1].suggestion.score==1); + + Assert.assertTrue(cs[2].originalTermIndexes.length==2); + Assert.assertTrue(cs[2].originalTermIndexes[0]==4); + Assert.assertTrue(cs[2].originalTermIndexes[1]==5); + Assert.assertTrue(cs[2].suggestion.string.equals("yeight")); + Assert.assertTrue(cs[2].suggestion.score==1); + + for(int i=3 ; i<5 ; i++) { + Assert.assertTrue(cs[i].originalTermIndexes.length==3); + Assert.assertTrue(cs[i].suggestion.score==2); + Assert.assertTrue( + (cs[i].originalTermIndexes[0]==1 && + cs[i].originalTermIndexes[1]==2 && + cs[i].originalTermIndexes[2]==3 && + cs[i].suggestion.string.equals("hundredeight")) || + (cs[i].originalTermIndexes[0]==3 && + cs[i].originalTermIndexes[1]==4 && + cs[i].originalTermIndexes[2]==5 && + cs[i].suggestion.string.equals("eightyeight")) + ); + } + + cs = wbsp.suggestWordCombinations(terms, 5, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX); + Assert.assertTrue(cs.length==2); + Assert.assertTrue(cs[0].originalTermIndexes.length==2); + Assert.assertTrue(cs[0].suggestion.score==1); + Assert.assertTrue(cs[0].originalTermIndexes[0]==1); + Assert.assertTrue(cs[0].originalTermIndexes[1]==2); + Assert.assertTrue(cs[0].suggestion.string.equals("hundred")); + Assert.assertTrue(cs[0].suggestion.score==1); + + Assert.assertTrue(cs[1].originalTermIndexes.length==3); + Assert.assertTrue(cs[1].suggestion.score==2); + Assert.assertTrue(cs[1].originalTermIndexes[0] == 1); + Assert.assertTrue(cs[1].originalTermIndexes[1] == 2); + Assert.assertTrue(cs[1].originalTermIndexes[2] == 3); + Assert.assertTrue(cs[1].suggestion.string.equals("hundredeight")); + } + } catch(Exception e) { + throw e; + } finally { + try { ir.close(); } catch(Exception e1) { } + } + } + + public void testBreakingWords() throws Exception { + IndexReader ir = null; + try { + ir = DirectoryReader.open(dir); + WordBreakSpellChecker wbsp = new WordBreakSpellChecker(); + + { + Term term = new Term("numbers", "ninetynine"); + wbsp.setMaxChanges(1); + wbsp.setMinBreakWordLength(1); + wbsp.setMinSuggestionFrequency(1); + SuggestWord[][] sw = wbsp.suggestWordBreaks(term, 5, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); + Assert.assertTrue(sw.length==1); + Assert.assertTrue(sw[0].length==2); + Assert.assertTrue(sw[0][0].string.equals("ninety")); + Assert.assertTrue(sw[0][1].string.equals("nine")); + Assert.assertTrue(sw[0][0].score == 1); + Assert.assertTrue(sw[0][1].score == 1); + } + { + Term term = new Term("numbers", "onethousand"); + wbsp.setMaxChanges(1); + wbsp.setMinBreakWordLength(1); + wbsp.setMinSuggestionFrequency(1); + SuggestWord[][] sw = wbsp.suggestWordBreaks(term, 2, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); + Assert.assertTrue(sw.length==1); + Assert.assertTrue(sw[0].length==2); + Assert.assertTrue(sw[0][0].string.equals("one")); + Assert.assertTrue(sw[0][1].string.equals("thousand")); + Assert.assertTrue(sw[0][0].score == 1); + Assert.assertTrue(sw[0][1].score == 1); + + wbsp.setMaxChanges(2); + wbsp.setMinSuggestionFrequency(1); + sw = wbsp.suggestWordBreaks(term, 1, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); + Assert.assertTrue(sw.length==1); + Assert.assertTrue(sw[0].length==2); + + wbsp.setMaxChanges(2); + wbsp.setMinSuggestionFrequency(2); + sw = wbsp.suggestWordBreaks(term, 2, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); + Assert.assertTrue(sw.length==1); + Assert.assertTrue(sw[0].length==2); + + wbsp.setMaxChanges(2); + wbsp.setMinSuggestionFrequency(1); + sw = wbsp.suggestWordBreaks(term, 2, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); + Assert.assertTrue(sw.length==2); + Assert.assertTrue(sw[0].length==2); + Assert.assertTrue(sw[0][0].string.equals("one")); + Assert.assertTrue(sw[0][1].string.equals("thousand")); + Assert.assertTrue(sw[0][0].score == 1); + Assert.assertTrue(sw[0][1].score == 1); + Assert.assertTrue(sw[0][1].freq>1); + Assert.assertTrue(sw[0][0].freq>sw[0][1].freq); + Assert.assertTrue(sw[1].length==3); + Assert.assertTrue(sw[1][0].string.equals("one")); + Assert.assertTrue(sw[1][1].string.equals("thou")); + Assert.assertTrue(sw[1][2].string.equals("sand")); + Assert.assertTrue(sw[1][0].score == 2); + Assert.assertTrue(sw[1][1].score == 2); + Assert.assertTrue(sw[1][2].score == 2); + Assert.assertTrue(sw[1][0].freq>1); + Assert.assertTrue(sw[1][1].freq==1); + Assert.assertTrue(sw[1][2].freq==1); + } + { + Term term = new Term("numbers", "onethousandonehundredeleven"); + wbsp.setMaxChanges(3); + wbsp.setMinBreakWordLength(1); + wbsp.setMinSuggestionFrequency(1); + SuggestWord[][] sw = wbsp.suggestWordBreaks(term, 5, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); + Assert.assertTrue(sw.length==0); + + wbsp.setMaxChanges(4); + sw = wbsp.suggestWordBreaks(term, 5, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); + Assert.assertTrue(sw.length==1); + Assert.assertTrue(sw[0].length==5); + + wbsp.setMaxChanges(5); + sw = wbsp.suggestWordBreaks(term, 5, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY); + Assert.assertTrue(sw.length==2); + Assert.assertTrue(sw[0].length==5); + Assert.assertTrue(sw[0][1].string.equals("thousand")); + Assert.assertTrue(sw[1].length==6); + Assert.assertTrue(sw[1][1].string.equals("thou")); + Assert.assertTrue(sw[1][2].string.equals("sand")); + } + + } catch(Exception e) { + throw e; + } finally { + try { ir.close(); } catch(Exception e1) { } + } + } + } diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/Average.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/Average.java index fa3241d..c405a3d 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/Average.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/Average.java @@ -1,7 +1,7 @@ package org.apache.lucene.search.suggest; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java index b3e931f..9126730 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java index 74fb1d7..ef948a8 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreq.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreq.java index 49b346b..2b02ac1 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreq.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreq.java @@ -2,7 +2,7 @@ package org.apache.lucene.search.suggest; import org.apache.lucene.util.BytesRef; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java index 1abf941..06d7301 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,7 +17,6 @@ package org.apache.lucene.search.suggest; * limitations under the License. */ -import java.io.IOException; import java.util.Arrays; import java.util.Comparator; import java.util.Iterator; @@ -50,7 +49,7 @@ public final class TermFreqArrayIterator implements TermFreqIterator { } @Override - public BytesRef next() throws IOException { + public BytesRef next() { if (i.hasNext()) { current = i.next(); spare.copyBytes(current.term); diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestBytesRefList.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestBytesRefList.java index 2b323b0..69c382f 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestBytesRefList.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestBytesRefList.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestHighFrequencyDictionary.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestHighFrequencyDictionary.java index e598f60..b7ef1df 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestHighFrequencyDictionary.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestHighFrequencyDictionary.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqIterator.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqIterator.java index bbfb849..5c317fa 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqIterator.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqIterator.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java index c08bb4f..554f349 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java index e2c5c87..c0a853e 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/LargeInputFST.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/LargeInputFST.java index 091aa11..79f0be9 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/LargeInputFST.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/LargeInputFST.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/TestSort.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/TestSort.java index c571d48..50516b8 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/TestSort.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/TestSort.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java index 4b3fe73..ebae174 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java @@ -1,6 +1,6 @@ package org.apache.lucene.search.suggest.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/ivy.xml b/lucene/test-framework/ivy.xml index d457de7..4e39a2b 100644 --- a/lucene/test-framework/ivy.xml +++ b/lucene/test-framework/ivy.xml @@ -18,13 +18,23 @@ --> - - - - - - + + + + + + + + + + + + + diff --git a/lucene/test-framework/lib/ant-1.7.1.jar.sha1 b/lucene/test-framework/lib/ant-1.7.1.jar.sha1 deleted file mode 100644 index fb5d89a..0000000 --- a/lucene/test-framework/lib/ant-1.7.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1d33711018e7649a8427fff62a87f94f4e7d310f diff --git a/lucene/test-framework/lib/ant-1.8.2.jar.sha1 b/lucene/test-framework/lib/ant-1.8.2.jar.sha1 new file mode 100644 index 0000000..564db78 --- /dev/null +++ b/lucene/test-framework/lib/ant-1.8.2.jar.sha1 @@ -0,0 +1 @@ +fc33bf7cd8c5309dd7b81228e8626515ee42efd9 diff --git a/lucene/test-framework/lib/ant-junit-1.7.1.jar.sha1 b/lucene/test-framework/lib/ant-junit-1.7.1.jar.sha1 deleted file mode 100644 index 2579a36..0000000 --- a/lucene/test-framework/lib/ant-junit-1.7.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -942d04e42bc6a9fc61a7744625d89726af8ec006 diff --git a/lucene/test-framework/lib/ant-junit-1.8.2.jar.sha1 b/lucene/test-framework/lib/ant-junit-1.8.2.jar.sha1 new file mode 100644 index 0000000..b079f25 --- /dev/null +++ b/lucene/test-framework/lib/ant-junit-1.8.2.jar.sha1 @@ -0,0 +1 @@ +1653e85a2710d59edef1fcdd899a35f2c45324b3 diff --git a/lucene/test-framework/lib/junit4-ant-1.5.0.jar.sha1 b/lucene/test-framework/lib/junit4-ant-1.5.0.jar.sha1 deleted file mode 100644 index 05060d2..0000000 --- a/lucene/test-framework/lib/junit4-ant-1.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4e920288c2d2cd39b7e15f3abcaa3c5e2213ec9c diff --git a/lucene/test-framework/lib/junit4-ant-1.6.0.jar.sha1 b/lucene/test-framework/lib/junit4-ant-1.6.0.jar.sha1 new file mode 100644 index 0000000..9a2ca80 --- /dev/null +++ b/lucene/test-framework/lib/junit4-ant-1.6.0.jar.sha1 @@ -0,0 +1 @@ +c7a65e96a2c62ba83ca404065305aec5dc7fc8f1 diff --git a/lucene/test-framework/lib/randomizedtesting-runner-1.5.0.jar.sha1 b/lucene/test-framework/lib/randomizedtesting-runner-1.5.0.jar.sha1 deleted file mode 100644 index 380e2fe..0000000 --- a/lucene/test-framework/lib/randomizedtesting-runner-1.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -378731cc7f26d45b68a6e5f600d4c7d071d165b1 diff --git a/lucene/test-framework/lib/randomizedtesting-runner-1.6.0.jar.sha1 b/lucene/test-framework/lib/randomizedtesting-runner-1.6.0.jar.sha1 new file mode 100644 index 0000000..be7ad32 --- /dev/null +++ b/lucene/test-framework/lib/randomizedtesting-runner-1.6.0.jar.sha1 @@ -0,0 +1 @@ +709f9549a0b0c2e2ecdd5af012d9531325d6551b diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java index 80229f6..56014b1 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -25,20 +25,23 @@ import java.io.Reader; import java.io.StringReader; import java.io.StringWriter; import java.io.Writer; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; -import java.util.Map; -import java.util.HashMap; +import java.util.*; import org.apache.lucene.analysis.tokenattributes.*; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.store.Directory; import org.apache.lucene.util.Attribute; import org.apache.lucene.util.AttributeImpl; import org.apache.lucene.util.IOUtils; -import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LineFileDocs; -import org.apache.lucene.util._TestUtil; +import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.Rethrow; +import org.apache.lucene.util._TestUtil; /** * Base class for all Lucene unit tests that use TokenStreams. @@ -175,10 +178,6 @@ public abstract class BaseTokenStreamTestCase extends LuceneTestCase { if (offsetAtt != null) { final int startOffset = offsetAtt.startOffset(); final int endOffset = offsetAtt.endOffset(); - assertTrue("startOffset must be >= 0", startOffset >= 0); - assertTrue("endOffset must be >= 0", endOffset >= 0); - assertTrue("endOffset must be >= startOffset, got startOffset=" + startOffset + ",endOffset=" + endOffset, - endOffset >= startOffset); if (finalOffset != null) { assertTrue("startOffset must be <= finalOffset", startOffset <= finalOffset.intValue()); assertTrue("endOffset must be <= finalOffset: got endOffset=" + endOffset + " vs finalOffset=" + finalOffset.intValue(), @@ -388,13 +387,14 @@ public abstract class BaseTokenStreamTestCase extends LuceneTestCase { final boolean useCharFilter; final boolean simple; final boolean offsetsAreCorrect; + final RandomIndexWriter iw; // NOTE: not volatile because we don't want the tests to // add memory barriers (ie alter how threads // interact)... so this is just "best effort": public boolean failed; - AnalysisThread(long seed, Analyzer a, int iterations, int maxWordLength, boolean useCharFilter, boolean simple, boolean offsetsAreCorrect) { + AnalysisThread(long seed, Analyzer a, int iterations, int maxWordLength, boolean useCharFilter, boolean simple, boolean offsetsAreCorrect, RandomIndexWriter iw) { this.seed = seed; this.a = a; this.iterations = iterations; @@ -402,6 +402,7 @@ public abstract class BaseTokenStreamTestCase extends LuceneTestCase { this.useCharFilter = useCharFilter; this.simple = simple; this.offsetsAreCorrect = offsetsAreCorrect; + this.iw = iw; } @Override @@ -410,7 +411,7 @@ public abstract class BaseTokenStreamTestCase extends LuceneTestCase { try { // see the part in checkRandomData where it replays the same text again // to verify reproducability/reuse: hopefully this would catch thread hazards. - checkRandomData(new Random(seed), a, iterations, maxWordLength, useCharFilter, simple, offsetsAreCorrect); + checkRandomData(new Random(seed), a, iterations, maxWordLength, useCharFilter, simple, offsetsAreCorrect, iw); success = true; } catch (IOException e) { Rethrow.rethrow(e); @@ -427,34 +428,92 @@ public abstract class BaseTokenStreamTestCase extends LuceneTestCase { public static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength, boolean simple, boolean offsetsAreCorrect) throws IOException { long seed = random.nextLong(); boolean useCharFilter = random.nextBoolean(); - checkRandomData(new Random(seed), a, iterations, maxWordLength, useCharFilter, simple, offsetsAreCorrect); - // now test with multiple threads: note we do the EXACT same thing we did before in each thread, - // so this should only really fail from another thread if its an actual thread problem - int numThreads = _TestUtil.nextInt(random, 2, 4); - AnalysisThread threads[] = new AnalysisThread[numThreads]; - for (int i = 0; i < threads.length; i++) { - threads[i] = new AnalysisThread(seed, a, iterations, maxWordLength, useCharFilter, simple, offsetsAreCorrect); - } - for (int i = 0; i < threads.length; i++) { - threads[i].start(); - } - for (int i = 0; i < threads.length; i++) { - try { - threads[i].join(); - } catch (InterruptedException e) { - throw new RuntimeException(e); + Directory dir = null; + RandomIndexWriter iw = null; + final String postingsFormat = _TestUtil.getPostingsFormat("dummy"); + boolean codecOk = iterations * maxWordLength < 100000 || + !(postingsFormat.equals("Memory") || + postingsFormat.equals("SimpleText")); + if (rarely(random) && codecOk) { + dir = newFSDirectory(_TestUtil.getTempDir("bttc")); + iw = new RandomIndexWriter(new Random(seed), dir, a); + } + boolean success = false; + try { + checkRandomData(new Random(seed), a, iterations, maxWordLength, useCharFilter, simple, offsetsAreCorrect, iw); + // now test with multiple threads: note we do the EXACT same thing we did before in each thread, + // so this should only really fail from another thread if its an actual thread problem + int numThreads = _TestUtil.nextInt(random, 2, 4); + AnalysisThread threads[] = new AnalysisThread[numThreads]; + for (int i = 0; i < threads.length; i++) { + threads[i] = new AnalysisThread(seed, a, iterations, maxWordLength, useCharFilter, simple, offsetsAreCorrect, iw); } - } - for (int i = 0; i < threads.length; i++) { - if (threads[i].failed) { - throw new RuntimeException("some thread(s) failed"); + for (int i = 0; i < threads.length; i++) { + threads[i].start(); + } + for (int i = 0; i < threads.length; i++) { + try { + threads[i].join(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + for (int i = 0; i < threads.length; i++) { + if (threads[i].failed) { + throw new RuntimeException("some thread(s) failed"); + } + } + success = true; + } finally { + if (success) { + IOUtils.close(iw, dir); + } else { + IOUtils.closeWhileHandlingException(iw, dir); // checkindex } } } - private static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength, boolean useCharFilter, boolean simple, boolean offsetsAreCorrect) throws IOException { + static final Set doesntSupportOffsets = new HashSet(Arrays.asList( + "Lucene3x", + "MockFixedIntBlock", + "MockVariableIntBlock", + "MockSep", + "MockRandom" + )); + + private static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength, boolean useCharFilter, boolean simple, boolean offsetsAreCorrect, RandomIndexWriter iw) throws IOException { final LineFileDocs docs = new LineFileDocs(random); + Document doc = null; + Field field = null, currentField = null; + StringReader bogus = new StringReader(""); + if (iw != null) { + doc = new Document(); + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); + if (random.nextBoolean()) { + ft.setStoreTermVectors(true); + ft.setStoreTermVectorOffsets(random.nextBoolean()); + ft.setStoreTermVectorPositions(random.nextBoolean()); + } + if (random.nextBoolean()) { + ft.setOmitNorms(true); + } + String pf = _TestUtil.getPostingsFormat("dummy"); + boolean supportsOffsets = !doesntSupportOffsets.contains(pf); + switch(random.nextInt(4)) { + case 0: ft.setIndexOptions(IndexOptions.DOCS_ONLY); break; + case 1: ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS); break; + case 2: ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); break; + default: + if (supportsOffsets && offsetsAreCorrect) { + ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); + } else { + ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); + } + } + currentField = field = new Field("dummy", bogus, ft); + doc.add(currentField); + } try { for (int i = 0; i < iterations; i++) { @@ -485,7 +544,23 @@ public abstract class BaseTokenStreamTestCase extends LuceneTestCase { } try { - checkAnalysisConsistency(random, a, useCharFilter, text, offsetsAreCorrect); + checkAnalysisConsistency(random, a, useCharFilter, text, offsetsAreCorrect, currentField); + if (iw != null) { + if (random.nextInt(7) == 0) { + // pile up a multivalued field + FieldType ft = field.fieldType(); + currentField = new Field("dummy", bogus, ft); + doc.add(currentField); + } else { + iw.addDocument(doc); + if (doc.getFields().size() > 1) { + // back to 1 field + currentField = field; + doc.removeFields("dummy"); + doc.add(currentField); + } + } + } } catch (Throwable t) { // TODO: really we should pass a random seed to // checkAnalysisConsistency then print it here too: @@ -532,6 +607,10 @@ public abstract class BaseTokenStreamTestCase extends LuceneTestCase { } public static void checkAnalysisConsistency(Random random, Analyzer a, boolean useCharFilter, String text, boolean offsetsAreCorrect) throws IOException { + checkAnalysisConsistency(random, a, useCharFilter, text, offsetsAreCorrect, null); + } + + private static void checkAnalysisConsistency(Random random, Analyzer a, boolean useCharFilter, String text, boolean offsetsAreCorrect, Field field) throws IOException { if (VERBOSE) { System.out.println(Thread.currentThread().getName() + ": NOTE: BaseTokenStreamTestCase: get first token stream now text=" + text); @@ -653,6 +732,8 @@ public abstract class BaseTokenStreamTestCase extends LuceneTestCase { } reader = new StringReader(text); + long seed = random.nextLong(); + random = new Random(seed); if (random.nextInt(30) == 7) { if (VERBOSE) { System.out.println(Thread.currentThread().getName() + ": NOTE: BaseTokenStreamTestCase: using spoon-feed reader"); @@ -722,6 +803,20 @@ public abstract class BaseTokenStreamTestCase extends LuceneTestCase { assertTokenStreamContents(ts, tokens.toArray(new String[tokens.size()])); } + + if (field != null) { + reader = new StringReader(text); + random = new Random(seed); + if (random.nextInt(30) == 7) { + if (VERBOSE) { + System.out.println(Thread.currentThread().getName() + ": NOTE: BaseTokenStreamTestCase: indexing using spoon-feed reader"); + } + + reader = new MockReaderWrapper(random, reader); + } + + field.setReaderValue(useCharFilter ? new MockCharFilter(reader, remainder) : reader); + } } private static String randomAnalysisString(Random random, int maxLength, boolean simple) { diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedTokenStream.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedTokenStream.java index 655c574..cc5058b 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedTokenStream.java +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedTokenStream.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,8 +17,6 @@ package org.apache.lucene.analysis; * limitations under the License. */ -import java.io.IOException; - import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; @@ -42,7 +40,7 @@ public final class CannedTokenStream extends TokenStream { } @Override - public boolean incrementToken() throws IOException { + public boolean incrementToken() { if (upto < tokens.length) { final Token token = tokens[upto++]; // TODO: can we just capture/restoreState so diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/CollationTestBase.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/CollationTestBase.java index 4bc3a09..685796f 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/analysis/CollationTestBase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/CollationTestBase.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -29,6 +29,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -44,7 +45,6 @@ import org.apache.lucene.search.TermRangeFilter; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.IndexableBinaryStringTools; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; @@ -59,24 +59,6 @@ public abstract class CollationTestBase extends LuceneTestCase { protected String secondRangeBeginningOriginal = "\u0633"; protected String secondRangeEndOriginal = "\u0638"; - /** - * Convenience method to perform the same function as CollationKeyFilter. - * - * @param keyBits the result from - * collator.getCollationKey(original).toByteArray() - * @return The encoded collation key for the original String - * @deprecated only for testing deprecated filters - */ - @Deprecated - protected String encodeCollationKey(byte[] keyBits) { - // Ensure that the backing char[] array is large enough to hold the encoded - // Binary String - int encodedLength = IndexableBinaryStringTools.getEncodedLength(keyBits, 0, keyBits.length); - char[] encodedBegArray = new char[encodedLength]; - IndexableBinaryStringTools.encode(keyBits, 0, keyBits.length, encodedBegArray, 0, encodedLength); - return new String(encodedBegArray); - } - public void testFarsiRangeFilterCollating(Analyzer analyzer, BytesRef firstBeg, BytesRef firstEnd, BytesRef secondBeg, BytesRef secondEnd) throws Exception { @@ -84,11 +66,11 @@ public abstract class CollationTestBase extends LuceneTestCase { IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( TEST_VERSION_CURRENT, analyzer)); Document doc = new Document(); - doc.add(new Field("content", "\u0633\u0627\u0628", TextField.TYPE_STORED)); - doc.add(new Field("body", "body", StringField.TYPE_STORED)); + doc.add(new TextField("content", "\u0633\u0627\u0628", Field.Store.YES)); + doc.add(new StringField("body", "body", Field.Store.YES)); writer.addDocument(doc); writer.close(); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); Query query = new TermQuery(new Term("body","body")); @@ -121,10 +103,10 @@ public abstract class CollationTestBase extends LuceneTestCase { // orders the U+0698 character before the U+0633 character, so the single // index Term below should NOT be returned by a TermRangeQuery with a Farsi // Collator (or an Arabic one for the case when Farsi is not supported). - doc.add(new Field("content", "\u0633\u0627\u0628", TextField.TYPE_STORED)); + doc.add(new TextField("content", "\u0633\u0627\u0628", Field.Store.YES)); writer.addDocument(doc); writer.close(); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); Query query = new TermRangeQuery("content", firstBeg, firstEnd, true, true); @@ -145,12 +127,12 @@ public abstract class CollationTestBase extends LuceneTestCase { IndexWriter writer = new IndexWriter(farsiIndex, new IndexWriterConfig( TEST_VERSION_CURRENT, analyzer)); Document doc = new Document(); - doc.add(new Field("content", "\u0633\u0627\u0628", TextField.TYPE_STORED)); - doc.add(new Field("body", "body", StringField.TYPE_STORED)); + doc.add(new TextField("content", "\u0633\u0627\u0628", Field.Store.YES)); + doc.add(new StringField("body", "body", Field.Store.YES)); writer.addDocument(doc); writer.close(); - IndexReader reader = IndexReader.open(farsiIndex); + IndexReader reader = DirectoryReader.open(farsiIndex); IndexSearcher search = newSearcher(reader); // Unicode order would include U+0633 in [ U+062F - U+0698 ], but Farsi @@ -213,7 +195,7 @@ public abstract class CollationTestBase extends LuceneTestCase { for (int i = 0 ; i < sortData.length ; ++i) { Document doc = new Document(); doc.add(new Field("tracer", sortData[i][0], customType)); - doc.add(new TextField("contents", sortData[i][1])); + doc.add(new TextField("contents", sortData[i][1], Field.Store.NO)); if (sortData[i][2] != null) doc.add(new TextField("US", usAnalyzer.tokenStream("US", new StringReader(sortData[i][2])))); if (sortData[i][3] != null) @@ -226,7 +208,7 @@ public abstract class CollationTestBase extends LuceneTestCase { } writer.forceMerge(1); writer.close(); - IndexReader reader = IndexReader.open(indexStore); + IndexReader reader = DirectoryReader.open(indexStore); IndexSearcher searcher = new IndexSearcher(reader); Sort sort = new Sort(); diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/EmptyTokenizer.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/EmptyTokenizer.java index 33d7049..3d33330 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/analysis/EmptyTokenizer.java +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/EmptyTokenizer.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,7 +17,6 @@ package org.apache.lucene.analysis; * limitations under the License. */ -import java.io.IOException; import java.io.Reader; /** @@ -30,7 +29,7 @@ public final class EmptyTokenizer extends Tokenizer { } @Override - public boolean incrementToken() throws IOException { + public boolean incrementToken() { return false; } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/LookaheadTokenFilter.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/LookaheadTokenFilter.java index 9515ae9..544bff4 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/analysis/LookaheadTokenFilter.java +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/LookaheadTokenFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java index 0654077..2007597 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockBytesAnalyzer.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockBytesAnalyzer.java new file mode 100644 index 0000000..17b2519 --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockBytesAnalyzer.java @@ -0,0 +1,33 @@ +package org.apache.lucene.analysis; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.Reader; + +/** + * Analyzer for testing that encodes terms as UTF-16 bytes. + */ +public class MockBytesAnalyzer extends Analyzer { + private final MockBytesAttributeFactory factory = new MockBytesAttributeFactory(); + + @Override + protected TokenStreamComponents createComponents(String fieldName, Reader reader) { + Tokenizer t = new MockTokenizer(factory, reader, MockTokenizer.KEYWORD, false, MockTokenizer.DEFAULT_MAX_TOKEN_LENGTH); + return new TokenStreamComponents(t); + } +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockBytesAttributeFactory.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockBytesAttributeFactory.java new file mode 100644 index 0000000..9e4b351 --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockBytesAttributeFactory.java @@ -0,0 +1,40 @@ +package org.apache.lucene.analysis; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.util.Attribute; +import org.apache.lucene.util.AttributeImpl; +import org.apache.lucene.util.AttributeSource; + +/** + * Attribute factory that implements CharTermAttribute with + * {@link MockUTF16TermAttributeImpl} + */ +public class MockBytesAttributeFactory extends AttributeSource.AttributeFactory { + private final AttributeSource.AttributeFactory delegate = + AttributeSource.AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY; + + @Override + public AttributeImpl createAttributeInstance( + Class attClass) { + return attClass.isAssignableFrom(MockUTF16TermAttributeImpl.class) + ? new MockUTF16TermAttributeImpl() + : delegate.createAttributeInstance(attClass); + } + +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockCharFilter.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockCharFilter.java index 5a11b97..20839cc 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockCharFilter.java +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockCharFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java index bbe2f37..fef74f0 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,7 +21,7 @@ import java.io.IOException; import java.util.Random; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; -import org.apache.lucene.index.Payload; +import org.apache.lucene.util.BytesRef; /** * TokenFilter that adds random fixed-length payloads. @@ -30,7 +30,7 @@ public final class MockFixedLengthPayloadFilter extends TokenFilter { private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class); private final Random random; private final byte[] bytes; - private final Payload payload; + private final BytesRef payload; public MockFixedLengthPayloadFilter(Random random, TokenStream in, int length) { super(in); @@ -39,7 +39,7 @@ public final class MockFixedLengthPayloadFilter extends TokenFilter { } this.random = random; this.bytes = new byte[length]; - this.payload = new Payload(bytes); + this.payload = new BytesRef(bytes); } @Override diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockGraphTokenFilter.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockGraphTokenFilter.java index a4376d8..30c49fe 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockGraphTokenFilter.java +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockGraphTokenFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockHoleInjectingTokenFilter.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockHoleInjectingTokenFilter.java index 7baa76e..7685da2 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockHoleInjectingTokenFilter.java +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockHoleInjectingTokenFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java index 54234a0..2c17c78 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java @@ -1,5 +1,5 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,7 +20,7 @@ package org.apache.lucene.analysis; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.index.Payload; +import org.apache.lucene.util.BytesRef; import java.io.IOException; import java.io.Reader; @@ -67,9 +67,9 @@ final class MockPayloadFilter extends TokenFilter { @Override public boolean incrementToken() throws IOException { if (input.incrementToken()) { - payloadAttr.setPayload(new Payload(("pos: " + pos).getBytes())); + payloadAttr.setPayload(new BytesRef(("pos: " + pos).getBytes("UTF-8"))); int posIncr; - if (i % 2 == 1) { + if (pos == 0 || i % 2 == 1) { posIncr = 1; } else { posIncr = 0; diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockRandomLookaheadTokenFilter.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockRandomLookaheadTokenFilter.java index 44215e7..1c0c061 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockRandomLookaheadTokenFilter.java +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockRandomLookaheadTokenFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockReaderWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockReaderWrapper.java index fe266dc..d089f4d 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockReaderWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockReaderWrapper.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenFilter.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenFilter.java index aad40a3..436e0c2 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenFilter.java +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java index 262c1f9..3753d16 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockUTF16TermAttributeImpl.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockUTF16TermAttributeImpl.java new file mode 100644 index 0000000..89f40ab --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockUTF16TermAttributeImpl.java @@ -0,0 +1,41 @@ +package org.apache.lucene.analysis; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.nio.charset.Charset; + +import org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl; +import org.apache.lucene.util.BytesRef; + +/** + * Extension of {@link CharTermAttributeImpl} that encodes the term + * text as UTF-16 bytes instead of as UTF-8 bytes. + */ +public class MockUTF16TermAttributeImpl extends CharTermAttributeImpl { + static final Charset charset = Charset.forName("UTF-16LE"); + + @Override + public int fillBytesRef() { + BytesRef bytes = getBytesRef(); + byte[] utf16 = toString().getBytes(charset); + bytes.bytes = utf16; + bytes.offset = 0; + bytes.length = utf16.length; + return bytes.hashCode(); + } +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java index 0397dee..88403fa 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,7 +21,7 @@ import java.io.IOException; import java.util.Random; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; -import org.apache.lucene.index.Payload; +import org.apache.lucene.util.BytesRef; /** * TokenFilter that adds random variable-length payloads. @@ -32,19 +32,19 @@ public final class MockVariableLengthPayloadFilter extends TokenFilter { private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class); private final Random random; private final byte[] bytes = new byte[MAXLENGTH]; - private final Payload payload; + private final BytesRef payload; public MockVariableLengthPayloadFilter(Random random, TokenStream in) { super(in); this.random = random; - this.payload = new Payload(bytes); + this.payload = new BytesRef(bytes); } @Override public boolean incrementToken() throws IOException { if (input.incrementToken()) { random.nextBytes(bytes); - payload.setData(bytes, 0, random.nextInt(MAXLENGTH)); + payload.length = random.nextInt(MAXLENGTH); payloadAtt.setPayload(payload); return true; } else { diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/TokenStreamToDot.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/TokenStreamToDot.java index aeb1314..0749098 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/analysis/TokenStreamToDot.java +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/TokenStreamToDot.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/ValidatingTokenFilter.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/ValidatingTokenFilter.java index f213545..a7d6456 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/analysis/ValidatingTokenFilter.java +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/ValidatingTokenFilter.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -96,15 +96,6 @@ public final class ValidatingTokenFilter extends TokenFilter { startOffset = offsetAtt.startOffset(); endOffset = offsetAtt.endOffset(); - if (startOffset < 0) { - throw new IllegalStateException(name + ": startOffset=" + startOffset + " is < 0"); - } - if (endOffset < 0) { - throw new IllegalStateException(name + ": endOffset=" + endOffset + " is < 0"); - } - if (endOffset < startOffset) { - throw new IllegalStateException(name + ": startOffset=" + startOffset + " is > endOffset=" + endOffset + " pos=" + pos + "; token=" + termAtt); - } if (offsetsAreCorrect && offsetAtt.startOffset() < lastStartOffset) { throw new IllegalStateException(name + ": offsets must not go backwards startOffset=" + startOffset + " is < lastStartOffset=" + lastStartOffset); } diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/VocabularyAssert.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/VocabularyAssert.java index 7a1dedc..cbc9694 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/analysis/VocabularyAssert.java +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/VocabularyAssert.java @@ -1,6 +1,6 @@ package org.apache.lucene.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWCodec.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWCodec.java deleted file mode 100644 index 022f441..0000000 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWCodec.java +++ /dev/null @@ -1,106 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.codecs.FieldInfosFormat; -import org.apache.lucene.codecs.LiveDocsFormat; -import org.apache.lucene.codecs.NormsFormat; -import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.SegmentInfoFormat; -import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.TermVectorsFormat; -import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat; -import org.apache.lucene.util.LuceneTestCase; - -/** - * Writes 3.x-like indexes (not perfect emulation yet) for testing only! - * @lucene.experimental - */ -public class PreFlexRWCodec extends Lucene3xCodec { - private final PostingsFormat postings = new PreFlexRWPostingsFormat(); - private final Lucene3xNormsFormat norms = new PreFlexRWNormsFormat(); - private final FieldInfosFormat fieldInfos = new PreFlexRWFieldInfosFormat(); - private final TermVectorsFormat termVectors = new PreFlexRWTermVectorsFormat(); - private final SegmentInfoFormat segmentInfos = new PreFlexRWSegmentInfoFormat(); - private final StoredFieldsFormat storedFields = new PreFlexRWStoredFieldsFormat(); - // TODO: this should really be a different impl - private final LiveDocsFormat liveDocs = new Lucene40LiveDocsFormat(); - - @Override - public PostingsFormat postingsFormat() { - if (LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE) { - return postings; - } else { - return super.postingsFormat(); - } - } - - @Override - public NormsFormat normsFormat() { - if (LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE) { - return norms; - } else { - return super.normsFormat(); - } - } - - @Override - public SegmentInfoFormat segmentInfoFormat() { - if (LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE) { - return segmentInfos ; - } else { - return super.segmentInfoFormat(); - } - } - - @Override - public FieldInfosFormat fieldInfosFormat() { - if (LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE) { - return fieldInfos; - } else { - return super.fieldInfosFormat(); - } - } - - @Override - public TermVectorsFormat termVectorsFormat() { - if (LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE) { - return termVectors; - } else { - return super.termVectorsFormat(); - } - } - - @Override - public LiveDocsFormat liveDocsFormat() { - if (LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE) { - return liveDocs; - } else { - return super.liveDocsFormat(); - } - } - - @Override - public StoredFieldsFormat storedFieldsFormat() { - if (LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE) { - return storedFields; - } else { - return super.storedFieldsFormat(); - } - } -} diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldInfosFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldInfosFormat.java deleted file mode 100644 index b342568..0000000 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldInfosFormat.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.codecs.FieldInfosReader; -import org.apache.lucene.codecs.FieldInfosWriter; - -/** - * - * @lucene.internal - * @lucene.experimental - */ -class PreFlexRWFieldInfosFormat extends Lucene3xFieldInfosFormat { - - @Override - public FieldInfosReader getFieldInfosReader() throws IOException { - return new PreFlexRWFieldInfosReader(); - } - - @Override - public FieldInfosWriter getFieldInfosWriter() throws IOException { - return new PreFlexRWFieldInfosWriter(); - } -} diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldInfosReader.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldInfosReader.java deleted file mode 100644 index 5918149..0000000 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldInfosReader.java +++ /dev/null @@ -1,110 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.io.IOException; -import java.util.Set; - -import org.apache.lucene.codecs.FieldInfosReader; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.DocValues.Type; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.IndexFormatTooNewException; -import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; - -/** - * @lucene.internal - * @lucene.experimental - */ -class PreFlexRWFieldInfosReader extends FieldInfosReader { - static final int FORMAT_MINIMUM = PreFlexRWFieldInfosWriter.FORMAT_START; - - @Override - public FieldInfos read(Directory directory, String segmentName, IOContext iocontext) throws IOException { - final String fileName = IndexFileNames.segmentFileName(segmentName, "", PreFlexRWFieldInfosWriter.FIELD_INFOS_EXTENSION); - IndexInput input = directory.openInput(fileName, iocontext); - - try { - final int format = input.readVInt(); - - if (format > FORMAT_MINIMUM) { - throw new IndexFormatTooOldException(input, format, FORMAT_MINIMUM, PreFlexRWFieldInfosWriter.FORMAT_CURRENT); - } - if (format < PreFlexRWFieldInfosWriter.FORMAT_CURRENT && format != PreFlexRWFieldInfosWriter.FORMAT_PREFLEX_RW) { - throw new IndexFormatTooNewException(input, format, FORMAT_MINIMUM, PreFlexRWFieldInfosWriter.FORMAT_CURRENT); - } - - final int size = input.readVInt(); //read in the size - FieldInfo infos[] = new FieldInfo[size]; - - for (int i = 0; i < size; i++) { - String name = input.readString(); - final int fieldNumber = format == PreFlexRWFieldInfosWriter.FORMAT_PREFLEX_RW ? input.readInt() : i; - byte bits = input.readByte(); - boolean isIndexed = (bits & PreFlexRWFieldInfosWriter.IS_INDEXED) != 0; - boolean storeTermVector = (bits & PreFlexRWFieldInfosWriter.STORE_TERMVECTOR) != 0; - boolean omitNorms = (bits & PreFlexRWFieldInfosWriter.OMIT_NORMS) != 0; - boolean storePayloads = (bits & PreFlexRWFieldInfosWriter.STORE_PAYLOADS) != 0; - final IndexOptions indexOptions; - if ((bits & PreFlexRWFieldInfosWriter.OMIT_TERM_FREQ_AND_POSITIONS) != 0) { - indexOptions = IndexOptions.DOCS_ONLY; - } else if ((bits & PreFlexRWFieldInfosWriter.OMIT_POSITIONS) != 0) { - if (format <= PreFlexRWFieldInfosWriter.FORMAT_OMIT_POSITIONS) { - indexOptions = IndexOptions.DOCS_AND_FREQS; - } else { - throw new CorruptIndexException("Corrupt fieldinfos, OMIT_POSITIONS set but format=" + format + " (resource: " + input + ")"); - } - } else { - indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; - } - - // LUCENE-3027: past indices were able to write - // storePayloads=true when omitTFAP is also true, - // which is invalid. We correct that, here: - if (indexOptions != IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { - storePayloads = false; - } - - Type normType = isIndexed && !omitNorms ? Type.FIXED_INTS_8 : null; - if (format == PreFlexRWFieldInfosWriter.FORMAT_PREFLEX_RW && normType != null) { - // RW can have norms but doesn't write them - normType = input.readByte() != 0 ? Type.FIXED_INTS_8 : null; - } - - infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, - omitNorms, storePayloads, indexOptions, null, normType, null); - } - - if (input.getFilePointer() != input.length()) { - throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")"); - } - return new FieldInfos(infos); - } finally { - input.close(); - } - } - - public static void files(Directory dir, SegmentInfo info, Set files) throws IOException { - files.add(IndexFileNames.segmentFileName(info.name, "", PreFlexRWFieldInfosWriter.FIELD_INFOS_EXTENSION)); - } -} diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldInfosWriter.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldInfosWriter.java deleted file mode 100644 index ffa91ce..0000000 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldInfosWriter.java +++ /dev/null @@ -1,96 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.io.IOException; - -import org.apache.lucene.codecs.FieldInfosWriter; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexOutput; - -/** - * @lucene.internal - * @lucene.experimental - */ -class PreFlexRWFieldInfosWriter extends FieldInfosWriter { - // TODO move to test-framework preflex RW? - - /** Extension of field infos */ - static final String FIELD_INFOS_EXTENSION = "fnm"; - - // First used in 2.9; prior to 2.9 there was no format header - static final int FORMAT_START = -2; - // First used in 3.4: omit only positional information - static final int FORMAT_OMIT_POSITIONS = -3; - - static final int FORMAT_PREFLEX_RW = Integer.MIN_VALUE; - - // whenever you add a new format, make it 1 smaller (negative version logic)! - static final int FORMAT_CURRENT = FORMAT_OMIT_POSITIONS; - - static final byte IS_INDEXED = 0x1; - static final byte STORE_TERMVECTOR = 0x2; - static final byte OMIT_NORMS = 0x10; - static final byte STORE_PAYLOADS = 0x20; - static final byte OMIT_TERM_FREQ_AND_POSITIONS = 0x40; - static final byte OMIT_POSITIONS = -128; - - @Override - public void write(Directory directory, String segmentName, FieldInfos infos, IOContext context) throws IOException { - final String fileName = IndexFileNames.segmentFileName(segmentName, "", FIELD_INFOS_EXTENSION); - IndexOutput output = directory.createOutput(fileName, context); - try { - output.writeVInt(FORMAT_PREFLEX_RW); - output.writeVInt(infos.size()); - for (FieldInfo fi : infos) { - assert fi.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !fi.hasPayloads(); - byte bits = 0x0; - if (fi.isIndexed()) bits |= IS_INDEXED; - if (fi.hasVectors()) bits |= STORE_TERMVECTOR; - if (fi.omitsNorms()) bits |= OMIT_NORMS; - if (fi.hasPayloads()) bits |= STORE_PAYLOADS; - if (fi.getIndexOptions() == IndexOptions.DOCS_ONLY) { - bits |= OMIT_TERM_FREQ_AND_POSITIONS; - } else if (fi.getIndexOptions() == IndexOptions.DOCS_AND_FREQS) { - bits |= OMIT_POSITIONS; - } - output.writeString(fi.name); - /* - * we need to write the field number since IW tries - * to stabelize the field numbers across segments so the - * FI ordinal is not necessarily equivalent to the field number - */ - output.writeInt(fi.number); - output.writeByte(bits); - if (fi.isIndexed() && !fi.omitsNorms()) { - // to allow null norm types we need to indicate if norms are written - // only in RW case - output.writeByte((byte) (fi.getNormType() == null ? 0 : 1)); - } - assert fi.attributes() == null; // not used or supported - } - } finally { - output.close(); - } - } - -} diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldsWriter.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldsWriter.java deleted file mode 100644 index 0d01752..0000000 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldsWriter.java +++ /dev/null @@ -1,224 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Comparator; - -import org.apache.lucene.codecs.FieldsConsumer; -import org.apache.lucene.codecs.PostingsConsumer; -import org.apache.lucene.codecs.TermStats; -import org.apache.lucene.codecs.TermsConsumer; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.IOUtils; - -class PreFlexRWFieldsWriter extends FieldsConsumer { - - private final TermInfosWriter termsOut; - private final IndexOutput freqOut; - private final IndexOutput proxOut; - private final PreFlexRWSkipListWriter skipListWriter; - private final int totalNumDocs; - - public PreFlexRWFieldsWriter(SegmentWriteState state) throws IOException { - termsOut = new TermInfosWriter(state.directory, - state.segmentInfo.name, - state.fieldInfos, - state.termIndexInterval); - - boolean success = false; - try { - final String freqFile = IndexFileNames.segmentFileName(state.segmentInfo.name, "", Lucene3xPostingsFormat.FREQ_EXTENSION); - freqOut = state.directory.createOutput(freqFile, state.context); - totalNumDocs = state.segmentInfo.getDocCount(); - success = true; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(termsOut); - } - } - - success = false; - try { - if (state.fieldInfos.hasProx()) { - final String proxFile = IndexFileNames.segmentFileName(state.segmentInfo.name, "", Lucene3xPostingsFormat.PROX_EXTENSION); - proxOut = state.directory.createOutput(proxFile, state.context); - } else { - proxOut = null; - } - success = true; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(termsOut, freqOut); - } - } - - skipListWriter = new PreFlexRWSkipListWriter(termsOut.skipInterval, - termsOut.maxSkipLevels, - totalNumDocs, - freqOut, - proxOut); - //System.out.println("\nw start seg=" + segment); - } - - @Override - public TermsConsumer addField(FieldInfo field) throws IOException { - assert field.number != -1; - if (field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0) { - throw new UnsupportedOperationException("this codec cannot index offsets"); - } - //System.out.println("w field=" + field.name + " storePayload=" + field.storePayloads + " number=" + field.number); - return new PreFlexTermsWriter(field); - } - - @Override - public void close() throws IOException { - IOUtils.close(termsOut, freqOut, proxOut); - } - - private class PreFlexTermsWriter extends TermsConsumer { - private final FieldInfo fieldInfo; - private final boolean omitTF; - private final boolean storePayloads; - - private final TermInfo termInfo = new TermInfo(); - private final PostingsWriter postingsWriter = new PostingsWriter(); - - public PreFlexTermsWriter(FieldInfo fieldInfo) { - this.fieldInfo = fieldInfo; - omitTF = fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY; - storePayloads = fieldInfo.hasPayloads(); - } - - private class PostingsWriter extends PostingsConsumer { - private int lastDocID; - private int lastPayloadLength = -1; - private int lastPosition; - private int df; - - public PostingsWriter reset() { - df = 0; - lastDocID = 0; - lastPayloadLength = -1; - return this; - } - - @Override - public void startDoc(int docID, int termDocFreq) throws IOException { - //System.out.println(" w doc=" + docID); - - final int delta = docID - lastDocID; - if (docID < 0 || (df > 0 && delta <= 0)) { - throw new CorruptIndexException("docs out of order (" + docID + " <= " + lastDocID + " )"); - } - - if ((++df % termsOut.skipInterval) == 0) { - skipListWriter.setSkipData(lastDocID, storePayloads, lastPayloadLength); - skipListWriter.bufferSkip(df); - } - - lastDocID = docID; - - assert docID < totalNumDocs: "docID=" + docID + " totalNumDocs=" + totalNumDocs; - - if (omitTF) { - freqOut.writeVInt(delta); - } else { - final int code = delta << 1; - if (termDocFreq == 1) { - freqOut.writeVInt(code|1); - } else { - freqOut.writeVInt(code); - freqOut.writeVInt(termDocFreq); - } - } - lastPosition = 0; - } - - @Override - public void addPosition(int position, BytesRef payload, int startOffset, int endOffset) throws IOException { - assert proxOut != null; - assert startOffset == -1; - assert endOffset == -1; - //System.out.println(" w pos=" + position + " payl=" + payload); - final int delta = position - lastPosition; - lastPosition = position; - - if (storePayloads) { - final int payloadLength = payload == null ? 0 : payload.length; - if (payloadLength != lastPayloadLength) { - //System.out.println(" write payload len=" + payloadLength); - lastPayloadLength = payloadLength; - proxOut.writeVInt((delta<<1)|1); - proxOut.writeVInt(payloadLength); - } else { - proxOut.writeVInt(delta << 1); - } - if (payloadLength > 0) { - proxOut.writeBytes(payload.bytes, payload.offset, payload.length); - } - } else { - proxOut.writeVInt(delta); - } - } - - @Override - public void finishDoc() throws IOException { - } - } - - @Override - public PostingsConsumer startTerm(BytesRef text) throws IOException { - //System.out.println(" w term=" + text.utf8ToString()); - skipListWriter.resetSkip(); - termInfo.freqPointer = freqOut.getFilePointer(); - if (proxOut != null) { - termInfo.proxPointer = proxOut.getFilePointer(); - } - return postingsWriter.reset(); - } - - @Override - public void finishTerm(BytesRef text, TermStats stats) throws IOException { - if (stats.docFreq > 0) { - long skipPointer = skipListWriter.writeSkip(freqOut); - termInfo.docFreq = stats.docFreq; - termInfo.skipOffset = (int) (skipPointer - termInfo.freqPointer); - //System.out.println(" w finish term=" + text.utf8ToString() + " fnum=" + fieldInfo.number); - termsOut.add(fieldInfo.number, - text, - termInfo); - } - } - - @Override - public void finish(long sumTotalTermCount, long sumDocFreq, int docCount) throws IOException { - } - - @Override - public Comparator getComparator() throws IOException { - return BytesRef.getUTF8SortedAsUTF16Comparator(); - } - } -} \ No newline at end of file diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWNormsConsumer.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWNormsConsumer.java deleted file mode 100644 index 5debc5b..0000000 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWNormsConsumer.java +++ /dev/null @@ -1,293 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Arrays; - -import org.apache.lucene.codecs.DocValuesConsumer; -import org.apache.lucene.codecs.PerDocConsumer; -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.DocValues.Source; -import org.apache.lucene.index.DocValues.Type; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.MergeState; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.IOUtils; - -/** - * Writes and Merges Lucene 3.x norms format - * @lucene.experimental - */ -class PreFlexRWNormsConsumer extends PerDocConsumer { - - /** norms header placeholder */ - private static final byte[] NORMS_HEADER = new byte[]{'N','R','M',-1}; - - /** Extension of norms file */ - private static final String NORMS_EXTENSION = "nrm"; - - /** Extension of separate norms file - * @deprecated */ - @Deprecated - private static final String SEPARATE_NORMS_EXTENSION = "s"; - - private final Directory directory; - - private final String segment; - - private final IOContext context; - - private NormsWriter writer; - - public PreFlexRWNormsConsumer(Directory directory, String segment, IOContext context){ - this.directory = directory; - this.segment = segment; - this.context = context; - } - - @Override - public void merge(MergeState mergeState) throws IOException { - getNormsWriter().merge(mergeState); - } - - @Override - public void close() throws IOException { - if (writer != null) { - writer.finish(); - } - } - - @Override - protected boolean canMerge(FieldInfo info) { - return info.hasNorms(); - } - - @Override - protected Type getDocValuesType(FieldInfo info) { - return info.getNormType(); - } - - @Override - public DocValuesConsumer addValuesField(Type type, FieldInfo fieldInfo) - throws IOException { - if (type != Type.FIXED_INTS_8) { - throw new UnsupportedOperationException("Codec only supports single byte norm values. Type give: " + type); - } - return new Lucene3xNormsDocValuesConsumer(fieldInfo); - } - - class Lucene3xNormsDocValuesConsumer extends DocValuesConsumer { - // Holds all docID/norm pairs we've seen - private int[] docIDs = new int[1]; - private byte[] norms = new byte[1]; - private int upto; - private final FieldInfo fi; - - Lucene3xNormsDocValuesConsumer(FieldInfo fieldInfo) { - fi = fieldInfo; - } - - @Override - public void finish(int docCount) throws IOException { - final NormsWriter normsWriter = getNormsWriter(); - boolean success = false; - try { - int uptoDoc = 0; - normsWriter.setNumTotalDocs(docCount); - if (upto > 0) { - normsWriter.startField(fi); - int docID = 0; - for (; docID < docCount; docID++) { - if (uptoDoc < upto && docIDs[uptoDoc] == docID) { - normsWriter.writeNorm(norms[uptoDoc]); - uptoDoc++; - } else { - normsWriter.writeNorm((byte) 0); - } - } - // we should have consumed every norm - assert uptoDoc == upto; - - } else { - // Fill entire field with default norm: - normsWriter.startField(fi); - for (; upto < docCount; upto++) - normsWriter.writeNorm((byte) 0); - } - success = true; - } finally { - if (!success) { - normsWriter.abort(); - } - } - } - - @Override - public void add(int docID, IndexableField docValue) throws IOException { - add(docID, docValue.numericValue().longValue()); - } - - protected void add(int docID, long value) { - if (docIDs.length <= upto) { - assert docIDs.length == upto; - docIDs = ArrayUtil.grow(docIDs, 1 + upto); - } - if (norms.length <= upto) { - assert norms.length == upto; - norms = ArrayUtil.grow(norms, 1 + upto); - } - norms[upto] = (byte) value; - - docIDs[upto] = docID; - upto++; - } - - @Override - protected Type getType() { - return Type.FIXED_INTS_8; - } - - - } - - public NormsWriter getNormsWriter() throws IOException { - if (writer == null) { - writer = new NormsWriter(directory, segment, context); - } - return writer; - } - - private static class NormsWriter { - - private final IndexOutput output; - private int normCount = 0; - private int numTotalDocs = 0; - - public NormsWriter(Directory directory, String segment, IOContext context) throws IOException { - final String normsFileName = IndexFileNames.segmentFileName(segment, "", NORMS_EXTENSION); - boolean success = false; - IndexOutput out = null; - try { - out = directory.createOutput(normsFileName, context); - output = out; - output.writeBytes(NORMS_HEADER, 0, NORMS_HEADER.length); - success = true; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(out); - } - } - - } - - - public void setNumTotalDocs(int numTotalDocs) { - assert this.numTotalDocs == 0 || numTotalDocs == this.numTotalDocs; - this.numTotalDocs = numTotalDocs; - } - - public void startField(FieldInfo info) throws IOException { - assert info.omitsNorms() == false; - normCount++; - } - - public void writeNorm(byte norm) throws IOException { - output.writeByte(norm); - } - - public void abort() throws IOException { - IOUtils.close(output); - } - - public void finish() throws IOException { - IOUtils.close(output); - - if (4+normCount*(long)numTotalDocs != output.getFilePointer()) { - throw new IOException(".nrm file size mismatch: expected=" + (4+normCount*(long)numTotalDocs) + " actual=" + output.getFilePointer()); - } - } - // TODO: we can actually use the defaul DV merge here and drop this specific stuff entirely - /** we override merge and bulk-merge norms when there are no deletions */ - public void merge(MergeState mergeState) throws IOException { - int numMergedDocs = 0; - for (FieldInfo fi : mergeState.fieldInfos) { - if (fi.hasNorms()) { - startField(fi); - int numMergedDocsForField = 0; - for (MergeState.IndexReaderAndLiveDocs reader : mergeState.readers) { - final int maxDoc = reader.reader.maxDoc(); - byte[] normBuffer; - DocValues normValues = reader.reader.normValues(fi.name); - if (normValues == null) { - // Can be null if this segment doesn't have - // any docs with this field - normBuffer = new byte[maxDoc]; - Arrays.fill(normBuffer, (byte)0); - } else { - Source directSource = normValues.getDirectSource(); - assert directSource.hasArray(); - normBuffer = (byte[]) directSource.getArray(); - } - if (reader.liveDocs == null) { - //optimized case for segments without deleted docs - output.writeBytes(normBuffer, maxDoc); - numMergedDocsForField += maxDoc; - } else { - // this segment has deleted docs, so we have to - // check for every doc if it is deleted or not - final Bits liveDocs = reader.liveDocs; - for (int k = 0; k < maxDoc; k++) { - if (liveDocs.get(k)) { - numMergedDocsForField++; - output.writeByte(normBuffer[k]); - } - } - } - mergeState.checkAbort.work(maxDoc); - } - assert numMergedDocs == 0 || numMergedDocs == numMergedDocsForField; - numMergedDocs = numMergedDocsForField; - } - } - this.numTotalDocs = numMergedDocs; - } - } - - @Override - public void abort() { - try { - try { - if (writer != null) { - writer.abort(); - } - } finally { - directory.deleteFile(IndexFileNames.segmentFileName(segment, "", - NORMS_EXTENSION)); - } - } catch (IOException e) { - // ignore - } - } -} diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWNormsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWNormsFormat.java deleted file mode 100644 index 1ef1eec..0000000 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWNormsFormat.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.codecs.PerDocConsumer; -import org.apache.lucene.index.PerDocWriteState; - -/** - * @lucene.internal - * @lucene.experimental - */ -class PreFlexRWNormsFormat extends Lucene3xNormsFormat { - - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new PreFlexRWNormsConsumer(state.directory, state.segmentInfo.name, state.context); - } -} diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWPostingsFormat.java deleted file mode 100644 index 9b14b26..0000000 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWPostingsFormat.java +++ /dev/null @@ -1,74 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.codecs.FieldsConsumer; -import org.apache.lucene.codecs.FieldsProducer; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.util.LuceneTestCase; - -/** Codec, only for testing, that can write and read the - * pre-flex index format. - * - * @lucene.experimental - */ -class PreFlexRWPostingsFormat extends Lucene3xPostingsFormat { - - public PreFlexRWPostingsFormat() { - // NOTE: we impersonate the PreFlex codec so that it can - // read the segments we write! - } - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - return new PreFlexRWFieldsWriter(state); - } - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - - // Whenever IW opens readers, eg for merging, we have to - // keep terms order in UTF16: - - return new Lucene3xFields(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.termsIndexDivisor) { - @Override - protected boolean sortTermsByUnicode() { - // We carefully peek into stack track above us: if - // we are part of a "merge", we must sort by UTF16: - boolean unicodeSortOrder = true; - - StackTraceElement[] trace = new Exception().getStackTrace(); - for (int i = 0; i < trace.length; i++) { - //System.out.println(trace[i].getClassName()); - if ("merge".equals(trace[i].getMethodName())) { - unicodeSortOrder = false; - if (LuceneTestCase.VERBOSE) { - System.out.println("NOTE: PreFlexRW codec: forcing legacy UTF16 term sort order"); - } - break; - } - } - - return unicodeSortOrder; - } - }; - } -} diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWSegmentInfoFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWSegmentInfoFormat.java deleted file mode 100644 index 0e0592a..0000000 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWSegmentInfoFormat.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -import org.apache.lucene.codecs.SegmentInfoWriter; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * @lucene.experimental - */ -class PreFlexRWSegmentInfoFormat extends Lucene3xSegmentInfoFormat { - private final SegmentInfoWriter writer = new PreFlexRWSegmentInfoWriter(); - - @Override - public SegmentInfoWriter getSegmentInfosWriter() { - return writer; - } -} diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWSegmentInfoWriter.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWSegmentInfoWriter.java deleted file mode 100644 index 471b3ad..0000000 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWSegmentInfoWriter.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.codecs.SegmentInfoWriter; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentInfos; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; - -/** - * PreFlex implementation of {@link SegmentInfoWriter}. - * @lucene.experimental - */ -class PreFlexRWSegmentInfoWriter extends SegmentInfoWriter { - - // NOTE: this is not "really" 3.x format, because we are - // writing each SI to its own file, vs 3.x where the list - // of segments and SI for each segment is written into a - // single segments_N file - - /** Save a single segment's info. */ - @Override - public void write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext) throws IOException { - SegmentInfos.write3xInfo(dir, si, ioContext); - } -} diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWSkipListWriter.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWSkipListWriter.java deleted file mode 100644 index f6c8a7d..0000000 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWSkipListWriter.java +++ /dev/null @@ -1,127 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Arrays; - -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.codecs.MultiLevelSkipListWriter; - - -/** - * PreFlexRW skiplist implementation. - * @lucene.experimental - */ -public class PreFlexRWSkipListWriter extends MultiLevelSkipListWriter { - private int[] lastSkipDoc; - private int[] lastSkipPayloadLength; - private long[] lastSkipFreqPointer; - private long[] lastSkipProxPointer; - - private IndexOutput freqOutput; - private IndexOutput proxOutput; - - private int curDoc; - private boolean curStorePayloads; - private int curPayloadLength; - private long curFreqPointer; - private long curProxPointer; - - public PreFlexRWSkipListWriter(int skipInterval, int numberOfSkipLevels, int docCount, IndexOutput freqOutput, IndexOutput proxOutput) { - super(skipInterval, numberOfSkipLevels, docCount); - this.freqOutput = freqOutput; - this.proxOutput = proxOutput; - - lastSkipDoc = new int[numberOfSkipLevels]; - lastSkipPayloadLength = new int[numberOfSkipLevels]; - lastSkipFreqPointer = new long[numberOfSkipLevels]; - lastSkipProxPointer = new long[numberOfSkipLevels]; - } - - /** - * Sets the values for the current skip data. - */ - public void setSkipData(int doc, boolean storePayloads, int payloadLength) { - this.curDoc = doc; - this.curStorePayloads = storePayloads; - this.curPayloadLength = payloadLength; - this.curFreqPointer = freqOutput.getFilePointer(); - if (proxOutput != null) - this.curProxPointer = proxOutput.getFilePointer(); - } - - @Override - public void resetSkip() { - super.resetSkip(); - Arrays.fill(lastSkipDoc, 0); - Arrays.fill(lastSkipPayloadLength, -1); // we don't have to write the first length in the skip list - Arrays.fill(lastSkipFreqPointer, freqOutput.getFilePointer()); - if (proxOutput != null) - Arrays.fill(lastSkipProxPointer, proxOutput.getFilePointer()); - } - - @Override - protected void writeSkipData(int level, IndexOutput skipBuffer) throws IOException { - // To efficiently store payloads in the posting lists we do not store the length of - // every payload. Instead we omit the length for a payload if the previous payload had - // the same length. - // However, in order to support skipping the payload length at every skip point must be known. - // So we use the same length encoding that we use for the posting lists for the skip data as well: - // Case 1: current field does not store payloads - // SkipDatum --> DocSkip, FreqSkip, ProxSkip - // DocSkip,FreqSkip,ProxSkip --> VInt - // DocSkip records the document number before every SkipInterval th document in TermFreqs. - // Document numbers are represented as differences from the previous value in the sequence. - // Case 2: current field stores payloads - // SkipDatum --> DocSkip, PayloadLength?, FreqSkip,ProxSkip - // DocSkip,FreqSkip,ProxSkip --> VInt - // PayloadLength --> VInt - // In this case DocSkip/2 is the difference between - // the current and the previous value. If DocSkip - // is odd, then a PayloadLength encoded as VInt follows, - // if DocSkip is even, then it is assumed that the - // current payload length equals the length at the previous - // skip point - if (curStorePayloads) { - int delta = curDoc - lastSkipDoc[level]; - if (curPayloadLength == lastSkipPayloadLength[level]) { - // the current payload length equals the length at the previous skip point, - // so we don't store the length again - skipBuffer.writeVInt(delta * 2); - } else { - // the payload length is different from the previous one. We shift the DocSkip, - // set the lowest bit and store the current payload length as VInt. - skipBuffer.writeVInt(delta * 2 + 1); - skipBuffer.writeVInt(curPayloadLength); - lastSkipPayloadLength[level] = curPayloadLength; - } - } else { - // current field does not store payloads - skipBuffer.writeVInt(curDoc - lastSkipDoc[level]); - } - - skipBuffer.writeVInt((int) (curFreqPointer - lastSkipFreqPointer[level])); - skipBuffer.writeVInt((int) (curProxPointer - lastSkipProxPointer[level])); - - lastSkipDoc[level] = curDoc; - - lastSkipFreqPointer[level] = curFreqPointer; - lastSkipProxPointer[level] = curProxPointer; - } -} diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWStoredFieldsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWStoredFieldsFormat.java deleted file mode 100644 index c4cf514..0000000 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWStoredFieldsFormat.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.codecs.StoredFieldsWriter; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; - -class PreFlexRWStoredFieldsFormat extends Lucene3xStoredFieldsFormat { - - @Override - public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context) throws IOException { - return new PreFlexRWStoredFieldsWriter(directory, segmentInfo.name, context); - } -} diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWStoredFieldsWriter.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWStoredFieldsWriter.java deleted file mode 100644 index d14eb01..0000000 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWStoredFieldsWriter.java +++ /dev/null @@ -1,156 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Copyright 2004 The Apache Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -import java.io.IOException; - -import org.apache.lucene.codecs.StoredFieldsWriter; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.IOUtils; - -/** @lucene.experimental */ -final class PreFlexRWStoredFieldsWriter extends StoredFieldsWriter { - private final Directory directory; - private final String segment; - private IndexOutput fieldsStream; - private IndexOutput indexStream; - - public PreFlexRWStoredFieldsWriter(Directory directory, String segment, IOContext context) throws IOException { - assert directory != null; - this.directory = directory; - this.segment = segment; - - boolean success = false; - try { - fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", Lucene3xStoredFieldsReader.FIELDS_EXTENSION), context); - indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", Lucene3xStoredFieldsReader.FIELDS_INDEX_EXTENSION), context); - - fieldsStream.writeInt(Lucene3xStoredFieldsReader.FORMAT_CURRENT); - indexStream.writeInt(Lucene3xStoredFieldsReader.FORMAT_CURRENT); - - success = true; - } finally { - if (!success) { - abort(); - } - } - } - - // Writes the contents of buffer into the fields stream - // and adds a new entry for this document into the index - // stream. This assumes the buffer was already written - // in the correct fields format. - public void startDocument(int numStoredFields) throws IOException { - indexStream.writeLong(fieldsStream.getFilePointer()); - fieldsStream.writeVInt(numStoredFields); - } - - public void close() throws IOException { - try { - IOUtils.close(fieldsStream, indexStream); - } finally { - fieldsStream = indexStream = null; - } - } - - public void abort() { - try { - close(); - } catch (IOException ignored) {} - IOUtils.deleteFilesIgnoringExceptions(directory, - IndexFileNames.segmentFileName(segment, "", Lucene3xStoredFieldsReader.FIELDS_EXTENSION), - IndexFileNames.segmentFileName(segment, "", Lucene3xStoredFieldsReader.FIELDS_INDEX_EXTENSION)); - } - - public void writeField(FieldInfo info, IndexableField field) throws IOException { - fieldsStream.writeVInt(info.number); - int bits = 0; - final BytesRef bytes; - final String string; - // TODO: maybe a field should serialize itself? - // this way we don't bake into indexer all these - // specific encodings for different fields? and apps - // can customize... - - Number number = field.numericValue(); - if (number != null) { - if (number instanceof Byte || number instanceof Short || number instanceof Integer) { - bits |= Lucene3xStoredFieldsReader.FIELD_IS_NUMERIC_INT; - } else if (number instanceof Long) { - bits |= Lucene3xStoredFieldsReader.FIELD_IS_NUMERIC_LONG; - } else if (number instanceof Float) { - bits |= Lucene3xStoredFieldsReader.FIELD_IS_NUMERIC_FLOAT; - } else if (number instanceof Double) { - bits |= Lucene3xStoredFieldsReader.FIELD_IS_NUMERIC_DOUBLE; - } else { - throw new IllegalArgumentException("cannot store numeric type " + number.getClass()); - } - string = null; - bytes = null; - } else { - bytes = field.binaryValue(); - if (bytes != null) { - bits |= Lucene3xStoredFieldsReader.FIELD_IS_BINARY; - string = null; - } else { - string = field.stringValue(); - if (string == null) { - throw new IllegalArgumentException("field " + field.name() + " is stored but does not have binaryValue, stringValue nor numericValue"); - } - } - } - - fieldsStream.writeByte((byte) bits); - - if (bytes != null) { - fieldsStream.writeVInt(bytes.length); - fieldsStream.writeBytes(bytes.bytes, bytes.offset, bytes.length); - } else if (string != null) { - fieldsStream.writeString(field.stringValue()); - } else { - if (number instanceof Byte || number instanceof Short || number instanceof Integer) { - fieldsStream.writeInt(number.intValue()); - } else if (number instanceof Long) { - fieldsStream.writeLong(number.longValue()); - } else if (number instanceof Float) { - fieldsStream.writeInt(Float.floatToIntBits(number.floatValue())); - } else if (number instanceof Double) { - fieldsStream.writeLong(Double.doubleToLongBits(number.doubleValue())); - } else { - assert false; - } - } - } - - @Override - public void finish(FieldInfos fis, int numDocs) throws IOException { - if (4+((long) numDocs)*8 != indexStream.getFilePointer()) - // This is most likely a bug in Sun JRE 1.6.0_04/_05; - // we detect that the bug has struck, here, and - // throw an exception to prevent the corruption from - // entering the index. See LUCENE-1282 for - // details. - throw new RuntimeException("fdx size mismatch: docCount is " + numDocs + " but fdx file size is " + indexStream.getFilePointer() + " file=" + indexStream.toString() + "; now aborting this merge to prevent index corruption"); - } -} diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWTermVectorsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWTermVectorsFormat.java deleted file mode 100644 index b734c97..0000000 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWTermVectorsFormat.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.codecs.TermVectorsReader; -import org.apache.lucene.codecs.TermVectorsWriter; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.util.LuceneTestCase; - -class PreFlexRWTermVectorsFormat extends Lucene3xTermVectorsFormat { - - @Override - public TermVectorsWriter vectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context) throws IOException { - return new PreFlexRWTermVectorsWriter(directory, segmentInfo.name, context); - } - - @Override - public TermVectorsReader vectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) throws IOException { - return new Lucene3xTermVectorsReader(directory, segmentInfo, fieldInfos, context) { - @Override - protected boolean sortTermsByUnicode() { - // We carefully peek into stack track above us: if - // we are part of a "merge", we must sort by UTF16: - boolean unicodeSortOrder = true; - - StackTraceElement[] trace = new Exception().getStackTrace(); - for (int i = 0; i < trace.length; i++) { - //System.out.println(trace[i].getClassName()); - if ("merge".equals(trace[i].getMethodName())) { - unicodeSortOrder = false; - if (LuceneTestCase.VERBOSE) { - System.out.println("NOTE: PreFlexRW codec: forcing legacy UTF16 vector term sort order"); - } - break; - } - } - - return unicodeSortOrder; - } - }; - } -} diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWTermVectorsWriter.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWTermVectorsWriter.java deleted file mode 100644 index 206cbe0..0000000 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWTermVectorsWriter.java +++ /dev/null @@ -1,221 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Comparator; - -import org.apache.lucene.codecs.TermVectorsWriter; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.store.DataInput; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.IOUtils; -import org.apache.lucene.util.StringHelper; - -final class PreFlexRWTermVectorsWriter extends TermVectorsWriter { - private final Directory directory; - private final String segment; - private IndexOutput tvx = null, tvd = null, tvf = null; - - public PreFlexRWTermVectorsWriter(Directory directory, String segment, IOContext context) throws IOException { - this.directory = directory; - this.segment = segment; - boolean success = false; - try { - // Open files for TermVector storage - tvx = directory.createOutput(IndexFileNames.segmentFileName(segment, "", Lucene3xTermVectorsReader.VECTORS_INDEX_EXTENSION), context); - tvx.writeInt(Lucene3xTermVectorsReader.FORMAT_CURRENT); - tvd = directory.createOutput(IndexFileNames.segmentFileName(segment, "", Lucene3xTermVectorsReader.VECTORS_DOCUMENTS_EXTENSION), context); - tvd.writeInt(Lucene3xTermVectorsReader.FORMAT_CURRENT); - tvf = directory.createOutput(IndexFileNames.segmentFileName(segment, "", Lucene3xTermVectorsReader.VECTORS_FIELDS_EXTENSION), context); - tvf.writeInt(Lucene3xTermVectorsReader.FORMAT_CURRENT); - success = true; - } finally { - if (!success) { - abort(); - } - } - } - - @Override - public void startDocument(int numVectorFields) throws IOException { - lastFieldName = null; - this.numVectorFields = numVectorFields; - tvx.writeLong(tvd.getFilePointer()); - tvx.writeLong(tvf.getFilePointer()); - tvd.writeVInt(numVectorFields); - fieldCount = 0; - fps = ArrayUtil.grow(fps, numVectorFields); - } - - private long fps[] = new long[10]; // pointers to the tvf before writing each field - private int fieldCount = 0; // number of fields we have written so far for this document - private int numVectorFields = 0; // total number of fields we will write for this document - private String lastFieldName; - - @Override - public void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets) throws IOException { - assert lastFieldName == null || info.name.compareTo(lastFieldName) > 0: "fieldName=" + info.name + " lastFieldName=" + lastFieldName; - lastFieldName = info.name; - this.positions = positions; - this.offsets = offsets; - lastTerm.length = 0; - fps[fieldCount++] = tvf.getFilePointer(); - tvd.writeVInt(info.number); - tvf.writeVInt(numTerms); - byte bits = 0x0; - if (positions) - bits |= Lucene3xTermVectorsReader.STORE_POSITIONS_WITH_TERMVECTOR; - if (offsets) - bits |= Lucene3xTermVectorsReader.STORE_OFFSET_WITH_TERMVECTOR; - tvf.writeByte(bits); - - assert fieldCount <= numVectorFields; - if (fieldCount == numVectorFields) { - // last field of the document - // this is crazy because the file format is crazy! - for (int i = 1; i < fieldCount; i++) { - tvd.writeVLong(fps[i] - fps[i-1]); - } - } - } - - private final BytesRef lastTerm = new BytesRef(10); - - // NOTE: we override addProx, so we don't need to buffer when indexing. - // we also don't buffer during bulk merges. - private int offsetStartBuffer[] = new int[10]; - private int offsetEndBuffer[] = new int[10]; - private int offsetIndex = 0; - private int offsetFreq = 0; - private boolean positions = false; - private boolean offsets = false; - - @Override - public void startTerm(BytesRef term, int freq) throws IOException { - final int prefix = StringHelper.bytesDifference(lastTerm, term); - final int suffix = term.length - prefix; - tvf.writeVInt(prefix); - tvf.writeVInt(suffix); - tvf.writeBytes(term.bytes, term.offset + prefix, suffix); - tvf.writeVInt(freq); - lastTerm.copyBytes(term); - lastPosition = lastOffset = 0; - - if (offsets && positions) { - // we might need to buffer if its a non-bulk merge - offsetStartBuffer = ArrayUtil.grow(offsetStartBuffer, freq); - offsetEndBuffer = ArrayUtil.grow(offsetEndBuffer, freq); - offsetIndex = 0; - offsetFreq = freq; - } - } - - int lastPosition = 0; - int lastOffset = 0; - - @Override - public void addProx(int numProx, DataInput positions, DataInput offsets) throws IOException { - // TODO: technically we could just copy bytes and not re-encode if we knew the length... - if (positions != null) { - for (int i = 0; i < numProx; i++) { - tvf.writeVInt(positions.readVInt()); - } - } - - if (offsets != null) { - for (int i = 0; i < numProx; i++) { - tvf.writeVInt(offsets.readVInt()); - tvf.writeVInt(offsets.readVInt()); - } - } - } - - @Override - public void addPosition(int position, int startOffset, int endOffset) throws IOException { - if (positions && offsets) { - // write position delta - tvf.writeVInt(position - lastPosition); - lastPosition = position; - - // buffer offsets - offsetStartBuffer[offsetIndex] = startOffset; - offsetEndBuffer[offsetIndex] = endOffset; - offsetIndex++; - - // dump buffer if we are done - if (offsetIndex == offsetFreq) { - for (int i = 0; i < offsetIndex; i++) { - tvf.writeVInt(offsetStartBuffer[i] - lastOffset); - tvf.writeVInt(offsetEndBuffer[i] - offsetStartBuffer[i]); - lastOffset = offsetEndBuffer[i]; - } - } - } else if (positions) { - // write position delta - tvf.writeVInt(position - lastPosition); - lastPosition = position; - } else if (offsets) { - // write offset deltas - tvf.writeVInt(startOffset - lastOffset); - tvf.writeVInt(endOffset - startOffset); - lastOffset = endOffset; - } - } - - @Override - public void abort() { - try { - close(); - } catch (IOException ignored) {} - IOUtils.deleteFilesIgnoringExceptions(directory, IndexFileNames.segmentFileName(segment, "", Lucene3xTermVectorsReader.VECTORS_INDEX_EXTENSION), - IndexFileNames.segmentFileName(segment, "", Lucene3xTermVectorsReader.VECTORS_DOCUMENTS_EXTENSION), - IndexFileNames.segmentFileName(segment, "", Lucene3xTermVectorsReader.VECTORS_FIELDS_EXTENSION)); - } - - @Override - public void finish(FieldInfos fis, int numDocs) throws IOException { - if (4+((long) numDocs)*16 != tvx.getFilePointer()) - // This is most likely a bug in Sun JRE 1.6.0_04/_05; - // we detect that the bug has struck, here, and - // throw an exception to prevent the corruption from - // entering the index. See LUCENE-1282 for - // details. - throw new RuntimeException("tvx size mismatch: mergedDocs is " + numDocs + " but tvx size is " + tvx.getFilePointer() + " file=" + tvx.toString() + "; now aborting this merge to prevent index corruption"); - } - - /** Close all streams. */ - @Override - public void close() throws IOException { - // make an effort to close all streams we can but remember and re-throw - // the first exception encountered in this process - IOUtils.close(tvx, tvd, tvf); - tvx = tvd = tvf = null; - } - - @Override - public Comparator getComparator() throws IOException { - return BytesRef.getUTF8SortedAsUTF16Comparator(); - } -} diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/TermInfosWriter.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/TermInfosWriter.java deleted file mode 100644 index d7cc07a..0000000 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/TermInfosWriter.java +++ /dev/null @@ -1,281 +0,0 @@ -package org.apache.lucene.codecs.lucene3x; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -import java.io.Closeable; -import java.io.IOException; - -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CharsRef; -import org.apache.lucene.util.IOUtils; -import org.apache.lucene.util.UnicodeUtil; - - -/** This stores a monotonically increasing set of pairs in a - Directory. A TermInfos can be written once, in order. */ - -final class TermInfosWriter implements Closeable { - /** The file format version, a negative number. */ - public static final int FORMAT = -3; - - // Changed strings to true utf8 with length-in-bytes not - // length-in-chars - public static final int FORMAT_VERSION_UTF8_LENGTH_IN_BYTES = -4; - - // NOTE: always change this if you switch to a new format! - public static final int FORMAT_CURRENT = FORMAT_VERSION_UTF8_LENGTH_IN_BYTES; - - private FieldInfos fieldInfos; - private IndexOutput output; - private TermInfo lastTi = new TermInfo(); - private long size; - - // TODO: the default values for these two parameters should be settable from - // IndexWriter. However, once that's done, folks will start setting them to - // ridiculous values and complaining that things don't work well, as with - // mergeFactor. So, let's wait until a number of folks find that alternate - // values work better. Note that both of these values are stored in the - // segment, so that it's safe to change these w/o rebuilding all indexes. - - /** Expert: The fraction of terms in the "dictionary" which should be stored - * in RAM. Smaller values use more memory, but make searching slightly - * faster, while larger values use less memory and make searching slightly - * slower. Searching is typically not dominated by dictionary lookup, so - * tweaking this is rarely useful.*/ - int indexInterval = 128; - - /** Expert: The fraction of {@link TermDocs} entries stored in skip tables, - * used to accelerate {@link TermDocs#skipTo(int)}. Larger values result in - * smaller indexes, greater acceleration, but fewer accelerable cases, while - * smaller values result in bigger indexes, less acceleration and more - * accelerable cases. More detailed experiments would be useful here. */ - int skipInterval = 16; - - /** Expert: The maximum number of skip levels. Smaller values result in - * slightly smaller indexes, but slower skipping in big posting lists. - */ - int maxSkipLevels = 10; - - private long lastIndexPointer; - private boolean isIndex; - private final BytesRef lastTerm = new BytesRef(); - private int lastFieldNumber = -1; - - private TermInfosWriter other; - - TermInfosWriter(Directory directory, String segment, FieldInfos fis, - int interval) - throws IOException { - initialize(directory, segment, fis, interval, false); - boolean success = false; - try { - other = new TermInfosWriter(directory, segment, fis, interval, true); - other.other = this; - success = true; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(output); - - try { - directory.deleteFile(IndexFileNames.segmentFileName(segment, "", - (isIndex ? Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION - : Lucene3xPostingsFormat.TERMS_EXTENSION))); - } catch (IOException ignored) { - } - } - } - } - - private TermInfosWriter(Directory directory, String segment, FieldInfos fis, - int interval, boolean isIndex) throws IOException { - initialize(directory, segment, fis, interval, isIndex); - } - - private void initialize(Directory directory, String segment, FieldInfos fis, - int interval, boolean isi) throws IOException { - indexInterval = interval; - fieldInfos = fis; - isIndex = isi; - output = directory.createOutput(IndexFileNames.segmentFileName(segment, "", - (isIndex ? Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION - : Lucene3xPostingsFormat.TERMS_EXTENSION)), IOContext.DEFAULT); - boolean success = false; - try { - output.writeInt(FORMAT_CURRENT); // write format - output.writeLong(0); // leave space for size - output.writeInt(indexInterval); // write indexInterval - output.writeInt(skipInterval); // write skipInterval - output.writeInt(maxSkipLevels); // write maxSkipLevels - assert initUTF16Results(); - success = true; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(output); - - try { - directory.deleteFile(IndexFileNames.segmentFileName(segment, "", - (isIndex ? Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION - : Lucene3xPostingsFormat.TERMS_EXTENSION))); - } catch (IOException ignored) { - } - } - } - } - - // Currently used only by assert statements - CharsRef utf16Result1; - CharsRef utf16Result2; - private final BytesRef scratchBytes = new BytesRef(); - - // Currently used only by assert statements - private boolean initUTF16Results() { - utf16Result1 = new CharsRef(10); - utf16Result2 = new CharsRef(10); - return true; - } - - /** note: -1 is the empty field: "" !!!! */ - static String fieldName(FieldInfos infos, int fieldNumber) { - FieldInfo fi = infos.fieldInfo(fieldNumber); - return (fi != null) ? fi.name : ""; - } - - // Currently used only by assert statement - private int compareToLastTerm(int fieldNumber, BytesRef term) { - - if (lastFieldNumber != fieldNumber) { - final int cmp = fieldName(fieldInfos, lastFieldNumber).compareTo(fieldName(fieldInfos, fieldNumber)); - // If there is a field named "" (empty string) then we - // will get 0 on this comparison, yet, it's "OK". But - // it's not OK if two different field numbers map to - // the same name. - if (cmp != 0 || lastFieldNumber != -1) - return cmp; - } - - scratchBytes.copyBytes(term); - assert lastTerm.offset == 0; - UnicodeUtil.UTF8toUTF16(lastTerm.bytes, 0, lastTerm.length, utf16Result1); - - assert scratchBytes.offset == 0; - UnicodeUtil.UTF8toUTF16(scratchBytes.bytes, 0, scratchBytes.length, utf16Result2); - - final int len; - if (utf16Result1.length < utf16Result2.length) - len = utf16Result1.length; - else - len = utf16Result2.length; - - for(int i=0;i, TermInfo> pair to the set. - Term must be lexicographically greater than all previous Terms added. - TermInfo pointers must be positive and greater than all previous.*/ - public void add(int fieldNumber, BytesRef term, TermInfo ti) - throws IOException { - - assert compareToLastTerm(fieldNumber, term) < 0 || - (isIndex && term.length == 0 && lastTerm.length == 0) : - "Terms are out of order: field=" + fieldName(fieldInfos, fieldNumber) + " (number " + fieldNumber + ")" + - " lastField=" + fieldName(fieldInfos, lastFieldNumber) + " (number " + lastFieldNumber + ")" + - " text=" + term.utf8ToString() + " lastText=" + lastTerm.utf8ToString(); - - assert ti.freqPointer >= lastTi.freqPointer: "freqPointer out of order (" + ti.freqPointer + " < " + lastTi.freqPointer + ")"; - assert ti.proxPointer >= lastTi.proxPointer: "proxPointer out of order (" + ti.proxPointer + " < " + lastTi.proxPointer + ")"; - - if (!isIndex && size % indexInterval == 0) { - other.add(lastFieldNumber, lastTerm, lastTi); // add an index term - } - writeTerm(fieldNumber, term); // write term - - output.writeVInt(ti.docFreq); // write doc freq - output.writeVLong(ti.freqPointer - lastTi.freqPointer); // write pointers - output.writeVLong(ti.proxPointer - lastTi.proxPointer); - - if (ti.docFreq >= skipInterval) { - output.writeVInt(ti.skipOffset); - } - - if (isIndex) { - output.writeVLong(other.output.getFilePointer() - lastIndexPointer); - lastIndexPointer = other.output.getFilePointer(); // write pointer - } - - lastFieldNumber = fieldNumber; - lastTi.set(ti); - size++; - } - - private void writeTerm(int fieldNumber, BytesRef term) - throws IOException { - - //System.out.println(" tiw.write field=" + fieldNumber + " term=" + term.utf8ToString()); - - // TODO: UTF16toUTF8 could tell us this prefix - // Compute prefix in common with last term: - int start = 0; - final int limit = term.length < lastTerm.length ? term.length : lastTerm.length; - while(start < limit) { - if (term.bytes[start+term.offset] != lastTerm.bytes[start+lastTerm.offset]) - break; - start++; - } - - final int length = term.length - start; - output.writeVInt(start); // write shared prefix length - output.writeVInt(length); // write delta length - output.writeBytes(term.bytes, start+term.offset, length); // write delta bytes - output.writeVInt(fieldNumber); // write field num - lastTerm.copyBytes(term); - } - - /** Called to complete TermInfos creation. */ - public void close() throws IOException { - try { - output.seek(4); // write size after format - output.writeLong(size); - } finally { - try { - output.close(); - } finally { - if (!isIndex) { - other.close(); - } - } - } - } -} diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/package.html b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/package.html deleted file mode 100644 index 1d7d8fa..0000000 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/package.html +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - -Support for generating test indexes in the Lucene 3.x index format. -

      -NOTE: This is not a perfect simulation of the 3.x format, but its close. -Particularly, indexes generated with this codec cannot actually be read -with Lucene 3.x -

      - - diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40ords/Lucene40WithOrds.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40ords/Lucene40WithOrds.java index 9b5c69e..5f34483 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40ords/Lucene40WithOrds.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40ords/Lucene40WithOrds.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.lucene40ords; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java index 298c140..8a0e9e1 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.mockintblock; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -84,7 +84,7 @@ public class MockFixedIntBlockPostingsFormat extends PostingsFormat { return new FixedIntBlockIndexInput(dir.openInput(fileName, context)) { @Override - protected BlockReader getBlockReader(final IndexInput in, final int[] buffer) throws IOException { + protected BlockReader getBlockReader(final IndexInput in, final int[] buffer) { return new BlockReader() { public void seek(long pos) {} public void readBlock() throws IOException { diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java index 42a93f0..8f56956 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.mockintblock; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -85,7 +85,7 @@ public class MockVariableIntBlockPostingsFormat extends PostingsFormat { return new VariableIntBlockIndexInput(in) { @Override - protected BlockReader getBlockReader(final IndexInput in, final int[] buffer) throws IOException { + protected BlockReader getBlockReader(final IndexInput in, final int[] buffer) { return new BlockReader() { public void seek(long pos) {} public int readBlock() throws IOException { diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java index 5d3f804..6ed10f0 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.mockrandom; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSepDocValuesFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSepDocValuesFormat.java index 0799879..f183046 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSepDocValuesFormat.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSepDocValuesFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.mocksep; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSepPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSepPostingsFormat.java index 4cb6d50..45ce698 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSepPostingsFormat.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSepPostingsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.mocksep; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntFactory.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntFactory.java index b37df0d..8c48f1f 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntFactory.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntFactory.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.mocksep; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntIndexInput.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntIndexInput.java index 4debe89..924ba17 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntIndexInput.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntIndexInput.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.mocksep; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,12 +19,12 @@ package org.apache.lucene.codecs.mocksep; import java.io.IOException; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.sep.IntIndexInput; import org.apache.lucene.store.DataInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.CodecUtil; /** Reads IndexInputs written with {@link * MockSingleIntIndexOutput}. NOTE: this class is just for diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntIndexOutput.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntIndexOutput.java index 86d2f1a..a7fad99 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntIndexOutput.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntIndexOutput.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.mocksep; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,8 +20,8 @@ package org.apache.lucene.codecs.mocksep; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.IOUtils; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.sep.IntIndexOutput; import java.io.IOException; diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java index 7d72714..9cbe709 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.nestedpulsing; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -32,6 +32,7 @@ import org.apache.lucene.codecs.pulsing.PulsingPostingsReader; import org.apache.lucene.codecs.pulsing.PulsingPostingsWriter; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.util.IOUtils; /** * Pulsing(1, Pulsing(2, Lucene40)) @@ -47,32 +48,38 @@ public class NestedPulsingPostingsFormat extends PostingsFormat { @Override public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - PostingsWriterBase docsWriter = new Lucene40PostingsWriter(state); - - PostingsWriterBase pulsingWriterInner = new PulsingPostingsWriter(2, docsWriter); - PostingsWriterBase pulsingWriter = new PulsingPostingsWriter(1, pulsingWriterInner); + PostingsWriterBase docsWriter = null; + PostingsWriterBase pulsingWriterInner = null; + PostingsWriterBase pulsingWriter = null; // Terms dict boolean success = false; try { + docsWriter = new Lucene40PostingsWriter(state); + + pulsingWriterInner = new PulsingPostingsWriter(2, docsWriter); + pulsingWriter = new PulsingPostingsWriter(1, pulsingWriterInner); FieldsConsumer ret = new BlockTreeTermsWriter(state, pulsingWriter, BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE); success = true; return ret; } finally { if (!success) { - pulsingWriter.close(); + IOUtils.closeWhileHandlingException(docsWriter, pulsingWriterInner, pulsingWriter); } } } @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase docsReader = new Lucene40PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix); - PostingsReaderBase pulsingReaderInner = new PulsingPostingsReader(docsReader); - PostingsReaderBase pulsingReader = new PulsingPostingsReader(pulsingReaderInner); + PostingsReaderBase docsReader = null; + PostingsReaderBase pulsingReaderInner = null; + PostingsReaderBase pulsingReader = null; boolean success = false; try { + docsReader = new Lucene40PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix); + pulsingReaderInner = new PulsingPostingsReader(docsReader); + pulsingReader = new PulsingPostingsReader(pulsingReaderInner); FieldsProducer ret = new BlockTreeTermsReader( state.dir, state.fieldInfos, state.segmentInfo.name, pulsingReader, @@ -83,7 +90,7 @@ public class NestedPulsingPostingsFormat extends PostingsFormat { return ret; } finally { if (!success) { - pulsingReader.close(); + IOUtils.closeWhileHandlingException(docsReader, pulsingReaderInner, pulsingReader); } } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java index 74ebf16..cbe9c65 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java @@ -1,6 +1,6 @@ package org.apache.lucene.codecs.ramonly; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -28,6 +28,7 @@ import java.util.SortedMap; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; import org.apache.lucene.codecs.PostingsConsumer; @@ -48,7 +49,6 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.IOUtils; /** Stores all postings data in RAM, but writes a small diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java b/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java index ced221d..2e0c3e8 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -78,7 +78,7 @@ class DocHelper { public static final String KEYWORD_FIELD_KEY = "keyField"; public static Field keyField; static { - keyField = new Field(KEYWORD_FIELD_KEY, KEYWORD_TEXT, StringField.TYPE_STORED); + keyField = new StringField(KEYWORD_FIELD_KEY, KEYWORD_TEXT, Field.Store.YES); } public static final FieldType customType5; @@ -115,14 +115,14 @@ class DocHelper { public static final String UNSTORED_1_FIELD_TEXT = "unstored field text"; public static final String UNSTORED_FIELD_1_KEY = "unStoredField1"; - public static Field unStoredField1 = new Field(UNSTORED_FIELD_1_KEY, UNSTORED_1_FIELD_TEXT, TextField.TYPE_UNSTORED); + public static Field unStoredField1 = new TextField(UNSTORED_FIELD_1_KEY, UNSTORED_1_FIELD_TEXT, Field.Store.NO); public static final FieldType customType8; public static final String UNSTORED_2_FIELD_TEXT = "unstored field text"; public static final String UNSTORED_FIELD_2_KEY = "unStoredField2"; public static Field unStoredField2; static { - customType8 = new FieldType(TextField.TYPE_UNSTORED); + customType8 = new FieldType(TextField.TYPE_NOT_STORED); customType8.setStoreTermVectors(true); unStoredField2 = new Field(UNSTORED_FIELD_2_KEY, UNSTORED_2_FIELD_TEXT, customType8); } diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java index 737bc0f..f7f4cd8 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,8 +21,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Set; -import org.apache.lucene.index.FilterAtomicReader; - public final class FieldFilterAtomicReader extends FilterAtomicReader { private final Set fields; @@ -63,7 +61,7 @@ public final class FieldFilterAtomicReader extends FilterAtomicReader { } @Override - public void document(final int docID, final StoredFieldVisitor visitor) throws CorruptIndexException, IOException { + public void document(final int docID, final StoredFieldVisitor visitor) throws IOException { super.document(docID, new StoredFieldVisitor() { @Override public void binaryField(FieldInfo fieldInfo, byte[] value, int offset, int length) throws IOException { diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/MockIndexInput.java b/lucene/test-framework/src/java/org/apache/lucene/index/MockIndexInput.java index 87fa6bc..4b847c7 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/MockIndexInput.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/MockIndexInput.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java b/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java index fdfc224..1890501 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -60,7 +60,7 @@ public class MockRandomMergePolicy extends MergePolicy { @Override public MergeSpecification findForcedMerges( SegmentInfos segmentInfos, int maxSegmentCount, Map segmentsToMerge) - throws CorruptIndexException, IOException { + throws IOException { final List eligibleSegments = new ArrayList(); for(SegmentInfoPerCommit info : segmentInfos) { @@ -96,9 +96,7 @@ public class MockRandomMergePolicy extends MergePolicy { } @Override - public MergeSpecification findForcedDeletesMerges( - SegmentInfos segmentInfos) - throws CorruptIndexException, IOException { + public MergeSpecification findForcedDeletesMerges(SegmentInfos segmentInfos) throws IOException { return findMerges(segmentInfos); } diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java b/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java index 1a8f1a5..6224309 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,6 +20,7 @@ package org.apache.lucene.index; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -52,6 +53,9 @@ import org.apache.lucene.util._TestUtil; public class RandomCodec extends Lucene40Codec { /** Shuffled list of postings formats to use for new mappings */ private List formats = new ArrayList(); + + /** unique set of format names this codec knows about */ + public Set formatNames = new HashSet(); /** memorized field->postingsformat mappings */ // note: we have to sync this map even though its just for debugging/toString, @@ -83,12 +87,11 @@ public class RandomCodec extends Lucene40Codec { int minItemsPerBlock = _TestUtil.nextInt(random, 2, 100); int maxItemsPerBlock = 2*(Math.max(2, minItemsPerBlock-1)) + random.nextInt(100); - // TODO: make it possible to specify min/max iterms per block via CL: - minItemsPerBlock = _TestUtil.nextInt(random, 2, 100); - maxItemsPerBlock = 2*(Math.max(1, minItemsPerBlock-1)) + random.nextInt(100); add(avoidCodecs, new Lucene40PostingsFormat(minItemsPerBlock, maxItemsPerBlock), new Pulsing40PostingsFormat(1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock), + // add pulsing again with (usually) different parameters + new Pulsing40PostingsFormat(1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock), new MockSepPostingsFormat(), new MockFixedIntBlockPostingsFormat(_TestUtil.nextInt(random, 1, 2000)), new MockVariableIntBlockPostingsFormat( _TestUtil.nextInt(random, 1, 127)), @@ -96,9 +99,13 @@ public class RandomCodec extends Lucene40Codec { new NestedPulsingPostingsFormat(), new Lucene40WithOrds(), new SimpleTextPostingsFormat(), - new MemoryPostingsFormat(random.nextBoolean())); + new MemoryPostingsFormat(true, random.nextFloat()), + new MemoryPostingsFormat(false, random.nextFloat())); Collections.shuffle(formats, random); + + // Avoid too many open files: + formats.subList(4, formats.size()).clear(); } public RandomCodec(Random random) { @@ -109,6 +116,7 @@ public class RandomCodec extends Lucene40Codec { for (PostingsFormat p : postings) { if (!avoidCodecs.contains(p.getName())) { formats.add(p); + formatNames.add(p.getName()); } } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/RandomDocumentsWriterPerThreadPool.java b/lucene/test-framework/src/java/org/apache/lucene/index/RandomDocumentsWriterPerThreadPool.java index ba782c7..e465032 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/RandomDocumentsWriterPerThreadPool.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/RandomDocumentsWriterPerThreadPool.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -25,8 +25,7 @@ import java.util.Random; * @lucene.internal * @lucene.experimental */ -public class RandomDocumentsWriterPerThreadPool extends - DocumentsWriterPerThreadPool { +class RandomDocumentsWriterPerThreadPool extends DocumentsWriterPerThreadPool { private final ThreadState[] states; private final Random random; private final int maxRetry; diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java b/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java index 5220d4d..5629c0a 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -206,7 +206,7 @@ public class RandomIndexWriter implements Closeable { DocValues.Type[] values = DocValues.Type.values(); DocValues.Type type = values[r.nextInt(values.length)]; String name = "random_" + type.name() + "" + docValuesFieldPrefix; - if ("Lucene3x".equals(codec.getName()) || doc.getField(name) != null) { + if (doc.getField(name) != null) { return; } final Field f; @@ -326,28 +326,28 @@ public class RandomIndexWriter implements Closeable { maybeCommit(); } - public void addIndexes(Directory... dirs) throws CorruptIndexException, IOException { + public void addIndexes(Directory... dirs) throws IOException { w.addIndexes(dirs); } - public void addIndexes(IndexReader... readers) throws CorruptIndexException, IOException { + public void addIndexes(IndexReader... readers) throws IOException { w.addIndexes(readers); } - public void deleteDocuments(Term term) throws CorruptIndexException, IOException { + public void deleteDocuments(Term term) throws IOException { w.deleteDocuments(term); } - public void deleteDocuments(Query q) throws CorruptIndexException, IOException { + public void deleteDocuments(Query q) throws IOException { w.deleteDocuments(q); } - public void commit() throws CorruptIndexException, IOException { + public void commit() throws IOException { w.commit(); switchDoDocValues(); } - public int numDocs() throws IOException { + public int numDocs() { return w.numDocs(); } @@ -403,10 +403,7 @@ public class RandomIndexWriter implements Closeable { if (r.nextInt(20) == 2) { doRandomForceMerge(); } - // If we are writing with PreFlexRW, force a full - // IndexReader.open so terms are sorted in codepoint - // order during searching: - if (!applyDeletions || !codec.getName().equals("Lucene3x") && r.nextBoolean()) { + if (!applyDeletions || r.nextBoolean()) { if (LuceneTestCase.VERBOSE) { System.out.println("RIW.getReader: use NRT reader"); } @@ -421,7 +418,7 @@ public class RandomIndexWriter implements Closeable { w.commit(); switchDoDocValues(); if (r.nextBoolean()) { - return IndexReader.open(w.getDirectory(), _TestUtil.nextInt(r, 1, 10)); + return DirectoryReader.open(w.getDirectory(), _TestUtil.nextInt(r, 1, 10)); } else { return w.getReader(applyDeletions); } diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java index 164b204..69f946e 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java @@ -1,6 +1,6 @@ package org.apache.lucene.index; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -29,8 +29,6 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; @@ -119,8 +117,7 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas final long stopTime, final Set delIDs, final Set delPackIDs, - final List allSubDocs) - throws Exception { + final List allSubDocs) { final Thread[] threads = new Thread[numThreads]; for(int thread=0;thread docIDs = new ArrayList(); final SubDocs subDocs = new SubDocs(packID, docIDs); final List docsList = new ArrayList(); @@ -334,6 +331,20 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas try { final IndexSearcher s = getCurrentSearcher(); try { + // Verify 1) IW is correctly setting + // diagnostics, and 2) segment warming for + // merged segments is actually happening: + for(AtomicReader sub : ((DirectoryReader) s.getIndexReader()).getSequentialSubReaders()) { + SegmentReader segReader = (SegmentReader) sub; + Map diagnostics = segReader.getSegmentInfo().info.getDiagnostics(); + assertNotNull(diagnostics); + String source = diagnostics.get("source"); + assertNotNull(source); + if (source.equals("merge")) { + assertTrue("sub reader " + sub + " wasn't warmed: warmed=" + warmed + " diagnostics=" + diagnostics + " si=" + segReader.getSegmentInfo(), + !assertMergedSegmentsWarmed || warmed.containsKey(((SegmentReader) sub).core)); + } + } if (s.getIndexReader().numDocs() > 0) { smokeTestSearcher(s); Fields fields = MultiFields.getFields(s.getIndexReader()); @@ -405,6 +416,10 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas protected void doClose() throws Exception { } + protected boolean assertMergedSegmentsWarmed = true; + + private final Map warmed = Collections.synchronizedMap(new WeakHashMap()); + public void runTest(String testName) throws Exception { failed.set(false); @@ -415,7 +430,7 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas final long t0 = System.currentTimeMillis(); Random random = new Random(random().nextLong()); - final LineFileDocs docs = new LineFileDocs(random, defaultCodecSupportsDocValues()); + final LineFileDocs docs = new LineFileDocs(random, true); final File tempDir = _TestUtil.getTempDir(testName); dir = newFSDirectory(tempDir); ((MockDirectoryWrapper) dir).setCheckIndexOnClose(false); // don't double-checkIndex, we do it ourselves. @@ -442,6 +457,7 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas if (VERBOSE) { System.out.println("TEST: now warm merged reader=" + reader); } + warmed.put(((SegmentReader) reader).core, Boolean.TRUE); final int maxDoc = reader.maxDoc(); final Bits liveDocs = reader.getLiveDocs(); int sum = 0; @@ -620,12 +636,10 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas private int runQuery(IndexSearcher s, Query q) throws Exception { s.search(q, 10); int hitCount = s.search(q, null, 10, new Sort(new SortField("title", SortField.Type.STRING))).totalHits; - if (defaultCodecSupportsDocValues()) { - final Sort dvSort = new Sort(new SortField("title", SortField.Type.STRING)); - dvSort.getSort()[0].setUseIndexValues(true); - int hitCount2 = s.search(q, null, 10, dvSort).totalHits; - assertEquals(hitCount, hitCount2); - } + final Sort dvSort = new Sort(new SortField("title", SortField.Type.STRING)); + dvSort.getSort()[0].setUseIndexValues(true); + int hitCount2 = s.search(q, null, 10, dvSort).totalHits; + assertEquals(hitCount, hitCount2); return hitCount; } diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java index 0c048b7..2f29eea 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -91,7 +91,7 @@ public class AssertingIndexSearcher extends IndexSearcher { } @Override - public float getValueForNormalization() throws IOException { + public float getValueForNormalization() { throw new IllegalStateException("Weight already normalized."); } diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java b/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java index adab4d7..70a4a98 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -27,7 +27,6 @@ import junit.framework.Assert; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; /** @@ -185,8 +184,7 @@ public class CheckHits { } /** Tests that a Hits has an expected order of documents */ - public static void checkDocIds(String mes, int[] results, ScoreDoc[] hits) - throws IOException { + public static void checkDocIds(String mes, int[] results, ScoreDoc[] hits) { Assert.assertEquals(mes + " nr of hits", hits.length, results.length); for (int i = 0; i < results.length; i++) { Assert.assertEquals(mes + " doc nrs for hit " + i, results[i], hits[i].doc); @@ -200,15 +198,14 @@ public class CheckHits { Query query, ScoreDoc[] hits1, ScoreDoc[] hits2, - int[] results) - throws IOException { + int[] results) { checkDocIds("hits1", results, hits1); checkDocIds("hits2", results, hits2); checkEqual(query, hits1, hits2); } - public static void checkEqual(Query query, ScoreDoc[] hits1, ScoreDoc[] hits2) throws IOException { + public static void checkEqual(Query query, ScoreDoc[] hits1, ScoreDoc[] hits2) { final float scoreTolerance = 1.0e-6f; if (hits1.length != hits2.length) { Assert.fail("Unequal lengths: hits1="+hits1.length+",hits2="+hits2.length); @@ -231,7 +228,7 @@ public class CheckHits { } } - public static String hits2str(ScoreDoc[] hits1, ScoreDoc[] hits2, int start, int end) throws IOException { + public static String hits2str(ScoreDoc[] hits1, ScoreDoc[] hits2, int start, int end) { StringBuilder sb = new StringBuilder(); int len1=hits1==null ? 0 : hits1.length; int len2=hits2==null ? 0 : hits2.length; @@ -422,7 +419,7 @@ public class CheckHits { * @see ExplanationAsserter */ public static class ExplanationAssertingSearcher extends IndexSearcher { - public ExplanationAssertingSearcher(IndexReader r) throws IOException { + public ExplanationAssertingSearcher(IndexReader r) { super(r); } protected void checkExplanations(Query q) throws IOException { diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java b/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java index 64c42ce..9142978 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,6 +18,7 @@ package org.apache.lucene.search; */ import java.io.IOException; +import java.util.List; import java.util.Random; import junit.framework.Assert; @@ -140,7 +141,7 @@ public class QueryUtils { public static class FCInvisibleMultiReader extends MultiReader { private final Object cacheKey = new Object(); - public FCInvisibleMultiReader(IndexReader... readers) throws IOException { + public FCInvisibleMultiReader(IndexReader... readers) { super(readers); } @@ -233,7 +234,7 @@ public class QueryUtils { */ public static void checkSkipTo(final Query q, final IndexSearcher s) throws IOException { //System.out.println("Checking "+q); - final AtomicReaderContext[] readerContextArray = s.getTopReaderContext().leaves(); + final List readerContextArray = s.getTopReaderContext().leaves(); if (s.createNormalizedWeight(q).scoresDocsOutOfOrder()) return; // in this case order of skipTo() might differ from that of next(). final int skip_op = 0; @@ -267,7 +268,7 @@ public class QueryUtils { private int leafPtr; @Override - public void setScorer(Scorer scorer) throws IOException { + public void setScorer(Scorer scorer) { this.sc = scorer; } @@ -278,7 +279,7 @@ public class QueryUtils { try { if (scorer == null) { Weight w = s.createNormalizedWeight(q); - AtomicReaderContext context = readerContextArray[leafPtr]; + AtomicReaderContext context = readerContextArray.get(leafPtr); scorer = w.scorer(context, true, false, context.reader().getLiveDocs()); } @@ -334,7 +335,7 @@ public class QueryUtils { leafPtr++; } lastReader[0] = context.reader(); - assert readerContextArray[leafPtr].reader() == context.reader(); + assert readerContextArray.get(leafPtr).reader() == context.reader(); this.scorer = null; lastDoc[0] = -1; } @@ -368,13 +369,13 @@ public class QueryUtils { final float maxDiff = 1e-3f; final int lastDoc[] = {-1}; final AtomicReader lastReader[] = {null}; - final AtomicReaderContext[] context = s.getTopReaderContext().leaves(); + final List context = s.getTopReaderContext().leaves(); s.search(q,new Collector() { private Scorer scorer; private int leafPtr; private Bits liveDocs; @Override - public void setScorer(Scorer scorer) throws IOException { + public void setScorer(Scorer scorer) { this.scorer = scorer; } @Override @@ -384,7 +385,7 @@ public class QueryUtils { long startMS = System.currentTimeMillis(); for (int i=lastDoc[0]+1; i<=doc; i++) { Weight w = s.createNormalizedWeight(q); - Scorer scorer = w.scorer(context[leafPtr], true, false, liveDocs); + Scorer scorer = w.scorer(context.get(leafPtr), true, false, liveDocs); Assert.assertTrue("query collected "+doc+" but skipTo("+i+") says no more docs!",scorer.advance(i) != DocIdSetIterator.NO_MORE_DOCS); Assert.assertEquals("query collected "+doc+" but skipTo("+i+") got to "+scorer.docID(),doc,scorer.docID()); float skipToScore = scorer.score(); diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/RandomSimilarityProvider.java b/lucene/test-framework/src/java/org/apache/lucene/search/RandomSimilarityProvider.java index f2d2b1a..dd10c3b 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/RandomSimilarityProvider.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/RandomSimilarityProvider.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java b/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java index 9d798e6..6eb5c39 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -62,8 +62,8 @@ public abstract class SearchEquivalenceTestBase extends LuceneTestCase { analyzer = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false, stopset, true); RandomIndexWriter iw = new RandomIndexWriter(random, directory, analyzer); Document doc = new Document(); - Field id = new StringField("id", ""); - Field field = new TextField("field", ""); + Field id = new StringField("id", "", Field.Store.NO); + Field field = new TextField("field", "", Field.Store.NO); doc.add(id); doc.add(field); diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java b/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java index 21d6b98..004b379 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -31,11 +31,11 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.TermContext; // TODO // - doc blocks? so we can test joins/grouping... @@ -518,7 +518,7 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase { @Override public void run() { try { - final LineFileDocs docs = new LineFileDocs(random(), defaultCodecSupportsDocValues()); + final LineFileDocs docs = new LineFileDocs(random(), true); int numDocs = 0; while (System.nanoTime() < endTimeNanos) { final int what = random().nextInt(3); diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java index 0673566..c8e8cf8 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -282,7 +282,7 @@ public class MockDirectoryWrapper extends Directory { } } - public synchronized void clearCrash() throws IOException { + public synchronized void clearCrash() { crashed = false; openLocks.clear(); } @@ -617,7 +617,7 @@ public class MockDirectoryWrapper extends Directory { * checkIndex() or not. It might mask real problems, where we silently * don't checkindex at all. instead we look for a segments file. */ - private boolean indexPossiblyExists(Directory d) throws IOException { + private boolean indexPossiblyExists(Directory d) { String files[]; try { files = d.listAll(); diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexInputWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexInputWrapper.java index f175330..24f2e89 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexInputWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexInputWrapper.java @@ -3,7 +3,7 @@ package org.apache.lucene.store; import java.io.IOException; import java.util.Map; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java index 1b9b1ad..dcf910f 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/MockLockFactoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/MockLockFactoryWrapper.java index e056a1f..0795cbe 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/MockLockFactoryWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/MockLockFactoryWrapper.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/_TestHelper.java b/lucene/test-framework/src/java/org/apache/lucene/store/_TestHelper.java index fb90a87..c3789e0 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/_TestHelper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/_TestHelper.java @@ -1,6 +1,6 @@ package org.apache.lucene.store; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/AbstractBeforeAfterRule.java b/lucene/test-framework/src/java/org/apache/lucene/util/AbstractBeforeAfterRule.java index 141892c..45b598a 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/AbstractBeforeAfterRule.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/AbstractBeforeAfterRule.java @@ -10,7 +10,7 @@ import org.junit.runner.Description; import org.junit.runners.model.MultipleFailureException; import org.junit.runners.model.Statement; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -38,10 +38,10 @@ abstract class AbstractBeforeAfterRule implements TestRule { public Statement apply(final Statement s, final Description d) { return new Statement() { public void evaluate() throws Throwable { - before(); - final ArrayList errors = new ArrayList(); + try { + before(); s.evaluate(); } catch (Throwable t) { errors.add(t); diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/CloseableDirectory.java b/lucene/test-framework/src/java/org/apache/lucene/util/CloseableDirectory.java index 5420eca..11bfabc 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/CloseableDirectory.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/CloseableDirectory.java @@ -1,12 +1,11 @@ package org.apache.lucene.util; import java.io.Closeable; -import java.io.IOException; import org.apache.lucene.store.MockDirectoryWrapper; import org.junit.Assert; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -39,7 +38,7 @@ final class CloseableDirectory implements Closeable { } @Override - public void close() throws IOException { + public void close() { // We only attempt to check open/closed state if there were no other test // failures. try { diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/CloseableFile.java b/lucene/test-framework/src/java/org/apache/lucene/util/CloseableFile.java index 9ae58d6..aa44cfb 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/CloseableFile.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/CloseableFile.java @@ -2,7 +2,7 @@ package org.apache.lucene.util; import java.io.*; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/English.java b/lucene/test-framework/src/java/org/apache/lucene/util/English.java index 21dc26c..1f1766f 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/English.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/English.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/FailOnNonBulkMergesInfoStream.java b/lucene/test-framework/src/java/org/apache/lucene/util/FailOnNonBulkMergesInfoStream.java index 086157f..a9b8c35 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/FailOnNonBulkMergesInfoStream.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/FailOnNonBulkMergesInfoStream.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java b/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java index 6b3ee26..227cb40 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -162,7 +162,7 @@ public class LineFileDocs implements Closeable { public DocState(boolean useDocValues) { doc = new Document(); - title = new StringField("title", ""); + title = new StringField("title", "", Field.Store.NO); doc.add(title); FieldType ft = new FieldType(TextField.TYPE_STORED); @@ -176,10 +176,10 @@ public class LineFileDocs implements Closeable { body = new Field("body", "", ft); doc.add(body); - id = new Field("docid", "", StringField.TYPE_STORED); + id = new StringField("docid", "", Field.Store.YES); doc.add(id); - date = new Field("date", "", StringField.TYPE_STORED); + date = new StringField("date", "", Field.Store.YES); doc.add(date); if (useDocValues) { diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneJUnit3MethodProvider.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneJUnit3MethodProvider.java index 6f3df6b..15518fb 100755 --- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneJUnit3MethodProvider.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneJUnit3MethodProvider.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java index ec6497e..0a4a8b7 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,19 +17,20 @@ package org.apache.lucene.util; * limitations under the License. */ -import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsBoolean; -import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsInt; - import java.io.*; import java.lang.annotation.*; +import java.lang.reflect.Constructor; import java.lang.reflect.Method; import java.util.*; import java.util.concurrent.*; +import java.util.logging.Logger; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; import org.apache.lucene.index.*; import org.apache.lucene.index.IndexReader.ReaderClosedListener; import org.apache.lucene.search.*; @@ -42,12 +43,14 @@ import org.junit.*; import org.junit.rules.RuleChain; import org.junit.rules.TestRule; import org.junit.runner.RunWith; - import com.carrotsearch.randomizedtesting.*; import com.carrotsearch.randomizedtesting.annotations.*; import com.carrotsearch.randomizedtesting.generators.RandomPicks; import com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule; +import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsBoolean; +import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsInt; + /** * Base class for all Lucene unit tests, Junit3 or Junit4 variant. * @@ -113,9 +116,20 @@ import com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule; @ThreadLeaks(failTestIfLeaking = false) public abstract class LuceneTestCase extends Assert { - // ----------------------------------------------------------------- - // Test groups and other annotations modifying tests' behavior. - // ----------------------------------------------------------------- + // -------------------------------------------------------------------- + // Test groups, system properties and other annotations modifying tests + // -------------------------------------------------------------------- + + public static final String SYSPROP_NIGHTLY = "tests.nightly"; + public static final String SYSPROP_WEEKLY = "tests.weekly"; + public static final String SYSPROP_AWAITSFIX = "tests.awaitsfix"; + public static final String SYSPROP_SLOW = "tests.slow"; + + /** @see #ignoreAfterMaxFailures*/ + private static final String SYSPROP_MAXFAILURES = "tests.maxfailures"; + + /** @see #ignoreAfterMaxFailures*/ + private static final String SYSPROP_FAILFAST = "tests.failfast"; /** * Annotation for tests that should only be run during nightly builds. @@ -123,7 +137,7 @@ public abstract class LuceneTestCase extends Assert { @Documented @Inherited @Retention(RetentionPolicy.RUNTIME) - @TestGroup(enabled = false, sysProperty = "tests.nightly") + @TestGroup(enabled = false, sysProperty = SYSPROP_NIGHTLY) public @interface Nightly {} /** @@ -132,7 +146,7 @@ public abstract class LuceneTestCase extends Assert { @Documented @Inherited @Retention(RetentionPolicy.RUNTIME) - @TestGroup(enabled = false, sysProperty = "tests.weekly") + @TestGroup(enabled = false, sysProperty = SYSPROP_WEEKLY) public @interface Weekly {} /** @@ -141,22 +155,22 @@ public abstract class LuceneTestCase extends Assert { @Documented @Inherited @Retention(RetentionPolicy.RUNTIME) - @TestGroup(enabled = false, sysProperty = "tests.awaitsfix") + @TestGroup(enabled = false, sysProperty = SYSPROP_AWAITSFIX) public @interface AwaitsFix { /** Point to JIRA entry. */ public String bugUrl(); } /** - * Annotation for tests that are really slow and should be run only when specifically - * asked to run. + * Annotation for tests that are slow. Slow tests do run by default but can be + * disabled if a quick run is needed. */ @Documented @Inherited @Retention(RetentionPolicy.RUNTIME) - @TestGroup(enabled = false, sysProperty = "tests.slow") + @TestGroup(enabled = true, sysProperty = SYSPROP_SLOW) public @interface Slow {} - + /** * Annotation for test classes that should avoid certain codec types * (because they are expensive, for example). @@ -179,7 +193,7 @@ public abstract class LuceneTestCase extends Assert { * Use this constant when creating Analyzers and any other version-dependent stuff. *

      NOTE: Change this when development starts for new Lucene version: */ - public static final Version TEST_VERSION_CURRENT = Version.LUCENE_40; + public static final Version TEST_VERSION_CURRENT = Version.LUCENE_50; /** * True if and only if tests are run in verbose mode. If this flag is false @@ -214,8 +228,17 @@ public abstract class LuceneTestCase extends Assert { /** the line file used by LineFileDocs */ public static final String TEST_LINE_DOCS_FILE = System.getProperty("tests.linedocsfile", DEFAULT_LINE_DOCS_FILE); - /** Whether or not @nightly tests should run. */ - public static final boolean TEST_NIGHTLY = systemPropertyAsBoolean("tests.nightly", false); + /** Whether or not {@link Nightly} tests should run. */ + public static final boolean TEST_NIGHTLY = systemPropertyAsBoolean(SYSPROP_NIGHTLY, false); + + /** Whether or not {@link Weekly} tests should run. */ + public static final boolean TEST_WEEKLY = systemPropertyAsBoolean(SYSPROP_WEEKLY, false); + + /** Whether or not {@link AwaitsFix} tests should run. */ + public static final boolean TEST_AWAITSFIX = systemPropertyAsBoolean(SYSPROP_AWAITSFIX, false); + + /** Whether or not {@link Slow} tests should run. */ + public static final boolean TEST_SLOW = systemPropertyAsBoolean(SYSPROP_SLOW, false); /** Throttling, see {@link MockDirectoryWrapper#setThrottling(Throttling)}. */ public static final Throttling TEST_THROTTLING = TEST_NIGHTLY ? Throttling.SOMETIMES : Throttling.NEVER; @@ -282,8 +305,30 @@ public abstract class LuceneTestCase extends Assert { /** * Suite failure marker (any error in the test or suite scope). */ - public static TestRuleMarkFailure suiteFailureMarker; - + public final static TestRuleMarkFailure suiteFailureMarker = + new TestRuleMarkFailure(); + + /** + * Ignore tests after hitting a designated number of initial failures. + */ + final static TestRuleIgnoreAfterMaxFailures ignoreAfterMaxFailures; + static { + int maxFailures = systemPropertyAsInt(SYSPROP_MAXFAILURES, Integer.MAX_VALUE); + boolean failFast = systemPropertyAsBoolean(SYSPROP_FAILFAST, false); + + if (failFast) { + if (maxFailures == Integer.MAX_VALUE) { + maxFailures = 1; + } else { + Logger.getLogger(LuceneTestCase.class.getSimpleName()).warning( + "Property '" + SYSPROP_MAXFAILURES + "'=" + maxFailures + ", 'failfast' is" + + " ignored."); + } + } + + ignoreAfterMaxFailures = new TestRuleIgnoreAfterMaxFailures(maxFailures); + } + /** * This controls how suite-level rules are nested. It is important that _all_ rules declared * in {@link LuceneTestCase} are executed in proper order if they depend on each @@ -292,7 +337,8 @@ public abstract class LuceneTestCase extends Assert { @ClassRule public static TestRule classRules = RuleChain .outerRule(new TestRuleIgnoreTestSuites()) - .around(suiteFailureMarker = new TestRuleMarkFailure()) + .around(ignoreAfterMaxFailures) + .around(suiteFailureMarker) .around(new TestRuleAssertionsRequired()) .around(new TestRuleNoStaticHooksShadowing()) .around(new TestRuleNoInstanceHooksOverrides()) @@ -313,7 +359,7 @@ public abstract class LuceneTestCase extends Assert { /** Save test thread and name. */ private TestRuleThreadAndTestName threadAndTestNameRule = new TestRuleThreadAndTestName(); - /** Taint test failures. */ + /** Taint suite result with individual test failures. */ private TestRuleMarkFailure testFailureMarker = new TestRuleMarkFailure(suiteFailureMarker); /** @@ -324,6 +370,7 @@ public abstract class LuceneTestCase extends Assert { @Rule public final TestRule ruleChain = RuleChain .outerRule(testFailureMarker) + .around(ignoreAfterMaxFailures) .around(threadAndTestNameRule) .around(new TestRuleReportUncaughtExceptions()) .around(new SystemPropertiesInvariantRule(IGNORED_INVARIANT_PROPERTIES)) @@ -419,11 +466,12 @@ public abstract class LuceneTestCase extends Assert { * do tests on that segment's reader. This is an utility method to help them. */ public static SegmentReader getOnlySegmentReader(DirectoryReader reader) { - IndexReader[] subReaders = reader.getSequentialSubReaders(); - if (subReaders.length != 1) - throw new IllegalArgumentException(reader + " has " + subReaders.length + " segments instead of exactly one"); - assertTrue(subReaders[0] instanceof SegmentReader); - return (SegmentReader) subReaders[0]; + List subReaders = reader.getSequentialSubReaders(); + if (subReaders.size() != 1) + throw new IllegalArgumentException(reader + " has " + subReaders.size() + " segments instead of exactly one"); + final IndexReader r = subReaders.get(0); + assertTrue(r instanceof SegmentReader); + return (SegmentReader) r; } /** @@ -625,8 +673,11 @@ public abstract class LuceneTestCase extends Assert { try { if (rarely(r)) { + Class clazz = Class.forName("org.apache.lucene.index.RandomDocumentsWriterPerThreadPool"); + Constructor ctor = clazz.getConstructor(int.class, Random.class); + ctor.setAccessible(true); // random thread pool - setIndexerThreadPoolMethod.invoke(c, new RandomDocumentsWriterPerThreadPool(maxNumThreadStates, r)); + setIndexerThreadPoolMethod.invoke(c, ctor.newInstance(maxNumThreadStates, r)); } else { // random thread pool c.setMaxThreadStates(maxNumThreadStates); @@ -735,7 +786,7 @@ public abstract class LuceneTestCase extends Assert { * some features of Windows, such as not allowing open files to be * overwritten. */ - public static MockDirectoryWrapper newDirectory() throws IOException { + public static MockDirectoryWrapper newDirectory() { return newDirectory(random()); } @@ -743,7 +794,7 @@ public abstract class LuceneTestCase extends Assert { * Returns a new Directory instance, using the specified random. * See {@link #newDirectory()} for more information. */ - public static MockDirectoryWrapper newDirectory(Random r) throws IOException { + public static MockDirectoryWrapper newDirectory(Random r) { Directory impl = newDirectoryImpl(r, TEST_DIRECTORY); MockDirectoryWrapper dir = new MockDirectoryWrapper(r, maybeNRTWrap(r, impl)); closeAfterSuite(new CloseableDirectory(dir, suiteFailureMarker)); @@ -765,12 +816,12 @@ public abstract class LuceneTestCase extends Assert { } /** Returns a new FSDirectory instance over the given file, which must be a folder. */ - public static MockDirectoryWrapper newFSDirectory(File f) throws IOException { + public static MockDirectoryWrapper newFSDirectory(File f) { return newFSDirectory(f, null); } /** Returns a new FSDirectory instance over the given file, which must be a folder. */ - public static MockDirectoryWrapper newFSDirectory(File f, LockFactory lf) throws IOException { + public static MockDirectoryWrapper newFSDirectory(File f, LockFactory lf) { String fsdirClass = TEST_DIRECTORY; if (fsdirClass.equals("random")) { fsdirClass = RandomPicks.randomFrom(random(), FS_DIRECTORIES); @@ -824,6 +875,22 @@ public abstract class LuceneTestCase extends Assert { } } + public static Field newStringField(String name, String value, Store stored) { + return newField(random(), name, value, stored == Store.YES ? StringField.TYPE_STORED : StringField.TYPE_NOT_STORED); + } + + public static Field newTextField(String name, String value, Store stored) { + return newField(random(), name, value, stored == Store.YES ? TextField.TYPE_STORED : TextField.TYPE_NOT_STORED); + } + + public static Field newStringField(Random random, String name, String value, Store stored) { + return newField(random, name, value, stored == Store.YES ? StringField.TYPE_STORED : StringField.TYPE_NOT_STORED); + } + + public static Field newTextField(Random random, String name, String value, Store stored) { + return newField(random, name, value, stored == Store.YES ? TextField.TYPE_STORED : TextField.TYPE_NOT_STORED); + } + public static Field newField(String name, String value, FieldType type) { return newField(random(), name, value, type); } @@ -896,10 +963,6 @@ public abstract class LuceneTestCase extends Assert { } } - public static boolean defaultCodecSupportsDocValues() { - return !Codec.getDefault().getName().equals("Lucene3x"); - } - private static Directory newFSDirectoryImpl( Class clazz, File file) throws IOException { @@ -1088,11 +1151,4 @@ public abstract class LuceneTestCase extends Assert { throw new IOException("Cannot find resource: " + name); } } - - /** - * @see SuppressCodecs - */ - static boolean shouldAvoidCodec(String codec) { - return classEnvRule.shouldAvoidCodec(codec); - } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/NullInfoStream.java b/lucene/test-framework/src/java/org/apache/lucene/util/NullInfoStream.java index 9f44cfa..717d7de 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/NullInfoStream.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/NullInfoStream.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/Rethrow.java b/lucene/test-framework/src/java/org/apache/lucene/util/Rethrow.java index 1450e99..2ae9be1 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/Rethrow.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/Rethrow.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/RollingBuffer.java b/lucene/test-framework/src/java/org/apache/lucene/util/RollingBuffer.java index cc04972..c1ded25 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/RollingBuffer.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/RollingBuffer.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/RunListenerPrintReproduceInfo.java b/lucene/test-framework/src/java/org/apache/lucene/util/RunListenerPrintReproduceInfo.java index ca0df2f..cb4ebd2 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/RunListenerPrintReproduceInfo.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/RunListenerPrintReproduceInfo.java @@ -1,20 +1,12 @@ package org.apache.lucene.util; -import static org.apache.lucene.util.LuceneTestCase.DEFAULT_LINE_DOCS_FILE; -import static org.apache.lucene.util.LuceneTestCase.JENKINS_LARGE_LINE_DOCS_FILE; -import static org.apache.lucene.util.LuceneTestCase.RANDOM_MULTIPLIER; -import static org.apache.lucene.util.LuceneTestCase.TEST_CODEC; -import static org.apache.lucene.util.LuceneTestCase.TEST_DIRECTORY; -import static org.apache.lucene.util.LuceneTestCase.TEST_LINE_DOCS_FILE; -import static org.apache.lucene.util.LuceneTestCase.TEST_NIGHTLY; -import static org.apache.lucene.util.LuceneTestCase.TEST_POSTINGSFORMAT; -import static org.apache.lucene.util.LuceneTestCase.classEnvRule; +import static org.apache.lucene.util.LuceneTestCase.*; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.regex.Pattern; -import org.apache.lucene.codecs.Codec; import org.junit.runner.Description; import org.junit.runner.Result; import org.junit.runner.notification.Failure; @@ -23,7 +15,7 @@ import org.junit.runner.notification.RunListener; import com.carrotsearch.randomizedtesting.LifecycleScope; import com.carrotsearch.randomizedtesting.RandomizedContext; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -112,11 +104,11 @@ public final class RunListenerPrintReproduceInfo extends RunListener { reportAdditionalFailureInfo(null); } } - + /** print some useful debugging information about the environment */ - static void printDebuggingInformation() { + private static void printDebuggingInformation() { if (classEnvRule != null) { - System.err.println("NOTE: test params are: codec=" + Codec.getDefault() + + System.err.println("NOTE: test params are: codec=" + classEnvRule.codec + ", sim=" + classEnvRule.similarity + ", locale=" + classEnvRule.locale + ", timezone=" + (classEnvRule.timeZone == null ? "(null)" : classEnvRule.timeZone.getID())); @@ -134,41 +126,60 @@ public final class RunListenerPrintReproduceInfo extends RunListener { System.err.println("NOTE: All tests run in this JVM: " + Arrays.toString(testClassesRun.toArray())); } - // We get here from InterceptTestCaseEvents on the 'failed' event.... - public void reportAdditionalFailureInfo(final String testName) { + private void reportAdditionalFailureInfo(final String testName) { if (TEST_LINE_DOCS_FILE.endsWith(JENKINS_LARGE_LINE_DOCS_FILE)) { - System.err.println("NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory."); + System.err.println("NOTE: download the large Jenkins line-docs file by running " + + "'ant get-jenkins-line-docs' in the lucene directory."); } - StringBuilder b = new StringBuilder(); - b.append("NOTE: reproduce with: ant test ") - .append("-Dtestcase=").append(RandomizedContext.current().getTargetClass().getSimpleName()); - if (testName != null) { - b.append(" -Dtests.method=").append(testName); + final StringBuilder b = new StringBuilder(); + b.append("NOTE: reproduce with: ant test "); + + // Test case, method, seed. + addVmOpt(b, "testcase", RandomizedContext.current().getTargetClass().getSimpleName()); + addVmOpt(b, "tests.method", testName); + addVmOpt(b, "tests.seed", RandomizedContext.current().getRunnerSeedAsString()); + + // Test groups and multipliers. + if (RANDOM_MULTIPLIER > 1) addVmOpt(b, "tests.multiplier", RANDOM_MULTIPLIER); + if (TEST_NIGHTLY) addVmOpt(b, SYSPROP_NIGHTLY, TEST_NIGHTLY); + if (TEST_WEEKLY) addVmOpt(b, SYSPROP_WEEKLY, TEST_WEEKLY); + if (TEST_SLOW) addVmOpt(b, SYSPROP_SLOW, TEST_SLOW); + if (TEST_AWAITSFIX) addVmOpt(b, SYSPROP_AWAITSFIX, TEST_AWAITSFIX); + + // Codec, postings, directories. + if (!TEST_CODEC.equals("random")) addVmOpt(b, "tests.codec", TEST_CODEC); + if (!TEST_POSTINGSFORMAT.equals("random")) addVmOpt(b, "tests.postingsformat", TEST_POSTINGSFORMAT); + if (!TEST_DIRECTORY.equals("random")) addVmOpt(b, "tests.directory", TEST_DIRECTORY); + + // Environment. + if (!TEST_LINE_DOCS_FILE.equals(DEFAULT_LINE_DOCS_FILE)) addVmOpt(b, "tests.linedocsfile", TEST_LINE_DOCS_FILE); + if (classEnvRule != null) { + addVmOpt(b, "tests.locale", classEnvRule.locale); + if (classEnvRule.timeZone != null) { + addVmOpt(b, "tests.timezone", classEnvRule.timeZone.getID()); + } } - b.append(" -Dtests.seed=") - .append(RandomizedContext.current().getRunnerSeedAsString()) - .append(reproduceWithExtraParams()); + + addVmOpt(b, "tests.file.encoding", System.getProperty("file.encoding")); + System.err.println(b.toString()); } - // extra params that were overridden needed to reproduce the command - private static String reproduceWithExtraParams() { - StringBuilder sb = new StringBuilder(); - if (classEnvRule != null) { - if (classEnvRule.locale != null) sb.append(" -Dtests.locale=").append(classEnvRule.locale); - if (classEnvRule.timeZone != null) sb.append(" -Dtests.timezone=").append(classEnvRule.timeZone.getID()); + /** + * Append a VM option (-Dkey=value) to a {@link StringBuilder}. Add quotes if + * spaces or other funky characters are detected. + */ + static void addVmOpt(StringBuilder b, String key, Object value) { + if (value == null) return; + + b.append(" -D").append(key).append("="); + String v = value.toString(); + // Add simplistic quoting. This varies a lot from system to system and between + // shells... ANT should have some code for doing it properly. + if (Pattern.compile("[\\s=']").matcher(v).find()) { + v = '"' + v + '"'; } - if (!TEST_CODEC.equals("random")) sb.append(" -Dtests.codec=").append(TEST_CODEC); - if (!TEST_POSTINGSFORMAT.equals("random")) sb.append(" -Dtests.postingsformat=").append(TEST_POSTINGSFORMAT); - if (!TEST_DIRECTORY.equals("random")) sb.append(" -Dtests.directory=").append(TEST_DIRECTORY); - if (RANDOM_MULTIPLIER > 1) sb.append(" -Dtests.multiplier=").append(RANDOM_MULTIPLIER); - if (TEST_NIGHTLY) sb.append(" -Dtests.nightly=true"); - if (!TEST_LINE_DOCS_FILE.equals(DEFAULT_LINE_DOCS_FILE)) sb.append(" -Dtests.linedocsfile=" + TEST_LINE_DOCS_FILE); - - // TODO we can't randomize this yet (it drives ant crazy) but this makes tests reproduce - // in case machines have different default charsets... - sb.append(" -Dargs=\"-Dfile.encoding=" + System.getProperty("file.encoding") + "\""); - return sb.toString(); - } + b.append(v); + } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleAssertionsRequired.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleAssertionsRequired.java index b263631..9fc8b39 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleAssertionsRequired.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleAssertionsRequired.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleFieldCacheSanity.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleFieldCacheSanity.java index 3afcf80..ea5d632 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleFieldCacheSanity.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleFieldCacheSanity.java @@ -5,7 +5,7 @@ import org.junit.rules.TestRule; import org.junit.runner.Description; import org.junit.runners.model.Statement; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIcuHack.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIcuHack.java index 303a01e..91964d1 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIcuHack.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIcuHack.java @@ -7,7 +7,7 @@ import org.junit.rules.TestRule; import org.junit.runner.Description; import org.junit.runners.model.Statement; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIgnoreAfterMaxFailures.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIgnoreAfterMaxFailures.java new file mode 100644 index 0000000..6666714 --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIgnoreAfterMaxFailures.java @@ -0,0 +1,80 @@ +package org.apache.lucene.util; + +import org.junit.Assert; +import org.junit.internal.AssumptionViolatedException; +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; + +import com.carrotsearch.randomizedtesting.RandomizedTest; +import com.carrotsearch.randomizedtesting.annotations.Repeat; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This rule keeps a count of failed tests (suites) and will result in an + * {@link AssumptionViolatedException} after a given number of failures for all + * tests following this condition. + * + *

      + * Aborting quickly on failed tests can be useful when used in combination with + * test repeats (via the {@link Repeat} annotation or system property). + */ +public final class TestRuleIgnoreAfterMaxFailures implements TestRule { + /** + * Maximum failures. Package scope for tests. + */ + int maxFailures; + + /** + * Current count of failures. Package scope for tests. + */ + int failuresSoFar; + + /** + * @param maxFailures + * The number of failures after which all tests are ignored. Must be + * greater or equal 1. + */ + public TestRuleIgnoreAfterMaxFailures(int maxFailures) { + Assert.assertTrue("maxFailures must be >= 1: " + maxFailures, maxFailures >= 1); + this.maxFailures = maxFailures; + } + + @Override + public Statement apply(final Statement s, final Description d) { + return new Statement() { + @Override + public void evaluate() throws Throwable { + if (failuresSoFar >= maxFailures) { + RandomizedTest.assumeTrue("Ignored, failures limit reached (" + + failuresSoFar + " >= " + maxFailures + ").", false); + } + + try { + s.evaluate(); + } catch (Throwable t) { + if (!TestRuleMarkFailure.isAssumption(t)) { + failuresSoFar++; + } + throw t; + } + } + }; + } +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIgnoreTestSuites.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIgnoreTestSuites.java index 58fa30c..6020904 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIgnoreTestSuites.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIgnoreTestSuites.java @@ -1,11 +1,10 @@ package org.apache.lucene.util; -import org.junit.Assume; import org.junit.rules.TestRule; import org.junit.runner.Description; import org.junit.runners.model.Statement; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleMarkFailure.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleMarkFailure.java index 2f2e882..be9c625 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleMarkFailure.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleMarkFailure.java @@ -8,7 +8,7 @@ import org.junit.rules.TestRule; import org.junit.runner.Description; import org.junit.runners.model.Statement; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -47,11 +47,8 @@ public final class TestRuleMarkFailure implements TestRule { try { s.evaluate(); } catch (Throwable t) { - for (Throwable t2 : expandFromMultiple(t)) { - if (!(t2 instanceof AssumptionViolatedException)) { - markFailed(); - break; - } + if (!isAssumption(t)) { + markFailed(); } throw t; } @@ -60,6 +57,19 @@ public final class TestRuleMarkFailure implements TestRule { } /** + * Is a given exception (or a MultipleFailureException) an + * {@link AssumptionViolatedException}? + */ + public static boolean isAssumption(Throwable t) { + for (Throwable t2 : expandFromMultiple(t)) { + if (!(t2 instanceof AssumptionViolatedException)) { + return false; + } + } + return true; + } + + /** * Expand from multi-exception wrappers. */ private static List expandFromMultiple(Throwable t) { diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleNoInstanceHooksOverrides.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleNoInstanceHooksOverrides.java index 9304ca4..81dda90 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleNoInstanceHooksOverrides.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleNoInstanceHooksOverrides.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleNoStaticHooksShadowing.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleNoStaticHooksShadowing.java index 033f9e2..af28b6a 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleNoStaticHooksShadowing.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleNoStaticHooksShadowing.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleReportUncaughtExceptions.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleReportUncaughtExceptions.java index eb87fcb..fa75b7c 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleReportUncaughtExceptions.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleReportUncaughtExceptions.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java index 254f634..22dacfa 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java @@ -1,5 +1,22 @@ package org.apache.lucene.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + import java.util.Arrays; import java.util.Date; import java.util.HashMap; @@ -13,7 +30,6 @@ import java.util.TimeZone; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.appending.AppendingCodec; -import org.apache.lucene.codecs.lucene3x.PreFlexRWCodec; import org.apache.lucene.codecs.lucene40.Lucene40Codec; import org.apache.lucene.codecs.simpletext.SimpleTextCodec; import org.apache.lucene.index.RandomCodec; @@ -21,33 +37,15 @@ import org.apache.lucene.search.RandomSimilarityProvider; import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; +import org.junit.internal.AssumptionViolatedException; + import com.carrotsearch.randomizedtesting.RandomizedContext; import static org.apache.lucene.util.LuceneTestCase.*; -import static org.apache.lucene.util.LuceneTestCase.INFOSTREAM; -import static org.apache.lucene.util.LuceneTestCase.TEST_CODEC; -import static org.apache.lucene.util.LuceneTestCase.VERBOSE; /** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** * Setup and restore suite-level environment (fine grained junk that * doesn't fit anywhere else). */ @@ -65,6 +63,7 @@ final class TestRuleSetupAndRestoreClassEnv extends AbstractBeforeAfterRule { Locale locale; TimeZone timeZone; Similarity similarity; + Codec codec; /** * @see SuppressCodecs @@ -81,7 +80,11 @@ final class TestRuleSetupAndRestoreClassEnv extends AbstractBeforeAfterRule { if (System.getProperty("solr.directoryFactory") == null) { System.setProperty("solr.directoryFactory", "org.apache.solr.core.MockDirectoryFactory"); } - + + // Restore more Solr properties. + restoreProperties.put("solr.solr.home", System.getProperty("solr.solr.home")); + restoreProperties.put("solr.data.dir", System.getProperty("solr.data.dir")); + // enable the Lucene 3.x PreflexRW codec explicitly, to work around bugs in IBM J9 / Harmony ServiceLoader: try { final java.lang.reflect.Field spiLoaderField = Codec.class.getDeclaredField("loader"); @@ -89,6 +92,7 @@ final class TestRuleSetupAndRestoreClassEnv extends AbstractBeforeAfterRule { final Object spiLoader = spiLoaderField.get(null); final java.lang.reflect.Field modifiableServicesField = NamedSPILoader.class.getDeclaredField("modifiableServices"); modifiableServicesField.setAccessible(true); + /* note: re-enable this if we make a Lucene4x impersonator @SuppressWarnings({"unchecked","rawtypes"}) final Map serviceMap = (Map) modifiableServicesField.get(spiLoader); if (!(Codec.forName("Lucene3x") instanceof PreFlexRWCodec)) { @@ -103,12 +107,12 @@ final class TestRuleSetupAndRestoreClassEnv extends AbstractBeforeAfterRule { " and does not respect classpath order, please report this to the vendor."); } serviceMap.put("Lucene3x", new PreFlexRWCodec()); - } + } */ } catch (Exception e) { throw new RuntimeException("Cannot access internals of Codec and NamedSPILoader classes", e); } - // if verbose: print some debugging stuff about which codecs are loaded + // if verbose: print some debugging stuff about which codecs are loaded. if (VERBOSE) { Set codecs = Codec.availableCodecs(); for (String codec : codecs) { @@ -131,7 +135,7 @@ final class TestRuleSetupAndRestoreClassEnv extends AbstractBeforeAfterRule { final String name; if (Thread.currentThread().getName().startsWith("TEST-")) { // The name of the main thread is way too - // long when looking at IW verbose output...: + // long when looking at IW verbose output... name = "main"; } else { name = Thread.currentThread().getName(); @@ -148,27 +152,21 @@ final class TestRuleSetupAndRestoreClassEnv extends AbstractBeforeAfterRule { if (targetClass.isAnnotationPresent(SuppressCodecs.class)) { SuppressCodecs a = targetClass.getAnnotation(SuppressCodecs.class); avoidCodecs.addAll(Arrays.asList(a.value())); - System.err.println("NOTE: Suppressing codecs " + Arrays.toString(a.value()) - + " for " + targetClass.getSimpleName() + "."); } PREFLEX_IMPERSONATION_IS_ACTIVE = false; savedCodec = Codec.getDefault(); - final Codec codec; int randomVal = random.nextInt(10); - if ("Lucene3x".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal < 2 && !shouldAvoidCodec("Lucene3x"))) { // preflex-only setup + + /* note: re-enable this if we make a 4.x impersonator + if ("Lucene3x".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && + "random".equals(TEST_POSTINGSFORMAT) && + randomVal < 2 && + !shouldAvoidCodec("Lucene3x"))) { // preflex-only setup codec = Codec.forName("Lucene3x"); assert (codec instanceof PreFlexRWCodec) : "fix your classpath to have tests-framework.jar before lucene-core.jar"; PREFLEX_IMPERSONATION_IS_ACTIVE = true; - } else if ("SimpleText".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal == 9 && !shouldAvoidCodec("SimpleText"))) { - codec = new SimpleTextCodec(); - } else if ("Appending".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal == 8 && !shouldAvoidCodec("Appending"))) { - codec = new AppendingCodec(); - } else if (!"random".equals(TEST_CODEC)) { - codec = Codec.forName(TEST_CODEC); - } else if ("random".equals(TEST_POSTINGSFORMAT)) { - codec = new RandomCodec(random, avoidCodecs); - } else { + } else */ if (!"random".equals(TEST_POSTINGSFORMAT)) { codec = new Lucene40Codec() { private final PostingsFormat format = PostingsFormat.forName(TEST_POSTINGSFORMAT); @@ -182,6 +180,16 @@ final class TestRuleSetupAndRestoreClassEnv extends AbstractBeforeAfterRule { return super.toString() + ": " + format.toString(); } }; + } else if ("SimpleText".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal == 9 && !shouldAvoidCodec("SimpleText"))) { + codec = new SimpleTextCodec(); + } else if ("Appending".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal == 8 && !shouldAvoidCodec("Appending"))) { + codec = new AppendingCodec(); + } else if (!"random".equals(TEST_CODEC)) { + codec = Codec.forName(TEST_CODEC); + } else if ("random".equals(TEST_POSTINGSFORMAT)) { + codec = new RandomCodec(random, avoidCodecs); + } else { + assert false; } Codec.setDefault(codec); @@ -202,7 +210,40 @@ final class TestRuleSetupAndRestoreClassEnv extends AbstractBeforeAfterRule { TimeZone randomTimeZone = randomTimeZone(random()); timeZone = testTimeZone.equals("random") ? randomTimeZone : TimeZone.getTimeZone(testTimeZone); TimeZone.setDefault(timeZone); - similarity = random().nextBoolean() ? new DefaultSimilarity() : new RandomSimilarityProvider(random()); + similarity = random().nextBoolean() ? new DefaultSimilarity() : new RandomSimilarityProvider(random()); + + // Check codec restrictions once at class level. + try { + checkCodecRestrictions(codec); + } catch (AssumptionViolatedException e) { + System.err.println("NOTE: " + e.getMessage() + " Suppressed codecs: " + + Arrays.toString(avoidCodecs.toArray())); + throw e; + } + } + + /** + * Check codec restrictions. + * + * @throws AssumptionViolatedException if the class does not work with a given codec. + */ + private void checkCodecRestrictions(Codec codec) { + assumeFalse("Class not allowed to use codec: " + codec.getName() + ".", + shouldAvoidCodec(codec.getName())); + + if (codec instanceof RandomCodec && !avoidCodecs.isEmpty()) { + for (String name : ((RandomCodec)codec).formatNames) { + assumeFalse("Class not allowed to use postings format: " + name + ".", + shouldAvoidCodec(name)); + } + } + + PostingsFormat pf = codec.postingsFormat(); + assumeFalse("Class not allowed to use postings format: " + pf.getName() + ".", + shouldAvoidCodec(pf.getName())); + + assumeFalse("Class not allowed to use postings format: " + LuceneTestCase.TEST_POSTINGSFORMAT + ".", + shouldAvoidCodec(LuceneTestCase.TEST_POSTINGSFORMAT)); } /** @@ -221,17 +262,14 @@ final class TestRuleSetupAndRestoreClassEnv extends AbstractBeforeAfterRule { Codec.setDefault(savedCodec); InfoStream.setDefault(savedInfoStream); - Locale.setDefault(savedLocale); - TimeZone.setDefault(savedTimeZone); - - System.clearProperty("solr.solr.home"); - System.clearProperty("solr.data.dir"); + if (savedLocale != null) Locale.setDefault(savedLocale); + if (savedTimeZone != null) TimeZone.setDefault(savedTimeZone); } /** * Should a given codec be avoided for the currently executing suite? */ - public boolean shouldAvoidCodec(String codec) { + private boolean shouldAvoidCodec(String codec) { return !avoidCodecs.isEmpty() && avoidCodecs.contains(codec); } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreInstanceEnv.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreInstanceEnv.java index 555f788..d8f8555 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreInstanceEnv.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreInstanceEnv.java @@ -1,9 +1,8 @@ package org.apache.lucene.util; import org.apache.lucene.search.BooleanQuery; -import org.junit.internal.AssumptionViolatedException; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -29,12 +28,6 @@ final class TestRuleSetupAndRestoreInstanceEnv extends AbstractBeforeAfterRule { protected void before() { savedBoolMaxClauseCount = BooleanQuery.getMaxClauseCount(); - - final String defFormat = _TestUtil.getPostingsFormat("thisCodeMakesAbsolutelyNoSenseCanWeDeleteIt"); - if (LuceneTestCase.shouldAvoidCodec(defFormat)) { - throw new AssumptionViolatedException( - "Method not allowed to use codec: " + defFormat + "."); - } } protected void after() { diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupTeardownChained.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupTeardownChained.java index 7cdd5c1..3073694 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupTeardownChained.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupTeardownChained.java @@ -5,7 +5,7 @@ import org.junit.rules.TestRule; import org.junit.runner.Description; import org.junit.runners.model.Statement; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleStoreClassName.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleStoreClassName.java index ec6bf17..093ed1d 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleStoreClassName.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleStoreClassName.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleThreadAndTestName.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleThreadAndTestName.java index cfe0589..dadc965 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleThreadAndTestName.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleThreadAndTestName.java @@ -4,7 +4,7 @@ import org.junit.rules.TestRule; import org.junit.runner.Description; import org.junit.runners.model.Statement; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java b/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java index 5564a31..b25106b 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java index adb3595..1fb842a 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java @@ -1,6 +1,6 @@ package org.apache.lucene.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -26,6 +26,8 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.PrintStream; import java.lang.reflect.Method; +import java.math.BigDecimal; +import java.math.BigInteger; import java.nio.CharBuffer; import java.util.*; import java.util.concurrent.ExecutorService; @@ -77,6 +79,9 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.junit.Assert; +import com.carrotsearch.randomizedtesting.generators.RandomInts; +import com.carrotsearch.randomizedtesting.generators.RandomPicks; + /** * General utility methods for Lucene unit tests. */ @@ -207,7 +212,23 @@ public class _TestUtil { /** start and end are BOTH inclusive */ public static int nextInt(Random r, int start, int end) { - return start + r.nextInt(end-start+1); + return RandomInts.randomIntBetween(r, start, end); + } + + /** start and end are BOTH inclusive */ + public static long nextLong(Random r, long start, long end) { + assert end >= start; + final BigInteger range = BigInteger.valueOf(end).add(BigInteger.valueOf(1)).subtract(BigInteger.valueOf(start)); + if (range.compareTo(BigInteger.valueOf(Integer.MAX_VALUE)) <= 0) { + return start + r.nextInt(range.intValue()); + } else { + // probably not evenly distributed when range is large, but OK for tests + final BigInteger augend = new BigDecimal(range).multiply(new BigDecimal(r.nextDouble())).toBigInteger(); + final long result = BigInteger.valueOf(start).add(augend).longValue(); + assert result >= start; + assert result <= end; + return result; + } } public static String randomSimpleString(Random r, int maxLength) { @@ -293,37 +314,49 @@ public class _TestUtil { public static String randomRegexpishString(Random r) { return randomRegexpishString(r, 20); } - + + /** + * Maximum recursion bound for '+' and '*' replacements in + * {@link #randomRegexpishString(Random, int)}. + */ + private final static int maxRecursionBound = 5; + + /** + * Operators for {@link #randomRegexpishString(Random, int)}. + */ + private final static List ops = Arrays.asList( + ".", "?", + "{0," + maxRecursionBound + "}", // bounded replacement for '*' + "{1," + maxRecursionBound + "}", // bounded replacement for '+' + "(", + ")", + "-", + "[", + "]", + "|" + ); + /** * Returns a String thats "regexpish" (contains lots of operators typically found in regular expressions) * If you call this enough times, you might get a valid regex! + * + *

      Note: to avoid practically endless backtracking patterns we replace asterisk and plus + * operators with bounded repetitions. See LUCENE-4111 for more info. + * + * @param maxLength A hint about maximum length of the regexpish string. It may be exceeded by a few characters. */ public static String randomRegexpishString(Random r, int maxLength) { - final int end = nextInt(r, 0, maxLength); - if (end == 0) { - // allow 0 length - return ""; - } - final char[] buffer = new char[end]; - for (int i = 0; i < end; i++) { - int t = r.nextInt(11); - if (t == 0) { - buffer[i] = (char) _TestUtil.nextInt(r, 97, 102); + final StringBuilder regexp = new StringBuilder(maxLength); + for (int i = nextInt(r, 0, maxLength); i > 0; i--) { + if (r.nextBoolean()) { + regexp.append((char) RandomInts.randomIntBetween(r, 'a', 'z')); + } else { + regexp.append(RandomPicks.randomFrom(r, ops)); } - else if (1 == t) buffer[i] = '.'; - else if (2 == t) buffer[i] = '?'; - else if (3 == t) buffer[i] = '*'; - else if (4 == t) buffer[i] = '+'; - else if (5 == t) buffer[i] = '('; - else if (6 == t) buffer[i] = ')'; - else if (7 == t) buffer[i] = '-'; - else if (8 == t) buffer[i] = '['; - else if (9 == t) buffer[i] = ']'; - else if (10 == t) buffer[i] = '|'; } - return new String(buffer, 0, end); + return regexp.toString(); } - + private static final String[] HTML_CHAR_ENTITIES = { "AElig", "Aacute", "Acirc", "Agrave", "Alpha", "AMP", "Aring", "Atilde", "Auml", "Beta", "COPY", "Ccedil", "Chi", "Dagger", "Delta", "ETH", @@ -646,10 +679,12 @@ public class _TestUtil { if (mp instanceof LogMergePolicy) { LogMergePolicy lmp = (LogMergePolicy) mp; lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor())); + lmp.setUseCompoundFile(true); } else if (mp instanceof TieredMergePolicy) { TieredMergePolicy tmp = (TieredMergePolicy) mp; tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce())); tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier())); + tmp.setUseCompoundFile(true); } MergeScheduler ms = w.getConfig().getMergeScheduler(); if (ms instanceof ConcurrentMergeScheduler) { @@ -933,8 +968,9 @@ public class _TestUtil { Pattern p = Pattern.compile(_TestUtil.randomRegexpishString(random)); // Make sure the result of applying the pattern to a string with extended // unicode characters is a valid utf16 string. See LUCENE-4078 for discussion. - if (UnicodeUtil.validUTF16String(p.matcher(nonBmpString).replaceAll("_"))) + if (UnicodeUtil.validUTF16String(p.matcher(nonBmpString).replaceAll("_"))) { return p; + } } catch (PatternSyntaxException ignored) { // Loop trying until we hit something that compiles. } diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/automaton/AutomatonTestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/util/automaton/AutomatonTestUtil.java index 149b5ad..c0cf4ca 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/automaton/AutomatonTestUtil.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/automaton/AutomatonTestUtil.java @@ -1,6 +1,6 @@ package org.apache.lucene.util.automaton; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java b/lucene/test-framework/src/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java deleted file mode 100644 index 784350b..0000000 --- a/lucene/test-framework/src/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java +++ /dev/null @@ -1,361 +0,0 @@ -package org.apache.lucene.util.automaton; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.*; - -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CharsRef; -import org.apache.lucene.util.UnicodeUtil; - -/** - * Builds a minimal deterministic automaton that accepts a set of strings. The - * algorithm requires sorted input data, but is very fast (nearly linear with - * the input size). - */ -public final class DaciukMihovAutomatonBuilder { - /** - * DFSA state with char labels on transitions. - */ - public final static class State { - - /** An empty set of labels. */ - private final static int[] NO_LABELS = new int[0]; - - /** An empty set of states. */ - private final static State[] NO_STATES = new State[0]; - - /** - * Labels of outgoing transitions. Indexed identically to {@link #states}. - * Labels must be sorted lexicographically. - */ - int[] labels = NO_LABELS; - - /** - * States reachable from outgoing transitions. Indexed identically to - * {@link #labels}. - */ - State[] states = NO_STATES; - - /** - * true if this state corresponds to the end of at least one - * input sequence. - */ - boolean is_final; - - /** - * Returns the target state of a transition leaving this state and labeled - * with label. If no such transition exists, returns - * null. - */ - public State getState(int label) { - final int index = Arrays.binarySearch(labels, label); - return index >= 0 ? states[index] : null; - } - - /** - * Returns an array of outgoing transition labels. The array is sorted in - * lexicographic order and indexes correspond to states returned from - * {@link #getStates()}. - */ - public int[] getTransitionLabels() { - return this.labels; - } - - /** - * Returns an array of outgoing transitions from this state. The returned - * array must not be changed. - */ - public State[] getStates() { - return this.states; - } - - /** - * Two states are equal if: - *

        - *
      • they have an identical number of outgoing transitions, labeled with - * the same labels
        • - *
      • corresponding outgoing transitions lead to the same states (to states - * with an identical right-language). - *
      - */ - @Override - public boolean equals(Object obj) { - final State other = (State) obj; - return is_final == other.is_final - && Arrays.equals(this.labels, other.labels) - && referenceEquals(this.states, other.states); - } - - /** - * Return true if this state has any children (outgoing - * transitions). - */ - public boolean hasChildren() { - return labels.length > 0; - } - - /** - * Is this state a final state in the automaton? - */ - public boolean isFinal() { - return is_final; - } - - /** - * Compute the hash code of the current status of this state. - */ - @Override - public int hashCode() { - int hash = is_final ? 1 : 0; - - hash ^= hash * 31 + this.labels.length; - for (int c : this.labels) - hash ^= hash * 31 + c; - - /* - * Compare the right-language of this state using reference-identity of - * outgoing states. This is possible because states are interned (stored - * in registry) and traversed in post-order, so any outgoing transitions - * are already interned. - */ - for (State s : this.states) { - hash ^= System.identityHashCode(s); - } - - return hash; - } - - /** - * Create a new outgoing transition labeled label and return - * the newly created target state for this transition. - */ - State newState(int label) { - assert Arrays.binarySearch(labels, label) < 0 : "State already has transition labeled: " - + label; - - labels = copyOf(labels, labels.length + 1); - states = copyOf(states, states.length + 1); - - labels[labels.length - 1] = label; - return states[states.length - 1] = new State(); - } - - /** - * Return the most recent transitions's target state. - */ - State lastChild() { - assert hasChildren() : "No outgoing transitions."; - return states[states.length - 1]; - } - - /** - * Return the associated state if the most recent transition is labeled with - * label. - */ - State lastChild(int label) { - final int index = labels.length - 1; - State s = null; - if (index >= 0 && labels[index] == label) { - s = states[index]; - } - assert s == getState(label); - return s; - } - - /** - * Replace the last added outgoing transition's target state with the given - * state. - */ - void replaceLastChild(State state) { - assert hasChildren() : "No outgoing transitions."; - states[states.length - 1] = state; - } - - /** - * JDK1.5-replacement of {@link Arrays#copyOf(int[], int)} - */ - private static int[] copyOf(int[] original, int newLength) { - int[] copy = new int[newLength]; - System.arraycopy(original, 0, copy, 0, - Math.min(original.length, newLength)); - return copy; - } - - /** - * JDK1.5-replacement of {@link Arrays#copyOf(char[], int)} - */ - public static State[] copyOf(State[] original, int newLength) { - State[] copy = new State[newLength]; - System.arraycopy(original, 0, copy, 0, - Math.min(original.length, newLength)); - return copy; - } - - /** - * Compare two lists of objects for reference-equality. - */ - private static boolean referenceEquals(Object[] a1, Object[] a2) { - if (a1.length != a2.length) return false; - - for (int i = 0; i < a1.length; i++) - if (a1[i] != a2[i]) return false; - - return true; - } - } - - /** - * "register" for state interning. - */ - private HashMap register = new HashMap(); - - /** - * Root automaton state. - */ - private State root = new State(); - - /** - * Previous sequence added to the automaton in {@link #add(CharSequence)}. - */ - private CharsRef previous; - - private static final Comparator comparator = CharsRef.getUTF16SortedAsUTF8Comparator(); - - /** - * Add another character sequence to this automaton. The sequence must be - * lexicographically larger or equal compared to any previous sequences added - * to this automaton (the input must be sorted). - */ - public void add(CharsRef current) { - assert register != null : "Automaton already built."; - assert previous == null - || comparator.compare(previous, current) <= 0 : "Input must be sorted: " - + previous + " >= " + current; - assert setPrevious(current); - - // Descend in the automaton (find matching prefix). - int pos = 0, max = current.length(); - State next, state = root; - while (pos < max && (next = state.lastChild(Character.codePointAt(current, pos))) != null) { - state = next; - // todo, optimize me - pos += Character.charCount(Character.codePointAt(current, pos)); - } - - if (state.hasChildren()) replaceOrRegister(state); - - addSuffix(state, current, pos); - } - - /** - * Finalize the automaton and return the root state. No more strings can be - * added to the builder after this call. - * - * @return Root automaton state. - */ - public State complete() { - if (this.register == null) throw new IllegalStateException(); - - if (root.hasChildren()) replaceOrRegister(root); - - register = null; - return root; - } - - /** - * Internal recursive traversal for conversion. - */ - private static org.apache.lucene.util.automaton.State convert(State s, - IdentityHashMap visited) { - org.apache.lucene.util.automaton.State converted = visited.get(s); - if (converted != null) return converted; - - converted = new org.apache.lucene.util.automaton.State(); - converted.setAccept(s.is_final); - - visited.put(s, converted); - int i = 0; - int[] labels = s.labels; - for (DaciukMihovAutomatonBuilder.State target : s.states) { - converted.addTransition(new Transition(labels[i++], convert(target, - visited))); - } - - return converted; - } - - /** - * Build a minimal, deterministic automaton from a sorted list of strings. - */ - public static Automaton build(Collection input) { - final DaciukMihovAutomatonBuilder builder = new DaciukMihovAutomatonBuilder(); - - CharsRef scratch = new CharsRef(); - for (BytesRef b : input) { - UnicodeUtil.UTF8toUTF16(b, scratch); - builder.add(scratch); - } - - Automaton a = new Automaton(); - a.initial = convert(builder.complete(), new IdentityHashMap()); - a.deterministic = true; - return a; - } - - /** - * Copy current into an internal buffer. - */ - private boolean setPrevious(CharsRef current) { - // don't need to copy, once we fix https://issues.apache.org/jira/browse/LUCENE-3277 - // still, called only from assert - previous = CharsRef.deepCopyOf(current); - return true; - } - - /** - * Replace last child of state with an already registered state - * or register the last child state. - */ - private void replaceOrRegister(State state) { - final State child = state.lastChild(); - - if (child.hasChildren()) replaceOrRegister(child); - - final State registered = register.get(child); - if (registered != null) { - state.replaceLastChild(registered); - } else { - register.put(child, child); - } - } - - /** - * Add a suffix of current starting at fromIndex - * (inclusive) to state state. - */ - private void addSuffix(State state, CharSequence current, int fromIndex) { - final int len = current.length(); - while (fromIndex < len) { - int cp = Character.codePointAt(current, fromIndex); - state = state.newState(cp); - fromIndex += Character.charCount(cp); - } - state.is_final = true; - } -} diff --git a/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec b/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec index 3bf4313..4a812de 100644 --- a/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -13,4 +13,3 @@ # See the License for the specific language governing permissions and # limitations under the License. -org.apache.lucene.codecs.lucene3x.PreFlexRWCodec diff --git a/lucene/tools/junit4/cached-timehints.txt b/lucene/tools/junit4/cached-timehints.txt old mode 100755 new mode 100644 index 2a7e584..92e39c4 --- a/lucene/tools/junit4/cached-timehints.txt +++ b/lucene/tools/junit4/cached-timehints.txt @@ -1,1052 +1,1028 @@ -org.apache.lucene.TestAssertions=5,4,3,4,4,65 -org.apache.lucene.TestDemo=72,9,9,16,8,16 -org.apache.lucene.TestExternalCodecs=299,136,69,126,994,1102 -org.apache.lucene.TestMergeSchedulerExternal=278,539,590,305,451,338 -org.apache.lucene.TestSearch=22,44,71,17,32,140 -org.apache.lucene.TestSearchForDuplicates=113,98,55,115,126,30 -org.apache.lucene.analysis.TestCachingTokenFilter=7,32,36,9,8,12 -org.apache.lucene.analysis.TestGraphTokenizers=16571 -org.apache.lucene.analysis.TestLookaheadTokenFilter=15138 -org.apache.lucene.analysis.TestMockAnalyzer=1698,2243,5217,1473,734,1862 -org.apache.lucene.analysis.TestMockCharFilter=5,4,4,5,6,10 -org.apache.lucene.analysis.TestNumericTokenStream=15,4,6,7,6,57 -org.apache.lucene.analysis.TestToken=3168,1751,1631,1751,2500,1526 -org.apache.lucene.analysis.ar.TestArabicAnalyzer=861,315,404,472,455,650 -org.apache.lucene.analysis.ar.TestArabicLetterTokenizer=6,6,6,7,4,35 -org.apache.lucene.analysis.ar.TestArabicNormalizationFilter=22,23,37,21,16,75 -org.apache.lucene.analysis.ar.TestArabicStemFilter=39,41,36,102,26,105 -org.apache.lucene.analysis.bg.TestBulgarianAnalyzer=650,495,676,367,855,1127 -org.apache.lucene.analysis.bg.TestBulgarianStemmer=24,17,19,33,21,30 -org.apache.lucene.analysis.br.TestBrazilianStemmer=1001,1049,988,999,1070,1993 -org.apache.lucene.analysis.ca.TestCatalanAnalyzer=706,685,584,549,958,2201 -org.apache.lucene.analysis.charfilter.HTMLStripCharFilterTest=2695,3133,3300,3025,3182,3919 -org.apache.lucene.analysis.charfilter.TestCharFilter=7,7,7,8,7,35 -org.apache.lucene.analysis.charfilter.TestMappingCharFilter=570,743,803,793,1080,31058 -org.apache.lucene.analysis.cjk.TestCJKAnalyzer=3532,3604,4338,4222,1182,4228 -org.apache.lucene.analysis.cjk.TestCJKTokenizer=2089,1496,1885,2154,718,835 -org.apache.lucene.analysis.cjk.TestCJKWidthFilter=457,274,281,236,388,1934 -org.apache.lucene.analysis.cn.TestChineseTokenizer=758,533,984,508,891,875 -org.apache.lucene.analysis.cn.smart.TestSmartChineseAnalyzer=2786,2596,2856,2701,2922,5185 -org.apache.lucene.analysis.commongrams.CommonGramsFilterTest=1368,1167,1111,1160,1655,3648 -org.apache.lucene.analysis.compound.TestCompoundWordTokenFilter=3824,3623,3616,3465,4362,3489 -org.apache.lucene.analysis.core.TestAnalyzers=1354,723,1030,1309,1028,10765 -org.apache.lucene.analysis.core.TestClassicAnalyzer=585,642,663,495,494,4536 -org.apache.lucene.analysis.core.TestDuelingAnalyzers=842,877,836,701,882,13901 -org.apache.lucene.analysis.core.TestKeywordAnalyzer=835,1197,1526,872,884,1065 -org.apache.lucene.analysis.core.TestRandomChains=21010 -org.apache.lucene.analysis.core.TestStandardAnalyzer=2581,2360,1844,2154,719,7580 -org.apache.lucene.analysis.core.TestStopAnalyzer=269,74,93,121,8,20 -org.apache.lucene.analysis.core.TestStopFilter=10,12,10,13,8,30 -org.apache.lucene.analysis.core.TestTypeTokenFilter=6,7,25,6,19,10 -org.apache.lucene.analysis.core.TestUAX29URLEmailAnalyzer=2755 -org.apache.lucene.analysis.core.TestUAX29URLEmailTokenizer=788,437,726,745,694,4569 -org.apache.lucene.analysis.cz.TestCzechAnalyzer=434,376,556,537,542,2365 -org.apache.lucene.analysis.cz.TestCzechStemmer=22,19,15,17,18,65 -org.apache.lucene.analysis.da.TestDanishAnalyzer=3705,3360,4310,2629,669,1038 -org.apache.lucene.analysis.de.TestGermanAnalyzer=742,653,578,594,455,1300 -org.apache.lucene.analysis.de.TestGermanLightStemFilter=387,433,686,613,4243,895 -org.apache.lucene.analysis.de.TestGermanMinimalStemFilter=355,651,748,619,4435,1035 -org.apache.lucene.analysis.de.TestGermanNormalizationFilter=216,449,507,273,207,1213 -org.apache.lucene.analysis.de.TestGermanStemFilter=3040,2483,2514,2397,4258,1200 -org.apache.lucene.analysis.el.GreekAnalyzerTest=933,990,971,862,1175,4872 -org.apache.lucene.analysis.el.TestGreekStemmer=30,59,36,43,48,50 -org.apache.lucene.analysis.en.TestEnglishAnalyzer=1166,762,1395,1258,2629,1815 -org.apache.lucene.analysis.en.TestEnglishMinimalStemFilter=337,397,243,348,270,665 -org.apache.lucene.analysis.en.TestKStemmer=928,1149,1222,816,1857,1885 -org.apache.lucene.analysis.en.TestPorterStemFilter=375,325,463,289,434,600 -org.apache.lucene.analysis.es.TestSpanishAnalyzer=357,431,434,386,1573,4871 -org.apache.lucene.analysis.es.TestSpanishLightStemFilter=326,451,277,329,658,1315 -org.apache.lucene.analysis.eu.TestBasqueAnalyzer=585,425,1382,823,681,985 -org.apache.lucene.analysis.fa.TestPersianAnalyzer=513,670,485,314,637,5303 -org.apache.lucene.analysis.fa.TestPersianNormalizationFilter=11,13,13,11,15,25 -org.apache.lucene.analysis.fi.TestFinnishAnalyzer=792,900,685,939,603,1267 -org.apache.lucene.analysis.fi.TestFinnishLightStemFilter=500,839,613,438,630,1415 -org.apache.lucene.analysis.fr.TestElision=4,5,5,4,6,15 -org.apache.lucene.analysis.fr.TestFrenchAnalyzer=668,1074,489,479,1104,1275 -org.apache.lucene.analysis.fr.TestFrenchLightStemFilter=643,418,493,555,209,840 -org.apache.lucene.analysis.fr.TestFrenchMinimalStemFilter=3714,2525,4491,2535,597,1721 -org.apache.lucene.analysis.ga.TestIrishAnalyzer=995 -org.apache.lucene.analysis.ga.TestIrishLowerCaseFilter=10 -org.apache.lucene.analysis.gl.TestGalicianAnalyzer=918,958,1136,960,3116,1260 -org.apache.lucene.analysis.gl.TestGalicianMinimalStemFilter=673,706,640,369,342,735 -org.apache.lucene.analysis.gl.TestGalicianStemFilter=438,322,578,285,172,462 -org.apache.lucene.analysis.hi.TestHindiAnalyzer=726,636,654,612,1081,1278 -org.apache.lucene.analysis.hi.TestHindiNormalizer=10,10,12,11,8,35 -org.apache.lucene.analysis.hi.TestHindiStemmer=11,9,10,10,8,20 -org.apache.lucene.analysis.hu.TestHungarianAnalyzer=742,767,715,697,791,1280 -org.apache.lucene.analysis.hu.TestHungarianLightStemFilter=572,686,989,735,636,447 -org.apache.lucene.analysis.hunspell.HunspellDictionaryTest=12,12,12,14,42,12 -org.apache.lucene.analysis.hunspell.HunspellStemFilterTest=1327,1212,1648,1306,1881,1265 -org.apache.lucene.analysis.hunspell.HunspellStemmerTest=48,19,18,14,22,140 -org.apache.lucene.analysis.hy.TestArmenianAnalyzer=1002,487,903,580,1429,965 -org.apache.lucene.analysis.icu.TestICUFoldingFilter=2056,2573,1483,2241,3760,4343 -org.apache.lucene.analysis.icu.TestICUNormalizer2Filter=2187,2335,1831,1592,2768,4570 -org.apache.lucene.analysis.icu.TestICUTransformFilter=2678,4159,3243,3388,5000,4689 -org.apache.lucene.analysis.icu.segmentation.TestCharArrayIterator=26,53,21,19,17,82 -org.apache.lucene.analysis.icu.segmentation.TestICUTokenizer=2399,3110,2330,2314,4708,5943 -org.apache.lucene.analysis.icu.segmentation.TestLaoBreakIterator=79,36,111,148,11,380 -org.apache.lucene.analysis.icu.segmentation.TestWithCJKBigramFilter=673,707,760,667,35,175 -org.apache.lucene.analysis.id.TestIndonesianAnalyzer=421,491,665,536,621,1515 -org.apache.lucene.analysis.id.TestIndonesianStemmer=13,12,13,14,259,313 -org.apache.lucene.analysis.in.TestIndicNormalizer=14,14,14,9,12,30 -org.apache.lucene.analysis.it.TestItalianAnalyzer=1176,1091,1033,1204,1064,729 -org.apache.lucene.analysis.it.TestItalianLightStemFilter=427,268,449,493,720,924 -org.apache.lucene.analysis.ja.TestExtendedMode=11051 -org.apache.lucene.analysis.ja.TestJapaneseAnalyzer=2932 -org.apache.lucene.analysis.ja.TestJapaneseBaseFormFilter=5867 -org.apache.lucene.analysis.ja.TestJapaneseKatakanaStemFilter=4892 -org.apache.lucene.analysis.ja.TestJapaneseReadingFormFilter=2902 -org.apache.lucene.analysis.ja.TestJapaneseTokenizer=21035 -org.apache.lucene.analysis.ja.TestSearchMode=585 -org.apache.lucene.analysis.ja.dict.TestTokenInfoDictionary=1147 -org.apache.lucene.analysis.ja.dict.UserDictionaryTest=50 -org.apache.lucene.analysis.ja.util.TestToStringUtil=11 -org.apache.lucene.analysis.kuromoji.SegmenterTest=319,248,268,310,489 -org.apache.lucene.analysis.kuromoji.TestExtendedMode=1506,1174,1537,1780,2333 -org.apache.lucene.analysis.kuromoji.TestKuromojiAnalyzer=3668,3597,3703,3410,4044 -org.apache.lucene.analysis.kuromoji.TestKuromojiBaseFormFilter=2908,3299,3001,2800,3975 -org.apache.lucene.analysis.kuromoji.TestKuromojiTokenizer=4494,4640,4435,4159,4637 -org.apache.lucene.analysis.kuromoji.TestSearchMode=494,580,501,520,102 -org.apache.lucene.analysis.kuromoji.dict.TestTokenInfoDictionary=1406,1319,1249,1180,1187 -org.apache.lucene.analysis.kuromoji.dict.UserDictionaryTest=276,258,257,232,43 -org.apache.lucene.analysis.kuromoji.util.TestToStringUtil=17,14,9,24,11 -org.apache.lucene.analysis.lv.TestLatvianAnalyzer=655,605,760,451,757,1147 -org.apache.lucene.analysis.lv.TestLatvianStemmer=35,20,29,26,14,45 -org.apache.lucene.analysis.miscellaneous.PatternAnalyzerTest=882,936,1032,832,948,1515 -org.apache.lucene.analysis.miscellaneous.TestASCIIFoldingFilter=416,454,442,385,399,975 -org.apache.lucene.analysis.miscellaneous.TestCapitalizationFilter=294,479,376,593,806,1020 -org.apache.lucene.analysis.miscellaneous.TestEmptyTokenStream=5,4,6,3,3,20 -org.apache.lucene.analysis.miscellaneous.TestHyphenatedWordsFilter=813,427,434,750,406,1540 -org.apache.lucene.analysis.miscellaneous.TestKeepWordFilter=227,209,158,208,234,560 -org.apache.lucene.analysis.miscellaneous.TestKeywordMarkerFilter=7,6,5,5,248,128 -org.apache.lucene.analysis.miscellaneous.TestLengthFilter=21,4,5,6,5,60 -org.apache.lucene.analysis.miscellaneous.TestLimitTokenCountAnalyzer=129,152,173,190,89,516 -org.apache.lucene.analysis.miscellaneous.TestPerFieldAnalzyerWrapper=7,7,6,26,5,28 -org.apache.lucene.analysis.miscellaneous.TestPrefixAndSuffixAwareTokenFilter=18,39,26,12,7,10 -org.apache.lucene.analysis.miscellaneous.TestPrefixAwareTokenFilter=7,8,5,8,7,5 -org.apache.lucene.analysis.miscellaneous.TestRemoveDuplicatesTokenFilter=1223,1192,1311,1130,3032,7199 -org.apache.lucene.analysis.miscellaneous.TestSingleTokenTokenFilter=5,4,20,4,2,60 -org.apache.lucene.analysis.miscellaneous.TestStemmerOverrideFilter=4,4,4,4,3,5 -org.apache.lucene.analysis.miscellaneous.TestTrimFilter=572,423,380,498,858,3434 -org.apache.lucene.analysis.miscellaneous.TestWordDelimiterFilter=3187,2709,3407,2942,3216,5573 -org.apache.lucene.analysis.morfologik.TestMorfologikAnalyzer=1490,1559,1676,2357,1993,1974 -org.apache.lucene.analysis.ngram.EdgeNGramTokenFilterTest=3660,4136,3892,2591,1716,3785 -org.apache.lucene.analysis.ngram.EdgeNGramTokenizerTest=1927,2088,2489,1295,1208,5075 -org.apache.lucene.analysis.ngram.NGramTokenFilterTest=5941,4022,5224,5217,8889,4869 -org.apache.lucene.analysis.ngram.NGramTokenizerTest=12649,13480,11157,11566,13624,32608 -org.apache.lucene.analysis.nl.TestDutchStemmer=1493,1169,1590,1109,705,1045 -org.apache.lucene.analysis.no.TestNorwegianAnalyzer=686,594,827,894,791,1154 -org.apache.lucene.analysis.no.TestNorwegianLightStemFilter=605 -org.apache.lucene.analysis.no.TestNorwegianMinimalStemFilter=944 -org.apache.lucene.analysis.path.TestPathHierarchyTokenizer=674,631,649,565,468,1720 -org.apache.lucene.analysis.path.TestReversePathHierarchyTokenizer=488,376,567,332,481,2100 -org.apache.lucene.analysis.pattern.TestPatternReplaceCharFilter=1486,1425,1646,1598,3624,2207 -org.apache.lucene.analysis.pattern.TestPatternReplaceFilter=1159,1067,824,1219,1089,2137 -org.apache.lucene.analysis.pattern.TestPatternTokenizer=943,1426,1328,1555,656,2295 -org.apache.lucene.analysis.payloads.DelimitedPayloadTokenFilterTest=55,14,16,19,10,15 -org.apache.lucene.analysis.payloads.NumericPayloadTokenFilterTest=5,7,4,5,6,15 -org.apache.lucene.analysis.payloads.TokenOffsetPayloadTokenFilterTest=6,4,7,5,5,10 -org.apache.lucene.analysis.payloads.TypeAsPayloadTokenFilterTest=46,28,33,35,8,10 -org.apache.lucene.analysis.phonetic.DoubleMetaphoneFilterTest=1586,1861,1666,1821,1946,2345 -org.apache.lucene.analysis.phonetic.TestBeiderMorseFilter=952,792,1084,945,1038,1133 -org.apache.lucene.analysis.phonetic.TestPhoneticFilter=2858,2913,3046,2568,3197,4524 -org.apache.lucene.analysis.pl.TestPolishAnalyzer=1618,1484,1492,1569,1677,1607 -org.apache.lucene.analysis.position.PositionFilterTest=17,10,13,14,9,30 -org.apache.lucene.analysis.pt.TestPortugueseAnalyzer=497,415,382,407,504,955 -org.apache.lucene.analysis.pt.TestPortugueseLightStemFilter=585,664,657,584,484,1735 -org.apache.lucene.analysis.pt.TestPortugueseMinimalStemFilter=525,504,624,669,1655,1085 -org.apache.lucene.analysis.pt.TestPortugueseStemFilter=1264,1229,1397,1195,1292,2315 -org.apache.lucene.analysis.query.QueryAutoStopWordAnalyzerTest=206,272,382,330,590,215 -org.apache.lucene.analysis.reverse.TestReverseStringFilter=525,278,426,417,299,1030 -org.apache.lucene.analysis.ro.TestRomanianAnalyzer=966,597,972,684,1142,1590 -org.apache.lucene.analysis.ru.TestRussianAnalyzer=612,725,520,454,791,1700 -org.apache.lucene.analysis.ru.TestRussianLetterTokenizer=6,7,10,10,4,299 -org.apache.lucene.analysis.ru.TestRussianLightStemFilter=438,567,485,602,723,1870 -org.apache.lucene.analysis.shingle.ShingleAnalyzerWrapperTest=657,645,557,594,917,840 -org.apache.lucene.analysis.shingle.ShingleFilterTest=732,782,903,813,2519,8333 -org.apache.lucene.analysis.sinks.DateRecognizerSinkTokenizerTest=40,20,44,15,14,335 -org.apache.lucene.analysis.sinks.TestTeeSinkTokenFilter=101,70,174,32,321,70 -org.apache.lucene.analysis.sinks.TokenRangeSinkTokenizerTest=3,6,4,6,4,10 -org.apache.lucene.analysis.sinks.TokenTypeSinkTokenizerTest=48,10,6,17,15,10 -org.apache.lucene.analysis.snowball.TestSnowball=44,97,41,166,27,120 -org.apache.lucene.analysis.snowball.TestSnowballVocab=3614,3519,3640,3611,4575,7073 -org.apache.lucene.analysis.sv.TestSwedishAnalyzer=576,1216,1640,697,581,4070 -org.apache.lucene.analysis.sv.TestSwedishLightStemFilter=372,328,682,377,4413,2260 -org.apache.lucene.analysis.synonym.TestSolrSynonymParser=16,31,15,33,29,110 -org.apache.lucene.analysis.synonym.TestSynonymMapFilter=1614,1807,1181,1745,907,24940 -org.apache.lucene.analysis.synonym.TestWordnetSynonymParser=7,36,35,7,33,390 -org.apache.lucene.analysis.th.TestThaiAnalyzer=477,454,489,434,743,7038 -org.apache.lucene.analysis.tokenattributes.TestCharTermAttributeImpl=964,788,1020,653,650,1254 -org.apache.lucene.analysis.tokenattributes.TestSimpleAttributeImpl=5,5,3,4,4,6 -org.apache.lucene.analysis.tr.TestTurkishAnalyzer=656,857,623,697,973,1985 -org.apache.lucene.analysis.tr.TestTurkishLowerCaseFilter=7,7,6,7,7,59 -org.apache.lucene.analysis.uima.UIMABaseAnalyzerTest=6723,3808 -org.apache.lucene.analysis.uima.UIMATypeAwareAnalyzerTest=2287,6515 -org.apache.lucene.analysis.uima.ae.BasicAEProviderTest=368,378 -org.apache.lucene.analysis.uima.ae.OverridingParamsAEProviderTest=1842,1578 -org.apache.lucene.analysis.util.TestCharArrayIterator=179,262,165,197,147,185 -org.apache.lucene.analysis.util.TestCharArrayMap=70,133,99,92,134,210 -org.apache.lucene.analysis.util.TestCharArraySet=43,42,52,41,39,129 -org.apache.lucene.analysis.util.TestCharTokenizers=880,1051,979,734,906,2057 -org.apache.lucene.analysis.util.TestCharacterUtils=17,16,13,13,12,40 -org.apache.lucene.analysis.util.TestSegmentingTokenizerBase=705,748,931,1103,1144 -org.apache.lucene.analysis.util.TestWordlistLoader=7,6,13,7,9,15 -org.apache.lucene.analysis.wikipedia.WikipediaTokenizerTest=766,706,816,649,677,3740 -org.apache.lucene.benchmark.byTask.TestPerfTasksLogic=6924,7297,7622,7434,8195,8831 -org.apache.lucene.benchmark.byTask.TestPerfTasksParse=173,110,230,139,1453,1205 -org.apache.lucene.benchmark.byTask.feeds.DocMakerTest=319,392,327,275,318,838 -org.apache.lucene.benchmark.byTask.feeds.LineDocSourceTest=1097,1408,2193,2224,1720,3454 -org.apache.lucene.benchmark.byTask.feeds.TrecContentSourceTest=114,110,89,116,356,150 -org.apache.lucene.benchmark.byTask.feeds.demohtml.TestHtmlParser=261,254,244,282,49,330 -org.apache.lucene.benchmark.byTask.tasks.CreateIndexTaskTest=607,591,631,512,523,1078 -org.apache.lucene.benchmark.byTask.tasks.PerfTaskTest=194,212,224,182,781,256 -org.apache.lucene.benchmark.byTask.tasks.SearchWithSortTaskTest=75,109,86,103,115,342 -org.apache.lucene.benchmark.byTask.tasks.WriteLineDocTaskTest=443,616,929,364,470,657 -org.apache.lucene.benchmark.byTask.tasks.alt.AltPackageTaskTest=83,68,80,104,320,95 -org.apache.lucene.benchmark.byTask.utils.StreamUtilsTest=406,412,380,379,163,395 -org.apache.lucene.benchmark.byTask.utils.TestConfig=189,190,179,208,5,14 -org.apache.lucene.benchmark.quality.TestQualityRun=3117,1965,3256,3455,3779,3937 -org.apache.lucene.codecs.appending.TestAppendingCodec=18,55,26,28,96,20 -org.apache.lucene.codecs.intblock.TestIntBlockCodec=9,12,9,9,9,11 -org.apache.lucene.codecs.lucene3x.TestImpersonation=2,2,2,3,2,34 -org.apache.lucene.codecs.lucene3x.TestSurrogates=735,321,102,370,812,756 -org.apache.lucene.codecs.lucene3x.TestTermInfosReaderIndex=170,334,143,293,667,1017 -org.apache.lucene.codecs.lucene40.TestBitVector=653,739,937,911,802,1458 -org.apache.lucene.codecs.lucene40.TestDocValues=462,425,353,286 -org.apache.lucene.codecs.lucene40.TestReuseDocsEnum=554,620,247,460,562,517 -org.apache.lucene.codecs.lucene40.values.TestDocValues=200,346 -org.apache.lucene.codecs.perfield.TestPerFieldPostingsFormat=1142,741,765,1638,319,432 -org.apache.lucene.codecs.pulsing.Test10KPulsings=4,4,2,2,2,3 -org.apache.lucene.codecs.pulsing.TestPulsingReuse=10,53,10,9,16,310 -org.apache.lucene.collation.TestCollationKeyAnalyzer=1456,1241,790,206,789,440 -org.apache.lucene.collation.TestCollationKeyFilter=478,425,596,668,588,335 -org.apache.lucene.collation.TestICUCollationKeyAnalyzer=1340,681,3304,1934,1429,465 -org.apache.lucene.collation.TestICUCollationKeyFilter=1100,1118,1489,927,911,935 -org.apache.lucene.demo.TestDemo=531,969,964,985,533,597 -org.apache.lucene.document.TestBinaryDocument=12,11,10,11,26,15 -org.apache.lucene.document.TestDateTools=9,11,18,19,14,27 -org.apache.lucene.document.TestDocument=191,36,36,451,53,52 -org.apache.lucene.facet.enhancements.EnhancementsPayloadIteratorTest=59,40,66,274,57,180 -org.apache.lucene.facet.enhancements.TwoEnhancementsTest=115,79,203,89,503,61 -org.apache.lucene.facet.enhancements.association.AssociationPropertyTest=8,7,6,8,10,25 -org.apache.lucene.facet.enhancements.association.CustomAssociationPropertyTest=33,40,42,28,91,180 -org.apache.lucene.facet.enhancements.params.DefaultEnhancementsIndexingParamsTest=4,7,4,5,5,224 -org.apache.lucene.facet.example.TestAdaptiveExample=52,30,41,67,22,32 -org.apache.lucene.facet.example.TestAssociationExample=234,368,263,249,124,199 -org.apache.lucene.facet.example.TestMultiCLExample=167,145,137,271,17,654 -org.apache.lucene.facet.example.TestSimpleExample=54,56,71,78,55,206 -org.apache.lucene.facet.index.CategoryContainerTest=28,224,33,48,55,123 -org.apache.lucene.facet.index.CategoryListPayloadStreamTest=5,5,5,5,5,15 -org.apache.lucene.facet.index.FacetsPayloadProcessorProviderTest=593,929,672,587,2909,1345 -org.apache.lucene.facet.index.attributes.CategoryAttributeImplTest=7,10,8,8,10,75 -org.apache.lucene.facet.index.attributes.CategoryAttributesIterableTest=4,5,4,5,1,105 -org.apache.lucene.facet.index.categorypolicy.OrdinalPolicyTest=370,1572,853,415,1606,1359 -org.apache.lucene.facet.index.categorypolicy.PathPolicyTest=1568,547,899,708,885,103 -org.apache.lucene.facet.index.params.CategoryListParamsTest=6,7,7,7,12,17 -org.apache.lucene.facet.index.params.DefaultFacetIndexingParamsTest=9,7,7,5,7,15 -org.apache.lucene.facet.index.params.PerDimensionIndexingParamsTest=16,17,18,15,8,140 -org.apache.lucene.facet.index.streaming.CategoryAttributesStreamTest=9,8,12,9,3,9 -org.apache.lucene.facet.index.streaming.CategoryParentsStreamTest=98,129,127,166,107,112 -org.apache.lucene.facet.index.streaming.CategoryTokenizerTest=311,115,29,28,15,20 -org.apache.lucene.facet.search.AdaptiveAccumulatorTest=22499,11832,13654,10852,35241,17902 -org.apache.lucene.facet.search.CategoryListIteratorTest=307,413,284,289,114,53 -org.apache.lucene.facet.search.DrillDownTest=88,94,148,94,42,92 -org.apache.lucene.facet.search.SamplingWrapperTest=23585,14646,14012,15905,33551,17013 -org.apache.lucene.facet.search.TestCategoryListCache=57,58,59,123,582,48 -org.apache.lucene.facet.search.TestFacetArrays=3,3,2,3,2,82 -org.apache.lucene.facet.search.TestFacetsAccumulatorWithComplement=198,186,249,181,79,768 -org.apache.lucene.facet.search.TestMultipleCategoryLists=574,343,326,354,210,614 -org.apache.lucene.facet.search.TestScoredDocIdCollector=125,104,59,68,120,145 -org.apache.lucene.facet.search.TestTopKInEachNodeResultHandler=2707,605,816,972,455,989 -org.apache.lucene.facet.search.TestTopKResultsHandler=298,565,254,1042,510,532 -org.apache.lucene.facet.search.TestTopKResultsHandlerRandom=9714,7837,8206,14089,4201,5412 -org.apache.lucene.facet.search.TestTotalFacetCounts=287,1307,202,271,88,1423 -org.apache.lucene.facet.search.TestTotalFacetCountsCache=1568,3359,3670,3162,3095,1322 -org.apache.lucene.facet.search.association.AssociationsFacetRequestTest=303,112,112,246,120,87 -org.apache.lucene.facet.search.params.FacetRequestTest=63,47,43,74,49,25 -org.apache.lucene.facet.search.params.FacetSearchParamsTest=32,307,40,45,25,573 -org.apache.lucene.facet.search.params.MultiIteratorsPerCLParamsTest=56,56,108,71,33,123 -org.apache.lucene.facet.search.sampling.SamplingAccumulatorTest=17217,17997,16760,19828,33173,15443 -org.apache.lucene.facet.taxonomy.TestCategoryPath=296,287,268,268,84,610 -org.apache.lucene.facet.taxonomy.TestTaxonomyCombined=1924,2478,1944,2179,1228,2005 -org.apache.lucene.facet.taxonomy.directory.TestAddTaxonomies=592,571,2415,1608,3067,1561 -org.apache.lucene.facet.taxonomy.directory.TestDirectoryTaxonomyReader=905,996,835,943,136,438 -org.apache.lucene.facet.taxonomy.directory.TestDirectoryTaxonomyWriter=172,110,89,276,156,393 -org.apache.lucene.facet.taxonomy.directory.TestIndexClose=2324,3257,2841,2855,4987,1320 -org.apache.lucene.facet.taxonomy.writercache.cl2o.TestCharBlockArray=1094,1019,1165,1022,585,2117 -org.apache.lucene.facet.taxonomy.writercache.cl2o.TestCompactLabelToOrdinal=891,813,1331,836,754,928 -org.apache.lucene.facet.util.TestScoredDocIDsUtils=1435,1312,1768,993,1130,1569 -org.apache.lucene.index.Test2BPostings=1,1,1,2,1,2 -org.apache.lucene.index.Test2BTerms=2,3,2,2,4,3 -org.apache.lucene.index.TestAddIndexes=5656,8210,11930,11938,12789,3705 -org.apache.lucene.index.TestAtomicUpdate=2803,3435,1655,3604,3545,3787 -org.apache.lucene.index.TestBackwardsCompatibility=6788,6173,6638,7781,13854,5551 -org.apache.lucene.index.TestBinaryTerms=28,52,27,65,122,28 -org.apache.lucene.index.TestByteSlices=2116,2102,2677,2579,2281,4823 -org.apache.lucene.index.TestCheckIndex=23,23,13,414,434,130 -org.apache.lucene.index.TestCodecs=342,342,191,774,571,273 -org.apache.lucene.index.TestCompoundFile=2800,4008,4823,2880,2807,2756 -org.apache.lucene.index.TestConcurrentMergeScheduler=804,1182,1384,3413,1822,595 -org.apache.lucene.index.TestConsistentFieldNumbers=802,567,391,346,1390,499 -org.apache.lucene.index.TestCrash=441,624,261,491,1553,1050 -org.apache.lucene.index.TestCrashCausesCorruptIndex=157,240,144,213,458,120 -org.apache.lucene.index.TestCustomNorms=711,368,588,216,789,159 -org.apache.lucene.index.TestDeletionPolicy=8116,4136,5647,4380,4235,4265 -org.apache.lucene.index.TestDirectoryReader=1986,5251,1396,723,813,1540 -org.apache.lucene.index.TestDirectoryReaderReopen=4353,1720 -org.apache.lucene.index.TestDoc=1034,284,340,315,565,536 -org.apache.lucene.index.TestDocCount=34,58,33,33,353,51 -org.apache.lucene.index.TestDocTermOrds=149,266,156,160,1326,85 -org.apache.lucene.index.TestDocValuesIndexing=5176,2982,215,3811,3242,1539 -org.apache.lucene.index.TestDocsAndPositions=1747,1950,1671,1495,779,379 -org.apache.lucene.index.TestDocumentWriter=185,617,472,256,146,235 -org.apache.lucene.index.TestDocumentsWriterDeleteQueue=367,923,523,375,1085,111 -org.apache.lucene.index.TestDuelingCodecs=4296,9589,3472,5808,3645,6374 -org.apache.lucene.index.TestFieldInfos=6,6,4,4,5,7 -org.apache.lucene.index.TestFieldsReader=750,7225,569,600,974,810 -org.apache.lucene.index.TestFilterAtomicReader=97,137 -org.apache.lucene.index.TestFilterIndexReader=24,16,55,19 -org.apache.lucene.index.TestFlex=308,764,1203,291,265,149 -org.apache.lucene.index.TestFlushByRamOrCountsPolicy=7133,12285,6734,7873,3506,4316 -org.apache.lucene.index.TestForTooMuchCloning=185,316,329,124,1820,1067 -org.apache.lucene.index.TestForceMergeForever=756,2368,739,425,2869,1932 -org.apache.lucene.index.TestIndexCommit=2,3,17,2,2,8 -org.apache.lucene.index.TestIndexFileDeleter=29,109,24,29,30,93 -org.apache.lucene.index.TestIndexInput=3,3,5,3,2,337 -org.apache.lucene.index.TestIndexReader=1960,2129,2413,1600 -org.apache.lucene.index.TestIndexReaderReopen=1864,2585,2300,1574 -org.apache.lucene.index.TestIndexSplitter=1331,1457,1517,1413,1437,1630 -org.apache.lucene.index.TestIndexWriter=6282,11908,12177,8100,4809,8940 -org.apache.lucene.index.TestIndexWriterCommit=2366,1676,2636,2646,2383,1154 -org.apache.lucene.index.TestIndexWriterConfig=7,7,7,12,8,17 -org.apache.lucene.index.TestIndexWriterDelete=2281,2558,1979,3413,3524,1779 -org.apache.lucene.index.TestIndexWriterExceptions=3568,4436,4135,3765,6833,10861 -org.apache.lucene.index.TestIndexWriterForceMerge=2330,2051,2361,2155,4869,2636 -org.apache.lucene.index.TestIndexWriterLockRelease=10,8,101,6,6,11 -org.apache.lucene.index.TestIndexWriterMergePolicy=3397,2502,5830,3046,2464,2072 -org.apache.lucene.index.TestIndexWriterMerging=2561,3549,5304,4152,8030,3091 -org.apache.lucene.index.TestIndexWriterNRTIsCurrent=634,556,680,1152,1742,873 -org.apache.lucene.index.TestIndexWriterOnDiskFull=867,1106,335,2372,509,1599 -org.apache.lucene.index.TestIndexWriterOnJRECrash=3,4,3,3,2,3 -org.apache.lucene.index.TestIndexWriterReader=6795,15686,11066,16046,11674,19233 -org.apache.lucene.index.TestIndexWriterUnicode=1721,2844,1810,2625,2442,2611 -org.apache.lucene.index.TestIndexWriterWithThreads=7079,4587,9820,5817,5874,4717 -org.apache.lucene.index.TestIndexableField=382,1181,634,1124,1582,110 -org.apache.lucene.index.TestIsCurrent=11,12,7,13,16,29 -org.apache.lucene.index.TestLazyProxSkipping=47,750,403,549,99,3851 -org.apache.lucene.index.TestLongPostings=1844,3316,1490,1450,1608,4027 -org.apache.lucene.index.TestMaxTermFrequency=1081,1055,573,974,812,149 -org.apache.lucene.index.TestMixedCodecs=1293,1986,1292,9,1985,1328 -org.apache.lucene.index.TestMultiFields=268,88,179,1371,219,145 -org.apache.lucene.index.TestMultiLevelSkipList=341,117,186,268,142,201 -org.apache.lucene.index.TestMultiPassIndexSplitter=742,711,832,773,881,864 -org.apache.lucene.index.TestMultiReader=687,686,580,568 -org.apache.lucene.index.TestNRTReaderWithThreads=1493,2447,3215,1192,2467,2221 -org.apache.lucene.index.TestNRTThreads=5816,4106,3114,48,4373,5754 -org.apache.lucene.index.TestNeverDelete=2035,2120,1066,1972,2137,1247 -org.apache.lucene.index.TestNewestSegment=5,5,5,4,4,10 -org.apache.lucene.index.TestNoDeletionPolicy=297,64,59,58,140,54 -org.apache.lucene.index.TestNoMergePolicy=5,7,6,5,7,12 -org.apache.lucene.index.TestNoMergeScheduler=5,5,4,5,7,95 -org.apache.lucene.index.TestNorms=5100,1844,6351,2084,10684,2256 -org.apache.lucene.index.TestOmitNorms=1194,1404,2306,3802,795,2799 -org.apache.lucene.index.TestOmitPositions=369,229,82,130,137,120 -org.apache.lucene.index.TestOmitTf=1176,841,685,1187,1272,1167 -org.apache.lucene.index.TestPKIndexSplitter=937,882,857,788,921,1014 -org.apache.lucene.index.TestParallelAtomicReader=88,135 -org.apache.lucene.index.TestParallelCompositeReader=1632,649 -org.apache.lucene.index.TestParallelReader=115,87,70,204 -org.apache.lucene.index.TestParallelReaderEmptyIndex=39,64,56,31,127,98 -org.apache.lucene.index.TestParallelTermEnum=37,54,11,18,18,55 -org.apache.lucene.index.TestPayloadProcessorProvider=607,266,359,738,344,263 -org.apache.lucene.index.TestPayloads=95,696,61,67,213,315 -org.apache.lucene.index.TestPerSegmentDeletes=32,35,37,56,46,736 -org.apache.lucene.index.TestPersistentSnapshotDeletionPolicy=3713,3673,2795,2904,2385,2434 -org.apache.lucene.index.TestPostingsOffsets=39,33,1867,854,561,265 -org.apache.lucene.index.TestPrefixCodedTerms=357,210,411,646,366,562 -org.apache.lucene.index.TestRandomStoredFields=931,1020,3080,967,679,1555 -org.apache.lucene.index.TestReaderClosed=6,33,7,6,11,22 -org.apache.lucene.index.TestRollback=9,8,10,13,12,8 -org.apache.lucene.index.TestRollingUpdates=2867,832,724,1204,1141,1318 -org.apache.lucene.index.TestSameTokenSamePosition=64,34,30,43,23,66 -org.apache.lucene.index.TestSegmentMerger=348,472,269,819,463,548 -org.apache.lucene.index.TestSegmentReader=926,1133,1043,2495,1848,539 -org.apache.lucene.index.TestSegmentTermDocs=7228,843,10592,5894,628,1026 -org.apache.lucene.index.TestSegmentTermEnum=24,33,18,18,22,28 -org.apache.lucene.index.TestSizeBoundedForceMerge=154,153,40,182,127,317 -org.apache.lucene.index.TestSnapshotDeletionPolicy=1564,1644,1400,1354,2429,1607 -org.apache.lucene.index.TestStressAdvance=2038,1563,2260,1950,2734,885 -org.apache.lucene.index.TestStressIndexing=2026,2479,1748,1294,1984,1558 -org.apache.lucene.index.TestStressIndexing2=379,453,362,552,1449,436 -org.apache.lucene.index.TestStressNRT=967,1891,309,21646,2021,2231 -org.apache.lucene.index.TestSumDocFreq=130,126,157,42,137,525 -org.apache.lucene.index.TestTerm=2,3,2,3,230,4 -org.apache.lucene.index.TestTermVectorsReader=42,54,69,79,92,139 -org.apache.lucene.index.TestTermVectorsWriter=120,453,140,140,264,180 -org.apache.lucene.index.TestTermdocPerf=2,4,3,2,2,10 -org.apache.lucene.index.TestTermsEnum=4912,5669,5943,5380,4535,4739 -org.apache.lucene.index.TestTermsEnum2=1406,1819,3517,2180,547,430 -org.apache.lucene.index.TestThreadedForceMerge=762,750,1501,1700,1139,732 -org.apache.lucene.index.TestTieredMergePolicy=2463,1015,1707,1465,2645,1632 -org.apache.lucene.index.TestTransactionRollback=112,115,119,200,631,772 -org.apache.lucene.index.TestTransactions=623,800,718,779,1580,689 -org.apache.lucene.index.TestTypePromotion=3549,34,2525,35,4727,855 -org.apache.lucene.index.TestUniqueTermCount=29,42,18,25,22,34 -org.apache.lucene.index.memory.MemoryIndexTest=4979,4492,4902,4646,5152,3996 -org.apache.lucene.misc.SweetSpotSimilarityTest=315,312,308,336,345,367 -org.apache.lucene.misc.TestHighFreqTerms=1217,397,527,636,514,590 -org.apache.lucene.queries.BooleanFilterTest=908,1097,970,728,478,1003 -org.apache.lucene.queries.BoostingQueryTest=216,201,213,197,5,17 -org.apache.lucene.queries.ChainedFilterTest=1250,1188,1264,1057,2016,1883 -org.apache.lucene.queries.TermsFilterTest=501,684,516,487,107,576 -org.apache.lucene.queries.TestCustomScoreQuery=1913,2408,3202,2145,2669,7237 -org.apache.lucene.queries.function.TestFieldScoreQuery=515,402,500,420,974,418 -org.apache.lucene.queries.function.TestOrdValues=563,644,648,670,661,352 -org.apache.lucene.queries.mlt.TestMoreLikeThis=104,46,74,103,423,44 -org.apache.lucene.queryparser.analyzing.TestAnalyzingQueryParser=24,16,19,26,17,71 -org.apache.lucene.queryparser.classic.TestMultiAnalyzer=20,28,20,16,29,369 -org.apache.lucene.queryparser.classic.TestMultiFieldQueryParser=270,287,289,181,680,210 -org.apache.lucene.queryparser.classic.TestMultiPhraseQueryParsing=6,5,8,11,22,26 -org.apache.lucene.queryparser.classic.TestQueryParser=1152,998,1010,1173,580,539 -org.apache.lucene.queryparser.complexPhrase.TestComplexPhraseQuery=126,121,93,286,75,103 -org.apache.lucene.queryparser.ext.TestExtendableQueryParser=611,526,511,566,870,1217 -org.apache.lucene.queryparser.ext.TestExtensions=35,66,36,55,16,32 -org.apache.lucene.queryparser.flexible.core.builders.TestQueryTreeBuilder=45,8,7,9,8,14 -org.apache.lucene.queryparser.flexible.core.nodes.TestQueryNode=210,263,231,222,8,60 -org.apache.lucene.queryparser.flexible.messages.TestNLS=56,23,30,33,22,308 -org.apache.lucene.queryparser.flexible.precedence.TestPrecedenceQueryParser=387,476,528,392,124,549 -org.apache.lucene.queryparser.flexible.spans.TestSpanQueryParser=18,17,46,23,334,103 -org.apache.lucene.queryparser.flexible.spans.TestSpanQueryParserSimpleSample=9,27,12,10,28,15 -org.apache.lucene.queryparser.flexible.standard.TestMultiAnalyzerQPHelper=23,74,20,22,103,442 -org.apache.lucene.queryparser.flexible.standard.TestMultiFieldQPHelper=198,96,96,116,146,322 -org.apache.lucene.queryparser.flexible.standard.TestNumericQueryParser=556,462,506,612,401,704 -org.apache.lucene.queryparser.flexible.standard.TestQPHelper=763,803,1054,799,464,337 -org.apache.lucene.queryparser.surround.query.SrndQueryTest=461,516,420,424,54,29 -org.apache.lucene.queryparser.surround.query.Test01Exceptions=302,221,283,220,23,15 -org.apache.lucene.queryparser.surround.query.Test02Boolean=396,389,336,338,252,897 -org.apache.lucene.queryparser.surround.query.Test03Distance=1024,958,1189,1165,1414,2262 -org.apache.lucene.queryparser.xml.TestParser=1149,1577,1520,1789,2335,2309 -org.apache.lucene.queryparser.xml.TestQueryTemplateManager=401,293,332,279,353,882 -org.apache.lucene.queryparser.xml.builders.TestNumericRangeFilterBuilder=93,112,127,115,97,135 -org.apache.lucene.queryparser.xml.builders.TestNumericRangeQueryBuilder=62,56,66,65,193,96 -org.apache.lucene.sandbox.queries.DuplicateFilterTest=678,606,654,565,699,748 -org.apache.lucene.sandbox.queries.FuzzyLikeThisQueryTest=542,701,598,600,173,224 -org.apache.lucene.sandbox.queries.TestSlowCollationMethods=1471,1366,1464,1921,1697,2186 -org.apache.lucene.sandbox.queries.regex.TestJakartaRegexpCapabilities=272,222,211,234,6,25 -org.apache.lucene.sandbox.queries.regex.TestRegexQuery=479,531,452,439,741,886 -org.apache.lucene.sandbox.queries.regex.TestSpanRegexQuery=65,95,93,49,448,531 -org.apache.lucene.search.MultiCollectorTest=5,4,6,6,7,49 -org.apache.lucene.search.TestAutomatonQuery=320,204,62,61,668,399 -org.apache.lucene.search.TestAutomatonQueryUnicode=13,9,13,19,79,15 -org.apache.lucene.search.TestBoolean2=4798,1549,5126,5695,4014,17340 -org.apache.lucene.search.TestBooleanMinShouldMatch=427,1121,631,1473,1326,1728 -org.apache.lucene.search.TestBooleanOr=330,573,679,1027,455,993 -org.apache.lucene.search.TestBooleanQuery=135,100,289,120,182,72 -org.apache.lucene.search.TestBooleanScorer=61,14,21,28,116,14 -org.apache.lucene.search.TestCachingCollector=10,11,11,13,282,14 -org.apache.lucene.search.TestCachingWrapperFilter=103,78,25,31,25,180 -org.apache.lucene.search.TestComplexExplanations=1822,983,1373,1172,1204,1592 -org.apache.lucene.search.TestComplexExplanationsOfNonMatches=77,37,35,38,66,146 -org.apache.lucene.search.TestConstantScoreQuery=34,13,21,12,15,13 -org.apache.lucene.search.TestCustomSearcherSort=683,812,843,990,1451,1367 -org.apache.lucene.search.TestDateFilter=28,15,18,23,39,31 -org.apache.lucene.search.TestDateSort=22,13,15,11,19,10 -org.apache.lucene.search.TestDisjunctionMaxQuery=2407,930,1474,1290,1400,600 -org.apache.lucene.search.TestDocBoost=8,7,8,13,8,40 -org.apache.lucene.search.TestDocIdSet=25,8,12,12,9,18 -org.apache.lucene.search.TestDocValuesScoring=36,26,10,44,59,220 -org.apache.lucene.search.TestElevationComparator=18,35,18,25,391,110 -org.apache.lucene.search.TestExplanations=12,10,9,7,10,12 -org.apache.lucene.search.TestFieldCache=3264,786,664,502,1150,1279 -org.apache.lucene.search.TestFieldCacheRangeFilter=497,690,196,660,483,237 -org.apache.lucene.search.TestFieldCacheRewriteMethod=1266,875,1095,1141,2306,1079 -org.apache.lucene.search.TestFieldCacheTermsFilter=12,13,10,10,22,19 -org.apache.lucene.search.TestFieldValueFilter=31,27,36,28,90,22 -org.apache.lucene.search.TestFilteredQuery=240,304,323,432,241,280 -org.apache.lucene.search.TestFilteredSearch=33,25,41,39,264,23 -org.apache.lucene.search.TestFuzzyQuery=154,104,57,100,893,238 -org.apache.lucene.search.TestFuzzyQuery2=1099,450,1117,514,220,382 -org.apache.lucene.search.TestMatchAllDocsQuery=11,9,33,15,26,71 -org.apache.lucene.search.TestMultiPhraseQuery=504,833,826,791,245,210 -org.apache.lucene.search.TestMultiTermConstantScore=426,186,137,124,419,855 -org.apache.lucene.search.TestMultiTermQueryRewrites=24,22,15,45,26,27 -org.apache.lucene.search.TestMultiThreadTermVectors=1232,1360,1148,1634,287,209 -org.apache.lucene.search.TestMultiValuedNumericRangeQuery=1131,990,2837,817,896,420 -org.apache.lucene.search.TestNGramPhraseQuery=5,4,4,4,4,5 -org.apache.lucene.search.TestNRTManager=2556,60,5400,5146,2727,1729 -org.apache.lucene.search.TestNot=18,8,14,8,10,60 -org.apache.lucene.search.TestNumericRangeQuery32=3513,6453,5145,5126,8550,6148 -org.apache.lucene.search.TestNumericRangeQuery64=17221,12483,22047,7784,15993,6128 -org.apache.lucene.search.TestPhrasePrefixQuery=8,13,30,7,10,10 -org.apache.lucene.search.TestPhraseQuery=3164,6474,5922,4556,1475,1734 -org.apache.lucene.search.TestPositionIncrement=15,19,13,17,30,60 -org.apache.lucene.search.TestPositiveScoresOnlyCollector=5,4,6,17,4,8 -org.apache.lucene.search.TestPrefixFilter=6,8,6,16,16,14 -org.apache.lucene.search.TestPrefixInBooleanQuery=756,635,1199,646,3182,636 -org.apache.lucene.search.TestPrefixQuery=8,9,8,32,10,7 -org.apache.lucene.search.TestPrefixRandom=1826,3314,769,990,925,93 -org.apache.lucene.search.TestQueryWrapperFilter=966,320,1226,3325,449,877 -org.apache.lucene.search.TestRegexpQuery=46,81,45,47,509,130 -org.apache.lucene.search.TestRegexpRandom=110,84,162,164,213,165 -org.apache.lucene.search.TestRegexpRandom2=2429,5399,4815,5980,4176,1425 -org.apache.lucene.search.TestScoreCachingWrappingScorer=6,5,4,6,5,19 -org.apache.lucene.search.TestScorerPerf=574,1215,1022,661,261,478 -org.apache.lucene.search.TestSearchAfter=79,43,86,101,76,76 -org.apache.lucene.search.TestSearchWithThreads=1482,2318,2046,4084,2464,1303 -org.apache.lucene.search.TestSearcherManager=2658,9519,4796,2247,5070,4181 -org.apache.lucene.search.TestShardSearching=6964,4585,3932,5770,4762,4415 -org.apache.lucene.search.TestSimilarity=9,11,8,17,13,9 -org.apache.lucene.search.TestSimilarityProvider=11,9,6,11,12,26 -org.apache.lucene.search.TestSimpleExplanations=683,855,736,3021,1272,1326 -org.apache.lucene.search.TestSimpleExplanationsOfNonMatches=67,60,139,441,131,511 -org.apache.lucene.search.TestSimpleSearchEquivalence=194 -org.apache.lucene.search.TestSloppyPhraseQuery=3752,3070,3444,3542,3855,1790 -org.apache.lucene.search.TestSloppyPhraseQuery2=1943 -org.apache.lucene.search.TestSort=2189,2422,3173,2332,4442,1971 -org.apache.lucene.search.TestSubScorerFreqs=16,18,15,36,13,66 -org.apache.lucene.search.TestTermRangeFilter=565,778,1616,415,256,331 -org.apache.lucene.search.TestTermRangeQuery=45,55,71,65,224,262 -org.apache.lucene.search.TestTermScorer=19,19,47,19,26,46 -org.apache.lucene.search.TestTermVectors=252,408,547,312,738,180 -org.apache.lucene.search.TestTimeLimitingCollector=4680,4452,3286,2701,1733,3343 -org.apache.lucene.search.TestTopDocsCollector=55,55,37,95,512,82 -org.apache.lucene.search.TestTopDocsMerge=3992,4115,1374,3679,4153,3706 -org.apache.lucene.search.TestTopScoreDocCollector=5,7,7,9,7,11 -org.apache.lucene.search.TestWildcard=257,91,61,69,288,68 -org.apache.lucene.search.TestWildcardRandom=93,78,98,92,350,64 -org.apache.lucene.search.grouping.AllGroupHeadsCollectorTest=3657,3132,4805,4304,4269,1910 -org.apache.lucene.search.grouping.AllGroupsCollectorTest=456,430,426,444,514,83 -org.apache.lucene.search.grouping.DistinctValuesCollectorTest=1328 -org.apache.lucene.search.grouping.GroupFacetCollectorTest=4246 -org.apache.lucene.search.grouping.GroupingSearchTest=456 -org.apache.lucene.search.grouping.TestGrouping=5285,5376,5745,5765,6922,6788 -org.apache.lucene.search.highlight.HighlighterPhraseTest=228,161,162,229,131,118 -org.apache.lucene.search.highlight.HighlighterTest=1405,1161,1142,1411,1965,2060 -org.apache.lucene.search.highlight.OffsetLimitTokenFilterTest=11,12,10,11,5,11 -org.apache.lucene.search.highlight.TokenSourcesTest=214,193,158,165,116,157 -org.apache.lucene.search.highlight.custom.HighlightCustomQueryTest=11,7,11,8,21,57 -org.apache.lucene.search.join.TestBlockJoin=3159,4003,2733,3472,3350,3427 -org.apache.lucene.search.join.TestJoinUtil=5935,6234,6195,7757,8756,9496 -org.apache.lucene.search.payloads.TestPayloadExplanations=1410,324,339,314,310,297 -org.apache.lucene.search.payloads.TestPayloadNearQuery=759,613,120,1657,1403,210 -org.apache.lucene.search.payloads.TestPayloadTermQuery=463,1159,2769,695,1112,182 -org.apache.lucene.search.similarities.TestSimilarity2=132,71,101,125,274,237 -org.apache.lucene.search.similarities.TestSimilarityBase=2059,1801,1740,1474,1085,531 -org.apache.lucene.search.spans.TestBasics=9064,7511,8748,9945,9025,8026 -org.apache.lucene.search.spans.TestFieldMaskingSpanQuery=190,1213,467,273,267,202 -org.apache.lucene.search.spans.TestNearSpansOrdered=82,92,47,87,120,39 -org.apache.lucene.search.spans.TestPayloadSpans=755,677,1262,515,3286,571 -org.apache.lucene.search.spans.TestSpanExplanations=582,2817,2582,816,455,1506 -org.apache.lucene.search.spans.TestSpanExplanationsOfNonMatches=51,28,59,75,49,98 -org.apache.lucene.search.spans.TestSpanFirstQuery=44,7,9,11,14,12 -org.apache.lucene.search.spans.TestSpanMultiTermQueryWrapper=63,50,71,37,306,29 -org.apache.lucene.search.spans.TestSpanSearchEquivalence=99 -org.apache.lucene.search.spans.TestSpans=1628,1823,2607,2122,233,186 -org.apache.lucene.search.spans.TestSpansAdvanced=24,59,25,156,37,58 -org.apache.lucene.search.spans.TestSpansAdvanced2=85,104,108,145,210,358 -org.apache.lucene.search.spell.TestDirectSpellChecker=544,604,516,663,695,1019 -org.apache.lucene.search.spell.TestJaroWinklerDistance=7,10,8,7,5,11 -org.apache.lucene.search.spell.TestLevenshteinDistance=6,10,12,9,254,256 -org.apache.lucene.search.spell.TestLuceneDictionary=760,641,666,766,778,359 -org.apache.lucene.search.spell.TestNGramDistance=21,21,17,24,18,100 -org.apache.lucene.search.spell.TestPlainTextDictionary=356,296,269,324,45,554 -org.apache.lucene.search.spell.TestSpellChecker=3390,4686,4593,4629,3952,5968 -org.apache.lucene.search.suggest.LookupBenchmarkTest=14,9,14,12,3,5 -org.apache.lucene.search.suggest.PersistenceTest=41,38,43,42,132,205 -org.apache.lucene.search.suggest.TestBytesRefList=78 -org.apache.lucene.search.suggest.TestHighFrequencyDictionary=398 -org.apache.lucene.search.suggest.TestTermFreqIterator=707 -org.apache.lucene.search.suggest.fst.BytesRefSortersTest=324,331,295,288,294,158 -org.apache.lucene.search.suggest.fst.FSTCompletionTest=624,630,702,584,326,2279 -org.apache.lucene.search.suggest.fst.FloatMagicTest=9,12,15,10,9 -org.apache.lucene.search.suggest.fst.TestSort=3757,4325,4061,4481,4273,4597 -org.apache.lucene.search.suggest.fst.WFSTCompletionTest=1585,968 -org.apache.lucene.search.vectorhighlight.BreakIteratorBoundaryScannerTest=21,21,17,19,12,49 -org.apache.lucene.search.vectorhighlight.FieldPhraseListTest=293,255,256,282,274,887 -org.apache.lucene.search.vectorhighlight.FieldQueryTest=707,593,614,664,141,384 -org.apache.lucene.search.vectorhighlight.FieldTermStackTest=730,744,777,664,300,337 -org.apache.lucene.search.vectorhighlight.IndexTimeSynonymTest=373,406,389,379,398,946 -org.apache.lucene.search.vectorhighlight.ScoreOrderFragmentsBuilderTest=564,574,482,480,463,68 -org.apache.lucene.search.vectorhighlight.SimpleBoundaryScannerTest=8,7,8,10,4,35 -org.apache.lucene.search.vectorhighlight.SimpleFragListBuilderTest=426,367,392,343,864,941 -org.apache.lucene.search.vectorhighlight.SimpleFragmentsBuilderTest=660,713,774,800,296,375 -org.apache.lucene.search.vectorhighlight.SingleFragListBuilderTest=118,86,116,73,594,63 -org.apache.lucene.spatial.DistanceUtilsTest=33,23,28,36,264 -org.apache.lucene.spatial.TestTestFramework=246 -org.apache.lucene.spatial.geohash.TestGeoHashUtils=209,257,250,230,219 -org.apache.lucene.spatial.geometry.TestDistanceUnits=232,217,264,202,15 -org.apache.lucene.spatial.prefix.TestSpatialPrefixField=244 -org.apache.lucene.spatial.prefix.TestTermQueryPrefixGridStrategy=424 -org.apache.lucene.spatial.prefix.tree.SpatialPrefixTreeTest=216 -org.apache.lucene.spatial.tier.TestCartesian=1039,999,1060,1061,929 -org.apache.lucene.spatial.tier.TestDistance=495,539,510,529,510 -org.apache.lucene.spatial.tier.projections.SinusoidalProjectorTest=6,10,15,26,8 -org.apache.lucene.store.TestBufferedIndexInput=2269,1131,5331,3833,2510,2152 -org.apache.lucene.store.TestByteArrayDataInput=3,4,3,2,2,3 -org.apache.lucene.store.TestCopyBytes=558,757,556,650,324,449 -org.apache.lucene.store.TestDirectory=28,24,29,27,28,34 -org.apache.lucene.store.TestFileSwitchDirectory=31,36,73,34,58,42 -org.apache.lucene.store.TestHugeRamFile=859,794,2281,654,1308,1271 -org.apache.lucene.store.TestLock=13,12,13,22,12,15 -org.apache.lucene.store.TestLockFactory=1039,1025,1061,1237,1030,1039 -org.apache.lucene.store.TestMultiMMap=5453,4687,3883,5281,4681,4590 -org.apache.lucene.store.TestNRTCachingDirectory=3631,661,3190,2063,1291,1713 -org.apache.lucene.store.TestRAMDirectory=2038,2258,2197,2227,375,290 -org.apache.lucene.store.TestWindowsMMap=280,135,280,185,50,59 -org.apache.lucene.util.TestArrayUtil=2332,2353,2346,3829,2049,3773 -org.apache.lucene.util.TestAttributeSource=6,7,10,10,10,12 -org.apache.lucene.util.TestBitUtil=3,3,3,4,3,4 -org.apache.lucene.util.TestByteBlockPool=3,3,2,3,2,7 -org.apache.lucene.util.TestBytesRef=4,8,4,4,4,8 -org.apache.lucene.util.TestBytesRefHash=65,50,33,122,56,1277 -org.apache.lucene.util.TestCharsRef=13,12,14,13,44,20 -org.apache.lucene.util.TestCloseableThreadLocal=4,4,4,4,4,4 -org.apache.lucene.util.TestCollectionUtil=1772,1724,1887,1494,1343,1963 -org.apache.lucene.util.TestDoubleBarrelLRUCache=1007,1008,1009,1006,1005,1009 -org.apache.lucene.util.TestFieldCacheSanityChecker=560,559,414,759,2030,452 -org.apache.lucene.util.TestFixedBitSet=627,562,870,559,970,2356 -org.apache.lucene.util.TestIOUtils=4,3,4,5,149,10 -org.apache.lucene.util.TestIdentityHashSet=133 -org.apache.lucene.util.TestIndexableBinaryStringTools=129,352,166,126,732,239 -org.apache.lucene.util.TestJUnitRuleOrder=4,3,2,2 -org.apache.lucene.util.TestNamedSPILoader=3,5,2,5,3,32 -org.apache.lucene.util.TestNumericUtils=1288,1565,1011,910,1203,746 -org.apache.lucene.util.TestOpenBitSet=1607,2194,2600,2196,1461,3886 -org.apache.lucene.util.TestPagedBytes=537,295,199,387,318,1058 -org.apache.lucene.util.TestPriorityQueue=14,25,61,16,20,96 -org.apache.lucene.util.TestRamUsageEstimator=4,2,2,3,4,10 -org.apache.lucene.util.TestRamUsageEstimatorOnWildAnimals=890 -org.apache.lucene.util.TestRecyclingByteBlockAllocator=17,18,26,29,24,19 -org.apache.lucene.util.TestRollingBuffer=126 -org.apache.lucene.util.TestRollingCharBuffer=752 -org.apache.lucene.util.TestSentinelIntSet=132,106,108,113,168,177 -org.apache.lucene.util.TestSetOnce=18,39,15,16,16,20 -org.apache.lucene.util.TestSetupTeardownMethods=24,26,32,25 -org.apache.lucene.util.TestSmallFloat=27,34,31,29,30,66 -org.apache.lucene.util.TestSortedVIntList=21,21,22,27,22,83 -org.apache.lucene.util.TestTwoPhaseCommitTool=8,8,5,8,13,25 -org.apache.lucene.util.TestUnicodeUtil=257,256,314,244,178,211 -org.apache.lucene.util.TestVersion=2,2,2,4,2,17 -org.apache.lucene.util.TestVersionComparator=2,3,4,3,2,5 -org.apache.lucene.util.TestVirtualMethod=6,6,5,4,6,12 -org.apache.lucene.util.TestWeakIdentityMap=531,692,1114,445,535,472 -org.apache.lucene.util.UnsafeByteArrayInputStreamTest=233,253,272,218,23,26 -org.apache.lucene.util.UnsafeByteArrayOutputStreamTest=18,21,18,23,7,74 -org.apache.lucene.util.Vint8Test=12,9,12,15,8,505 -org.apache.lucene.util.automaton.TestBasicOperations=161,81,130,142,365,121 -org.apache.lucene.util.automaton.TestCompiledAutomaton=153,430,39,160,206,102 -org.apache.lucene.util.automaton.TestDeterminism=578,683,836,1021,566,464 -org.apache.lucene.util.automaton.TestDeterminizeLexicon=1174,1123,591,627,426,578 -org.apache.lucene.util.automaton.TestLevenshteinAutomata=662,800,596,463,1239,1713 -org.apache.lucene.util.automaton.TestMinimize=1910,2067,2633,1573,1965,1475 -org.apache.lucene.util.automaton.TestSpecialOperations=55,91,106,102,59,1228 -org.apache.lucene.util.automaton.TestUTF32ToUTF8=1485,1659,2310,788,849,809 -org.apache.lucene.util.collections.ArrayHashMapTest=97,100,33,89,37,95 -org.apache.lucene.util.collections.FloatToObjectMapTest=32,35,31,31,16,96 -org.apache.lucene.util.collections.IntArrayTest=6,6,8,12,12,172 -org.apache.lucene.util.collections.IntHashSetTest=81,59,62,53,47,152 -org.apache.lucene.util.collections.IntToDoubleMapTest=70,70,68,66,21,41 -org.apache.lucene.util.collections.IntToIntMapTest=49,42,34,34,58,135 -org.apache.lucene.util.collections.IntToObjectMapTest=165,28,73,61,36,45 -org.apache.lucene.util.collections.ObjectToFloatMapTest=68,57,79,72,19,47 -org.apache.lucene.util.collections.ObjectToIntMapTest=41,36,44,57,73,133 -org.apache.lucene.util.collections.TestLRUHashMap=8,4,5,4,2,9 -org.apache.lucene.util.encoding.EncodingTest=221,197,174,203,227,265 -org.apache.lucene.util.fst.TestFSTs=11876,220,10497,11092,10118,8691 -org.apache.lucene.util.junitcompat.TestExceptionInBeforeClassHooks=48 -org.apache.lucene.util.junitcompat.TestJUnitRuleOrder=9,40 -org.apache.lucene.util.junitcompat.TestReproduceMessage=61,201 -org.apache.lucene.util.junitcompat.TestSeedFromUncaught=49 -org.apache.lucene.util.junitcompat.TestSetupTeardownChaining=13,105 -org.apache.lucene.util.junitcompat.TestSystemPropertiesInvariantRule=60 -org.apache.lucene.util.packed.TestPackedInts=3696,2679,2803,3121,3115,4577 -org.apache.solr.BasicFunctionalityTest=1500,1334,1232,1102,2943,1626 -org.apache.solr.ConvertedLegacyTest=2435,1872,2131,2620,5835,2139 -org.apache.solr.DisMaxRequestHandlerTest=459,523,446,603,492,3005 -org.apache.solr.EchoParamsTest=93,93,91,81,182,127 -org.apache.solr.MinimalSchemaTest=173,151,162,151,311,214 -org.apache.solr.OutputWriterTest=193,374,199,230,195,191 -org.apache.solr.SampleTest=220,221,233,302,165,146 -org.apache.solr.SolrInfoMBeanTest=1481,434,1136,1501,2498,439 -org.apache.solr.TestDistributedGrouping=14182,13937,13655,14416,38066,17556 -org.apache.solr.TestDistributedSearch=16279,14823,15468,15004,36045,43894 -org.apache.solr.TestGroupingSearch=4797,5620,5990,6053,3933,2905 -org.apache.solr.TestJoin=7374,7450,7478,6796,16137,14055 -org.apache.solr.TestPluginEnable=71,54,56,58,172,66 -org.apache.solr.TestSolrCoreProperties=109,183,86,212,380,167 -org.apache.solr.TestTrie=2342,2345,2368,2558,1844,806 -org.apache.solr.analysis.CommonGramsFilterFactoryTest=21,8,11,20,7,12 -org.apache.solr.analysis.CommonGramsQueryFilterFactoryTest=6,7,6,8,6,12 -org.apache.solr.analysis.DoubleMetaphoneFilterFactoryTest=11,13,7,12,10,7 -org.apache.solr.analysis.LegacyHTMLStripCharFilterTest=194,88,98,159,119,141 -org.apache.solr.analysis.LengthFilterTest=6,5,6,7,2,7 -org.apache.solr.analysis.SnowballPorterFilterFactoryTest=10,11,12,9,13,18 -org.apache.solr.analysis.TestArabicFilters=8,10,11,6,11,18 -org.apache.solr.analysis.TestBeiderMorseFilterFactory=331,303,237,330,250,285 -org.apache.solr.analysis.TestBrazilianStemFilterFactory=6,4,7,17,3,6 -org.apache.solr.analysis.TestBulgarianStemFilterFactory=3,3,4,3,4,12 -org.apache.solr.analysis.TestCJKBigramFilterFactory=4,4,3,4,5,11 -org.apache.solr.analysis.TestCJKTokenizerFactory=4,9,5,8,26,34 -org.apache.solr.analysis.TestCJKWidthFilterFactory=3,5,3,4,2,7 -org.apache.solr.analysis.TestCapitalizationFilterFactory=7,12,8,10,25,41 -org.apache.solr.analysis.TestChineseFilterFactory=4,3,3,2,2,14 -org.apache.solr.analysis.TestChineseTokenizerFactory=6,5,5,6,3,8 -org.apache.solr.analysis.TestCollationKeyFilterFactory=118,93,109,109,74,126 -org.apache.solr.analysis.TestCollationKeyRangeQueries=149,105,164,105,313,122 -org.apache.solr.analysis.TestCzechStemFilterFactory=4,3,3,2,10,5 -org.apache.solr.analysis.TestDelimitedPayloadTokenFilterFactory=4,6,4,4,4,6 -org.apache.solr.analysis.TestDictionaryCompoundWordTokenFilterFactory=15,5,5,8,7,5 -org.apache.solr.analysis.TestElisionFilterFactory=5,8,6,5,10,14 -org.apache.solr.analysis.TestEnglishMinimalStemFilterFactory=2,2,3,3,5,8 -org.apache.solr.analysis.TestFinnishLightStemFilterFactory=10,6,5,9,3,5 -org.apache.solr.analysis.TestFoldingMultitermExtrasQuery=2595 -org.apache.solr.analysis.TestFrenchLightStemFilterFactory=7,3,8,8,11,10 -org.apache.solr.analysis.TestFrenchMinimalStemFilterFactory=3,6,4,3,5,5 -org.apache.solr.analysis.TestGalicianMinimalStemFilterFactory=13,14,14,16,14,16 -org.apache.solr.analysis.TestGalicianStemFilterFactory=22,20,27,23,16,16 -org.apache.solr.analysis.TestGermanLightStemFilterFactory=5,3,6,3,4,6 -org.apache.solr.analysis.TestGermanMinimalStemFilterFactory=4,3,3,3,4,4 -org.apache.solr.analysis.TestGermanNormalizationFilterFactory=3,2,3,4,3,4 -org.apache.solr.analysis.TestGermanStemFilterFactory=5,4,3,4,3,8 -org.apache.solr.analysis.TestGreekLowerCaseFilterFactory=3,2,3,2,7,6 -org.apache.solr.analysis.TestGreekStemFilterFactory=6,5,4,5,7,31 -org.apache.solr.analysis.TestHTMLStripCharFilterFactory=51,6,9,6,38,108 -org.apache.solr.analysis.TestHindiFilters=12,7,8,8,9,25 -org.apache.solr.analysis.TestHungarianLightStemFilterFactory=13,3,7,12,4,5 -org.apache.solr.analysis.TestHunspellStemFilterFactory=7,8,7,8,10,18 -org.apache.solr.analysis.TestHyphenationCompoundWordTokenFilterFactory=80,79,74,84,129,105 -org.apache.solr.analysis.TestICUCollationKeyFilterFactory=1033,876,858,854,705,861 -org.apache.solr.analysis.TestICUCollationKeyRangeQueries=278,273,311,236,2032,1628 -org.apache.solr.analysis.TestICUFoldingFilterFactory=319,291,301,285,18,55 -org.apache.solr.analysis.TestICUNormalizer2FilterFactory=334,323,311,272,301,18 -org.apache.solr.analysis.TestICUTokenizerFactory=156,150,156,493,194,388 -org.apache.solr.analysis.TestICUTransformFilterFactory=984,935,811,837,857,678 -org.apache.solr.analysis.TestIndonesianStemFilterFactory=6,4,3,7,8,6 -org.apache.solr.analysis.TestIrishLowerCaseFilterFactory=5 -org.apache.solr.analysis.TestItalianLightStemFilterFactory=2,3,4,3,3,6 -org.apache.solr.analysis.TestJapaneseBaseFormFilterFactory=230 -org.apache.solr.analysis.TestJapanesePartOfSpeechStopFilterFactory=222 -org.apache.solr.analysis.TestJapaneseTokenizerFactory=192 -org.apache.solr.analysis.TestKStemFilterFactory=103,98,100,109,123,111 -org.apache.solr.analysis.TestKeepFilterFactory=7,5,7,5,6,6 -org.apache.solr.analysis.TestKeywordMarkerFilterFactory=7,5,8,7,7,525 -org.apache.solr.analysis.TestKuromojiBaseFormFilterFactory=385,235,442,485,283 -org.apache.solr.analysis.TestKuromojiPartOfSpeechStopFilterFactory=206,201,195,216,4 -org.apache.solr.analysis.TestKuromojiTokenizerFactory=324,299,13,213,515 -org.apache.solr.analysis.TestLatvianStemFilterFactory=11,8,32,14,5,10 -org.apache.solr.analysis.TestLuceneMatchVersion=154,134,217,136,308,132 -org.apache.solr.analysis.TestMappingCharFilterFactory=3,3,2,4,2,3 -org.apache.solr.analysis.TestMorfologikFilterFactory=457 -org.apache.solr.analysis.TestMultiWordSynonyms=12,10,11,9,12,14 -org.apache.solr.analysis.TestNGramFilters=12,18,12,12,78,70 -org.apache.solr.analysis.TestNorwegianLightStemFilterFactory=9 -org.apache.solr.analysis.TestNorwegianMinimalStemFilterFactory=7 -org.apache.solr.analysis.TestPatternReplaceCharFilterFactory=16,5,8,16,8,34 -org.apache.solr.analysis.TestPatternReplaceFilterFactory=6,5,5,6,2,17 -org.apache.solr.analysis.TestPatternTokenizerFactory=5,5,5,6,2,353 -org.apache.solr.analysis.TestPersianNormalizationFilterFactory=5,5,5,5,2,6 -org.apache.solr.analysis.TestPhoneticFilterFactory=20120,5958,14344,15767,18979,6893 -org.apache.solr.analysis.TestPorterStemFilterFactory=18,4,4,6,5,8 -org.apache.solr.analysis.TestPortugueseLightStemFilterFactory=8,7,8,8,22,5 -org.apache.solr.analysis.TestPortugueseMinimalStemFilterFactory=16,11,22,18,19,45 -org.apache.solr.analysis.TestPortugueseStemFilterFactory=8,8,11,15,30,22 -org.apache.solr.analysis.TestRemoveDuplicatesTokenFilterFactory=2,3,2,1,2,6 -org.apache.solr.analysis.TestReverseStringFilterFactory=3,3,3,2,2,5 -org.apache.solr.analysis.TestReversedWildcardFilterFactory=464,393,542,591,669,1228 -org.apache.solr.analysis.TestRussianFilters=4,5,3,3,4,4 -org.apache.solr.analysis.TestRussianLightStemFilterFactory=10,11,8,8,11,8 -org.apache.solr.analysis.TestShingleFilterFactory=20,15,14,23,13,50 -org.apache.solr.analysis.TestSlowSynonymFilter=18,12,17,14,29,35 -org.apache.solr.analysis.TestSmartChineseFactories=441,474,430,450,296,920 -org.apache.solr.analysis.TestSpanishLightStemFilterFactory=7,4,8,4,5,5 -org.apache.solr.analysis.TestStandardFactories=15,27,19,15,18,23 -org.apache.solr.analysis.TestStemmerOverrideFilterFactory=5,7,4,4,5,12 -org.apache.solr.analysis.TestStempelPolishStemFilterFactory=235,151,176,157,176,605 -org.apache.solr.analysis.TestStopFilterFactory=4,7,5,5,7,43 -org.apache.solr.analysis.TestSwedishLightStemFilterFactory=5,3,8,8,3,16 -org.apache.solr.analysis.TestSynonymFilterFactory=17,10,14,10,15,13 -org.apache.solr.analysis.TestSynonymMap=8,8,7,8,14,15 -org.apache.solr.analysis.TestThaiWordFilterFactory=19,29,28,24,22,93 -org.apache.solr.analysis.TestTrimFilterFactory=10,3,6,3,3,5 -org.apache.solr.analysis.TestTurkishLowerCaseFilterFactory=4,4,8,4,15,5 -org.apache.solr.analysis.TestTypeTokenFilterFactory=12,7,8,7,6,13 -org.apache.solr.analysis.TestUAX29URLEmailTokenizerFactory=51,46,48,67,81,60 -org.apache.solr.analysis.TestWikipediaTokenizerFactory=7,6,6,9,7,15 -org.apache.solr.analysis.TestWordDelimiterFilterFactory=909,900,822,830,1455,914 -org.apache.solr.client.solrj.BasicHttpSolrServerTest=3811 -org.apache.solr.client.solrj.SolrExampleBinaryTest=7643,7512,6962,7091,6928,5165 -org.apache.solr.client.solrj.SolrExceptionTest=367,353,311,358,42,14 -org.apache.solr.client.solrj.SolrQueryTest=16,13,16,14,240,27 -org.apache.solr.client.solrj.TestBatchUpdate=4209,4045,4004,4079,2305,4178 -org.apache.solr.client.solrj.TestLBHttpSolrServer=8879,9830,9004,7846,7490,9731 -org.apache.solr.client.solrj.beans.TestDocumentObjectBinder=26,31,36,36,18,31 -org.apache.solr.client.solrj.embedded.JettyWebappTest=4743,4566,4392,4602,2111,1012 -org.apache.solr.client.solrj.embedded.LargeVolumeBinaryJettyTest=847,976,1062,825,1244,1745 -org.apache.solr.client.solrj.embedded.LargeVolumeEmbeddedTest=1221,1333,1358,1091,1778,1588 -org.apache.solr.client.solrj.embedded.LargeVolumeJettyTest=1319,1275,1364,1296,918,1102 -org.apache.solr.client.solrj.embedded.MergeIndexesEmbeddedTest=717,775,734,667,586,561 -org.apache.solr.client.solrj.embedded.MultiCoreEmbeddedTest=420,415,367,381,314,321 -org.apache.solr.client.solrj.embedded.MultiCoreExampleJettyTest=735,952,968,943,697,1396 -org.apache.solr.client.solrj.embedded.SolrExampleEmbeddedTest=4953,4762,4825,4703,4249,7319 -org.apache.solr.client.solrj.embedded.SolrExampleJettyTest=4491,4580,4337,4405,4428,4926 -org.apache.solr.client.solrj.embedded.SolrExampleStreamingBinaryTest=9544,9591,9796,9684,12707,13200 -org.apache.solr.client.solrj.embedded.SolrExampleStreamingTest=10177,9896,10353,10507,10402,10479 -org.apache.solr.client.solrj.embedded.TestEmbeddedSolrServer=349,546,621,406,1682,501 -org.apache.solr.client.solrj.embedded.TestSolrProperties=403,366,400,357,712,3070 -org.apache.solr.client.solrj.request.TestUpdateRequestCodec=8,6,8,6,5,51 -org.apache.solr.client.solrj.response.AnlysisResponseBaseTest=4,5,5,5,5,25 -org.apache.solr.client.solrj.response.DocumentAnalysisResponseTest=5,5,6,7,3,20 -org.apache.solr.client.solrj.response.FacetFieldTest=4,4,4,6,196,12 -org.apache.solr.client.solrj.response.FieldAnalysisResponseTest=7,3,3,4,2,282 -org.apache.solr.client.solrj.response.QueryResponseTest=29,40,31,29,18,22 -org.apache.solr.client.solrj.response.TermsResponseTest=2499,2741,2620,2502,509,600 -org.apache.solr.client.solrj.response.TestSpellCheckResponse=1118,1091,1060,985,3064,788 -org.apache.solr.client.solrj.util.ClientUtilsTest=4,5,7,2,7,7 -org.apache.solr.cloud.BasicDistributedZkTest=23174,23981,24200,23322,39077,34105 -org.apache.solr.cloud.BasicZkTest=9193,9055,9447,10470,10521,8457 -org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest=1,0,1,1,0,1 -org.apache.solr.cloud.ChaosMonkeySafeLeaderTest=1,0,0,1,0,1 -org.apache.solr.cloud.CloudStateTest=5,4,4,3,3,13 -org.apache.solr.cloud.CloudStateUpdateTest=11657,12138,13721,13338,18033,9772 -org.apache.solr.cloud.FullSolrCloudDistribCmdsTest=52827,44720,50511,51658,58897,36111 -org.apache.solr.cloud.FullSolrCloudTest=0,1,1,1,1,1 -org.apache.solr.cloud.LeaderElectionIntegrationTest=25451,24310,23188,23681,36677,23255 -org.apache.solr.cloud.LeaderElectionTest=34997,20373,17821,24752,26838,29423 -org.apache.solr.cloud.NodeStateWatcherTest=24232 -org.apache.solr.cloud.OverseerTest=29472,28619,29386,29074,53998,57563 -org.apache.solr.cloud.RecoveryZkTest=25932,27672,26705,26491,23463,29315 -org.apache.solr.cloud.TestHashPartitioner=4678,5325,5437,4308,14818,15647 -org.apache.solr.cloud.TestMultiCoreConfBootstrap=4425 -org.apache.solr.cloud.ZkControllerTest=7251,7097,7069,7015,7480,15652 -org.apache.solr.cloud.ZkNodePropsTest=10,12,9,12,16,4 -org.apache.solr.cloud.ZkSolrClientTest=15964,15673,15688,15912,16082,17076 -org.apache.solr.common.SolrDocumentTest=10,15,19,9,14,72 -org.apache.solr.common.params.ModifiableSolrParamsTest=9,9,10,11,15,38 -org.apache.solr.common.params.SolrParamTest=217,202,212,222,7,21 -org.apache.solr.common.util.ContentStreamTest=220,287,387,183,208,1104 -org.apache.solr.common.util.DOMUtilTest=16,20,16,18,11 -org.apache.solr.common.util.FileUtilsTest=3,3,3,4,3 -org.apache.solr.common.util.IteratorChainTest=11,9,12,14,8,62 -org.apache.solr.common.util.NamedListTest=8,7,3,3,2,6 -org.apache.solr.common.util.TestFastInputStream=7,6,22,4,3,21 -org.apache.solr.common.util.TestHash=44,65,67,53,128,178 -org.apache.solr.common.util.TestJavaBinCodec=110,127,295,157,92,149 -org.apache.solr.common.util.TestNamedListCodec=958,817,1042,878,771,879 -org.apache.solr.common.util.TestSystemIdResolver=4,6,4,4,2 -org.apache.solr.common.util.TestXMLEscaping=10,9,12,9,16,14 -org.apache.solr.core.AlternateDirectoryTest=396,309,349,472,375,404 -org.apache.solr.core.IndexReaderFactoryTest=470,506,635,452,610,371 -org.apache.solr.core.PluginInfoTest=24,34,61,34,34,29 -org.apache.solr.core.RAMDirectoryFactoryTest=2,2,3,3,3,8 -org.apache.solr.core.RequestHandlersTest=409,648,631,717,388,1234 -org.apache.solr.core.ResourceLoaderTest=24,24,29,25,9,11 -org.apache.solr.core.SOLR749Test=299,277,271,341,536,467 -org.apache.solr.core.SolrCoreCheckLockOnStartupTest=769 -org.apache.solr.core.SolrCoreTest=2185,2198,2247,2191,2543,2586 -org.apache.solr.core.TestArbitraryIndexDir=704,441,457,481,1804,449 -org.apache.solr.core.TestBadConfig=4,6,5,7,6,13 -org.apache.solr.core.TestCodecSupport=68,73,71,84,70,108 -org.apache.solr.core.TestConfig=109,108,108,103,135,313 -org.apache.solr.core.TestCoreContainer=1266,1268,1275,1212,5313,3500 -org.apache.solr.core.TestJmxIntegration=1745,1683,1579,1663,1451,824 -org.apache.solr.core.TestJmxMonitoredMap=690,76,184,158,294,104 -org.apache.solr.core.TestLegacyMergeSchedulerPolicyConfig=310,315,294,393,624 -org.apache.solr.core.TestMergePolicyConfig=292,332,284,297,546,415 -org.apache.solr.core.TestPropInject=711,699,619,723,561,774 -org.apache.solr.core.TestPropInjectDefaults=384,418,445,564,1957,390 -org.apache.solr.core.TestQuerySenderListener=450,306,352,303,289,455 -org.apache.solr.core.TestQuerySenderNoQuery=357,327,304,349,472,349 -org.apache.solr.core.TestSolrDeletionPolicy1=609,667,606,668,579,3393 -org.apache.solr.core.TestSolrDeletionPolicy2=1273,364,846,1457,398,2649 -org.apache.solr.core.TestSolrXMLSerializer=46,64,57,46,11,16 -org.apache.solr.core.TestXIncludeConfig=339,298,320,315,399,478 -org.apache.solr.handler.BinaryUpdateRequestHandlerTest=414,454,349,401,1578,516 -org.apache.solr.handler.CSVRequestHandlerTest=1187,377,944,1423,967,569 -org.apache.solr.handler.DocumentAnalysisRequestHandlerTest=732,823,727,751,2409,435 -org.apache.solr.handler.FieldAnalysisRequestHandlerTest=354,363,459,415,989,966 -org.apache.solr.handler.JsonLoaderTest=359,410,340,520,1926,547 -org.apache.solr.handler.MoreLikeThisHandlerTest=2497,2699,2495,2473,435,638 -org.apache.solr.handler.StandardRequestHandlerTest=1569,1623,1609,1603,574,479 -org.apache.solr.handler.TestCSVLoader=589,540,486,489,1444,1570 -org.apache.solr.handler.TestReplicationHandler=22484,24030,23080,23374,39748,23441 -org.apache.solr.handler.XmlUpdateRequestHandlerTest=569,503,731,612,1265,446 -org.apache.solr.handler.XsltUpdateRequestHandlerTest=856,863,892,892,2695,588 -org.apache.solr.handler.admin.CoreAdminHandlerTest=753,871,795,993,933,3415 -org.apache.solr.handler.admin.LogLevelHandlerTest=581 -org.apache.solr.handler.admin.LukeRequestHandlerTest=1118,1473,1204,1189,1329,1464 -org.apache.solr.handler.admin.MBeansHandlerTest=395 -org.apache.solr.handler.admin.ShowFileRequestHandlerTest=1335,1137,1149,1356,837,666 -org.apache.solr.handler.admin.SystemInfoHandlerTest=1,1,1,2,2,16 -org.apache.solr.handler.clustering.ClusteringComponentTest=2498,2606,2458,2459,2417,2645 -org.apache.solr.handler.clustering.DistributedClusteringComponentTest=6889,9315,8222,5929,7126,8239 -org.apache.solr.handler.clustering.carrot2.CarrotClusteringEngineTest=2927,3022,2956,2837,2941,3233 -org.apache.solr.handler.component.BadComponentTest=426,401,526,568,570,300 -org.apache.solr.handler.component.DebugComponentTest=503,575,573,821,996,520 -org.apache.solr.handler.component.DistributedQueryElevationComponentTest=2842 -org.apache.solr.handler.component.DistributedSpellCheckComponentTest=9117,11526,10039,11602,12358,10095 -org.apache.solr.handler.component.DistributedTermsComponentTest=10076,9811,9628,9013,9826,18652 -org.apache.solr.handler.component.QueryElevationComponentTest=4732,4506,4729,4419,3770,2932 -org.apache.solr.handler.component.SearchHandlerTest=495,386,531,403,397,366 -org.apache.solr.handler.component.SpellCheckComponentTest=5842,4847,3472,3600,19878,3323 -org.apache.solr.handler.component.StatsComponentTest=1715,2068,1992,1701,1973,2979 -org.apache.solr.handler.component.TermVectorComponentTest=484,410,388,375,384,635 -org.apache.solr.handler.component.TermsComponentTest=1595,608,1169,2046,567,591 -org.apache.solr.handler.dataimport.TestCachedSqlEntityProcessor=35,32,16,31,10,22 -org.apache.solr.handler.dataimport.TestClobTransformer=5,7,5,5,192,9 -org.apache.solr.handler.dataimport.TestContentStreamDataSource=1990,2014,1987,2000,1650,1626 -org.apache.solr.handler.dataimport.TestContextImpl=6,5,7,6,8,10 -org.apache.solr.handler.dataimport.TestDataConfig=149,146,149,142,145,154 -org.apache.solr.handler.dataimport.TestDateFormatTransformer=6,6,5,5,5,12 -org.apache.solr.handler.dataimport.TestDocBuilder=17,16,16,16,17,54 -org.apache.solr.handler.dataimport.TestDocBuilder2=301,300,292,289,303,383 -org.apache.solr.handler.dataimport.TestDocBuilderThreaded=1341,1346,1369,1311,356 -org.apache.solr.handler.dataimport.TestEntityProcessorBase=5,5,5,4,3,6 -org.apache.solr.handler.dataimport.TestEphemeralCache=159,156,150,152,173,185 -org.apache.solr.handler.dataimport.TestErrorHandling=261,246,239,237,281,307 -org.apache.solr.handler.dataimport.TestEvaluatorBag=6,6,6,7,8,17 -org.apache.solr.handler.dataimport.TestFieldReader=18,16,15,16,5,34 -org.apache.solr.handler.dataimport.TestFileListEntityProcessor=45,46,46,45,52,17 -org.apache.solr.handler.dataimport.TestJdbcDataSource=43,39,39,39,39,51 -org.apache.solr.handler.dataimport.TestLineEntityProcessor=7,7,7,6,5,35 -org.apache.solr.handler.dataimport.TestMailEntityProcessor=200,196,203,209,237,264 -org.apache.solr.handler.dataimport.TestNumberFormatTransformer=13,15,13,13,16,22 -org.apache.solr.handler.dataimport.TestPlainTextEntityProcessor=7,7,7,7,6,9 -org.apache.solr.handler.dataimport.TestRegexTransformer=6,15,10,5,6,17 -org.apache.solr.handler.dataimport.TestScriptTransformer=98,43,44,44,48,87 -org.apache.solr.handler.dataimport.TestSolrEntityProcessorEndToEnd=7135,7080,6979,7025,7498,6086 -org.apache.solr.handler.dataimport.TestSolrEntityProcessorUnit=9,8,9,9,11,16 -org.apache.solr.handler.dataimport.TestSortedMapBackedCache=27,25,26,24,11,7 -org.apache.solr.handler.dataimport.TestSqlEntityProcessor=11,9,9,9,8,11 -org.apache.solr.handler.dataimport.TestSqlEntityProcessor2=260,258,243,264,248,235 -org.apache.solr.handler.dataimport.TestSqlEntityProcessorDelta=579,574,585,581,1522,560 -org.apache.solr.handler.dataimport.TestSqlEntityProcessorDelta2=612,632,477,460,387,377 -org.apache.solr.handler.dataimport.TestSqlEntityProcessorDelta3=328,332,324,320,299,286 -org.apache.solr.handler.dataimport.TestSqlEntityProcessorDeltaPrefixedPk=280,269,274,311,282,207 -org.apache.solr.handler.dataimport.TestTemplateString=3,3,2,4,1,4 -org.apache.solr.handler.dataimport.TestTemplateTransformer=3,3,4,3,3,5 -org.apache.solr.handler.dataimport.TestThreaded=179,180,179,178,159 -org.apache.solr.handler.dataimport.TestTikaEntityProcessor=2454,2286,2330,2375,2640,2424 -org.apache.solr.handler.dataimport.TestURLDataSource=4,6,4,3,2,6 -org.apache.solr.handler.dataimport.TestVariableResolver=13,11,13,12,24,26 -org.apache.solr.handler.dataimport.TestXPathEntityProcessor=413,390,397,423,438,400 -org.apache.solr.handler.dataimport.TestXPathRecordReader=36,36,35,36,45,124 -org.apache.solr.handler.extraction.ExtractingRequestHandlerTest=3112,3082,3114,3071,3087,3351 -org.apache.solr.highlight.FastVectorHighlighterTest=647,632,609,492,486,906 -org.apache.solr.highlight.HighlighterConfigTest=353,344,288,583,493,546 -org.apache.solr.highlight.HighlighterTest=1014,851,980,1013,1788,1411 -org.apache.solr.internal.csv.CSVParserTest=28 -org.apache.solr.internal.csv.CSVPrinterTest=642 -org.apache.solr.internal.csv.CSVStrategyTest=23 -org.apache.solr.internal.csv.CSVUtilsTest=12 -org.apache.solr.internal.csv.CharBufferTest=17 -org.apache.solr.internal.csv.ExtendedBufferedReaderTest=12 -org.apache.solr.internal.csv.writer.CSVConfigGuesserTest=5 -org.apache.solr.internal.csv.writer.CSVConfigTest=5 -org.apache.solr.internal.csv.writer.CSVFieldTest=3 -org.apache.solr.internal.csv.writer.CSVWriterTest=6 -org.apache.solr.request.JSONWriterTest=427,442,382,731,472,709 -org.apache.solr.request.SimpleFacetsTest=3676,4663,4225,3916,12265,3945 -org.apache.solr.request.TestBinaryResponseWriter=664,787,674,977,1022,789 -org.apache.solr.request.TestFaceting=6576,7198,7028,7421,26765,7909 -org.apache.solr.request.TestRemoteStreaming=892,893,819,1245,523,724 -org.apache.solr.request.TestWriterPerf=1503,507,1236,1449,429,516 -org.apache.solr.response.TestCSVResponseWriter=352,599,511,405,1580,405 -org.apache.solr.response.TestPHPSerializedResponseWriter=524,533,767,544,403,398 -org.apache.solr.schema.BadIndexSchemaTest=1999,1993,1930,2216,1798,613 -org.apache.solr.schema.CopyFieldTest=1023,291,814,704,428,351 -org.apache.solr.schema.CurrencyFieldTest=698 -org.apache.solr.schema.DateFieldTest=16,17,12,10,16,24 -org.apache.solr.schema.IndexSchemaRuntimeFieldTest=442,420,368,327,447,406 -org.apache.solr.schema.IndexSchemaTest=439,449,470,522,438,1266 -org.apache.solr.schema.MultiTermTest=460,184,365,495,301,333 -org.apache.solr.schema.NotRequiredUniqueKeyTest=123,121,118,121,141,245 -org.apache.solr.schema.NumericFieldsTest=257,307,264,244,445,282 -org.apache.solr.schema.OpenExchangeRatesOrgProviderTest=20 -org.apache.solr.schema.PolyFieldTest=619,822,894,731,1312,598 -org.apache.solr.schema.PrimitiveFieldTypeTest=564 -org.apache.solr.schema.RequiredFieldsTest=463,518,490,531,474,607 -org.apache.solr.schema.TestBinaryField=212,224,229,231,639,250 -org.apache.solr.schema.TestCollationField=9,296,454,380,647,494 -org.apache.solr.schema.TestICUCollationField=2491,2367,2309,232,2482,810 -org.apache.solr.schema.TestICUCollationFieldOptions=2065,1893,1916,1745,1983,1954 -org.apache.solr.schema.TestOmitPositions=554,582,629,823,691,914 -org.apache.solr.schema.UUIDFieldTest=253,254,266,286,28,6 -org.apache.solr.search.QueryParsingTest=378,345,418,480,323,460 -org.apache.solr.search.ReturnFieldsTest=360 -org.apache.solr.search.SpatialFilterTest=644,862,652,855,1565,2630 -org.apache.solr.search.TestDocSet=687,625,601,795,439,434 -org.apache.solr.search.TestExtendedDismaxParser=419,477,476,493,1882,3720 -org.apache.solr.search.TestFastLRUCache=43,55,27,57,91,95 -org.apache.solr.search.TestFiltering=1695,1762,2065,2350,2900,2690 -org.apache.solr.search.TestFoldingMultitermQuery=701,509,667,509,1387,784 -org.apache.solr.search.TestIndexSearcher=565,528,620,558,476,569 -org.apache.solr.search.TestLFUCache=300,332,292,358,1728,377 -org.apache.solr.search.TestLRUCache=6,6,6,6,6,7 -org.apache.solr.search.TestPseudoReturnFields=1054,1145,1323,1178,1298,878 -org.apache.solr.search.TestQueryTypes=558,723,610,554,796,469 -org.apache.solr.search.TestQueryUtils=486,434,408,539,2475,698 -org.apache.solr.search.TestRangeQuery=3052,3155,3319,3579,8876,4956 -org.apache.solr.search.TestRealTimeGet=38908,10904,37602,36269,168006,55307 -org.apache.solr.search.TestRecovery=13035,4203,10753,13229,5746,5212 -org.apache.solr.search.TestSearchPerf=205,330,243,197,234,255 -org.apache.solr.search.TestSolrQueryParser=444,349,322,549,330,721 -org.apache.solr.search.TestSort=2493,2740,2523,2678,2288,4565 -org.apache.solr.search.TestSurroundQueryParser=1635,430,1089,1417,603,406 -org.apache.solr.search.TestValueSourceCache=501,499,652,558,458,539 -org.apache.solr.search.function.SortByFunctionTest=953,813,978,866,3944,865 -org.apache.solr.search.function.TestFunctionQuery=2923,3714,2794,5056,1793,2148 -org.apache.solr.search.function.distance.DistanceFunctionTest=894,706,634,596,2374,1905 -org.apache.solr.search.similarities.TestBM25SimilarityFactory=115,131,107,107,103,104 -org.apache.solr.search.similarities.TestDFRSimilarityFactory=67,76,112,96,101,191 -org.apache.solr.search.similarities.TestDefaultSimilarityFactory=187,75,170,158,78,130 -org.apache.solr.search.similarities.TestIBSimilarityFactory=234,68,122,177,66,85 -org.apache.solr.search.similarities.TestLMDirichletSimilarityFactory=1272,1300,1120,1175,79,134 -org.apache.solr.search.similarities.TestLMJelinekMercerSimilarityFactory=68,55,57,58,131,97 -org.apache.solr.search.similarities.TestPerFieldSimilarity=70,74,71,85,300,112 -org.apache.solr.servlet.CacheHeaderTest=682,714,660,824,2499,551 -org.apache.solr.servlet.DirectSolrConnectionTest=154,168,168,294,156,262 -org.apache.solr.servlet.NoCacheHeaderTest=775,770,915,713,738,548 -org.apache.solr.servlet.SolrRequestParserTest=1732,1613,1548,2117,1055,536 -org.apache.solr.spelling.DirectSolrSpellCheckerTest=1679,488,1159,1079,494,564 -org.apache.solr.spelling.FileBasedSpellCheckerTest=468,672,574,488,2696,506 -org.apache.solr.spelling.IndexBasedSpellCheckerTest=1367,1364,1401,1352,3147,666 -org.apache.solr.spelling.SpellCheckCollatorTest=889,708,912,964,966,565 -org.apache.solr.spelling.SpellPossibilityIteratorTest=38,37,33,34,50,49 -org.apache.solr.spelling.SpellingQueryConverterTest=6,6,7,9,4,24 -org.apache.solr.spelling.TestSuggestSpellingConverter=7 -org.apache.solr.spelling.suggest.SuggesterFSTTest=2825,2835,2856,2738,1065,743 -org.apache.solr.spelling.suggest.SuggesterTSTTest=632,855,796,879,1241,2627 -org.apache.solr.spelling.suggest.SuggesterTest=1980,605,1516,1304,3122,680 -org.apache.solr.spelling.suggest.SuggesterWFSTTest=571,750 -org.apache.solr.spelling.suggest.TestPhraseSuggestions=103 -org.apache.solr.uima.analysis.UIMAAnnotationsTokenizerFactoryTest=5499 -org.apache.solr.uima.analysis.UIMATypeAwareAnnotationsTokenizerFactoryTest=5442 -org.apache.solr.uima.processor.UIMAUpdateRequestProcessorTest=3940,3844,4066,3891,3873,5537 -org.apache.solr.update.AutoCommitTest=9271,9145,9368,9550,8352,9675 -org.apache.solr.update.DirectUpdateHandlerOptimizeTest=500,411,403,723,478,511 -org.apache.solr.update.DirectUpdateHandlerTest=3903,2586,3609,5868,1651,1709 -org.apache.solr.update.DocumentBuilderTest=966,621,903,1161,415,495 -org.apache.solr.update.PeerSyncTest=3435,3504,3263,3222,3232,2744 -org.apache.solr.update.SoftAutoCommitTest=10112,9973,10083,9950,11332,10454 -org.apache.solr.update.SolrCmdDistributorTest=1604,1613,1471,1342,1055,1004 -org.apache.solr.update.SolrIndexConfigTest=631 -org.apache.solr.update.TestIndexingPerformance=1260,530,1222,1755,899,665 -org.apache.solr.update.UpdateParamsTest=332,354,372,530,323,551 -org.apache.solr.update.processor.FieldMutatingUpdateProcessorTest=608,722,527,650,572,662 -org.apache.solr.update.processor.LangDetectLanguageIdentifierUpdateProcessorFactoryTest=2033,2090,2026,2078,2103,1977 -org.apache.solr.update.processor.SignatureUpdateProcessorFactoryTest=1282,1245,1174,1191,2107,1773 -org.apache.solr.update.processor.TikaLanguageIdentifierUpdateProcessorFactoryTest=2267,2271,2217,2231,2222,2102 -org.apache.solr.update.processor.URLClassifyProcessorTest=49 -org.apache.solr.update.processor.UniqFieldsUpdateProcessorFactoryTest=940,365,1044,1078,346,391 -org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest=378,332,375,381,1408,386 -org.apache.solr.util.DOMUtilTest=10 -org.apache.solr.util.DateMathParserTest=38,37,42,40,36,137 -org.apache.solr.util.FileUtilsTest=3 -org.apache.solr.util.PrimUtilsTest=27,29,24,27,14,54 -org.apache.solr.util.SolrPluginUtilsTest=517,533,500,522,346,554 -org.apache.solr.util.TestNumberUtils=210,250,290,354,176,210 -org.apache.solr.util.TestSystemIdResolver=13 -org.apache.solr.util.TestUtils=7,8,8,9,5,37 -org.apache.solr.velocity.VelocityResponseWriterTest=975,972,968,978,965,1273 -org.egothor.stemmer.TestCompile=239,269,209,186,266,241 -org.egothor.stemmer.TestStemmer=199,210,224,227,168,279 +org.apache.lucene.TestAssertions=39,30,37,26,58,40,26,39,31,56,189,47,48,251,36,32,32,229 +org.apache.lucene.TestDemo=40,56,89,35,39,68,209,72,31,56,88,33,56,79,31,31,1003,49 +org.apache.lucene.TestExternalCodecs=796,74,923,110,86,231,266,280,163,147,123,399,179,156,231,398,107,240 +org.apache.lucene.TestMergeSchedulerExternal=416,548,352,435,1030,322,396,375,465,541,482,1126,498,979,455,596,259,644 +org.apache.lucene.TestSearch=42,164,64,113,40,249,39,48,122,47,111,65,55,72,77,56,55,56 +org.apache.lucene.TestSearchForDuplicates=329,71,230,98,69,114,122,72,65,155,89,597,382,106,281,123,72,156 +org.apache.lucene.analysis.TestCachingTokenFilter=71,43,29,67,55,30,33,25,239,42,38,124,68,33,92,30,43,53 +org.apache.lucene.analysis.TestGraphTokenizers=8126,8562,8205,8426,9078,9650,13915,8285,12842,11875,6289,10195,13171,7628,8787,5925,12353,5502 +org.apache.lucene.analysis.TestLookaheadTokenFilter=3992,8465,4149,8191,5434,14093,5985,6505,6478,7811,6581,7867,7825,12854,7494,14377,9247,4785 +org.apache.lucene.analysis.TestMockAnalyzer=1330,3647,1107,3817,1047,2074,1616,1749,1418,922,1508,2878,1333,1203,1207,1449,1517,1611 +org.apache.lucene.analysis.TestMockCharFilter=25,58,25,48,140,40,33,38,48,65,27,27,234,26,50,128,51,210 +org.apache.lucene.analysis.TestNumericTokenStream=36,52,72,59,197,64,58,43,74,70,36,45,30,39,30,43,46,35 +org.apache.lucene.analysis.TestToken=1673,2027,2141,2198,1540,1870,1604,1525,2130,1509,2064,1576,2621,1601,1949,1726,1602,1357 +org.apache.lucene.analysis.ar.TestArabicAnalyzer=848,771,1902,1239,647,1507,2989,754,1562,955,651,1140,584,1624,547,2319,443,614 +org.apache.lucene.analysis.ar.TestArabicNormalizationFilter=146,70,128,85,175,63,92,58,57,56,97,347,79,150,61,178,63,99 +org.apache.lucene.analysis.ar.TestArabicStemFilter=389,131,65,61,506,64,50,118,226,98,144,96,134,127,177,154,374,73 +org.apache.lucene.analysis.bg.TestBulgarianAnalyzer=1715,839,2390,817,722,1372,638,973,1174,2821,979,1064,654,2212,621,1064,2579,655 +org.apache.lucene.analysis.bg.TestBulgarianStemmer=85,68,62,64,44,52,218,112,48,101,76,94,95,70,40,46,75,57 +org.apache.lucene.analysis.br.TestBrazilianStemmer=1449,1845,1555,925,1427,2544,1155,1206,1585,946,3530,1195,1190,1961,1205,2792,1150,1463 +org.apache.lucene.analysis.ca.TestCatalanAnalyzer=738,2149,1069,823,829,1730,969,869,1205,1407,972,906,705,1610,813,894,642,860 +org.apache.lucene.analysis.charfilter.HTMLStripCharFilterTest=5238,4518,3351,3432,6337,9373,5420,4202,3394,9616,8401,4055,3073,3932,5479,3800,4991,4506 +org.apache.lucene.analysis.charfilter.TestCharFilter=49,71,30,34,76,39,63,62,46,98,206,54,68,97,54,71,45,95 +org.apache.lucene.analysis.charfilter.TestMappingCharFilter=5315,4850,6308,5782,5421,4185,3421,6818,3791,7549,5822,3712,5892,4235,5340,9919,4084,6321 +org.apache.lucene.analysis.cjk.TestCJKAnalyzer=7086,3447,3989,2702,3896,3327,4928,4056,2393,2524,2169,3621,6851,3869,2045,4073,4248,3021 +org.apache.lucene.analysis.cjk.TestCJKWidthFilter=497,526,13950,2879,1079,960,653,454,6210,647,536,721,771,735,948,674,977,933 +org.apache.lucene.analysis.cn.smart.TestSmartChineseAnalyzer=7017,7878,7387,8025,6694,8026,13354,6943,6838,7946,8742,7741,7524,8517,8221,8080,7102,18657 +org.apache.lucene.analysis.commongrams.CommonGramsFilterTest=1896,1710,2686,1435,1621,2449,1937,1646,5185,2377,1427,1645,1039,2395,1521,1713,1460,1714 +org.apache.lucene.analysis.compound.TestCompoundWordTokenFilter=2860,2556,2896,3867,3190,2350,1950,4075,3669,5341,2367,5356,7227,4405,3837,4051,2263,2168 +org.apache.lucene.analysis.core.TestAnalyzers=5315,5607,6773,8660,12646,8990,18266,10913,22722,8269,5696,6341,10394,9071,5109,9498,7288,7249 +org.apache.lucene.analysis.core.TestBugInSomething=61,47,46,22,53,51,68,30,74,68,45,66,39,57,94,54,109,81 +org.apache.lucene.analysis.core.TestClassicAnalyzer=3036,4052,2602,2463,2612,4412,8806,3746,1912,13884,3415,3168,4848,4654,2316,2437,5986,2628 +org.apache.lucene.analysis.core.TestDuelingAnalyzers=18522,24687,28016,31478,27187,20831,26799,33446,19583,28715,31812,31088,31704,25309,23033,27902,27255,28128 +org.apache.lucene.analysis.core.TestKeywordAnalyzer=1553,519,2820,808,679,630,944,571,572,667,510,1875,1526,1205,10245,726,2375,763 +org.apache.lucene.analysis.core.TestRandomChains=6057,5290,4579,3170,7523,9607,4737,6289,6312,7471,5557,6496,4233,6284,5134,6188,3731,6939 +org.apache.lucene.analysis.core.TestStandardAnalyzer=6168,10076,6105,6464,7540,9575,5537,8511,5775,16923,5743,5916,9213,7593,5020,7539,5886,6612 +org.apache.lucene.analysis.core.TestStopAnalyzer=54,34,47,30,68,68,46,29,85,36,57,69,61,38,35,54,40,54 +org.apache.lucene.analysis.core.TestStopFilter=70,51,46,76,59,54,63,38,117,116,48,294,70,64,51,52,70,114 +org.apache.lucene.analysis.core.TestTypeTokenFilter=68,55,63,31,43,78,35,52,38,61,61,45,54,53,31,45,52,61 +org.apache.lucene.analysis.core.TestUAX29URLEmailAnalyzer=887,746,1385,818,2520,2997,676,809,1344,1760,1320,1509,519,1617,1851,838,824,977 +org.apache.lucene.analysis.core.TestUAX29URLEmailTokenizer=4550,2620,3219,7175,2254,4059,2867,2918,1926,9247,2208,3994,2417,3842,2470,3042,7990,2303 +org.apache.lucene.analysis.cz.TestCzechAnalyzer=852,1052,1187,977,754,1240,630,1080,2446,1181,678,976,3367,1688,730,970,2636,906 +org.apache.lucene.analysis.cz.TestCzechStemmer=61,63,78,107,60,83,75,57,87,345,95,73,71,62,61,128,308,194 +org.apache.lucene.analysis.da.TestDanishAnalyzer=839,1129,1087,746,771,1260,812,2794,1280,1130,803,770,388,2178,611,833,878,1246 +org.apache.lucene.analysis.de.TestGermanAnalyzer=931,3502,1271,896,888,1606,630,793,1421,1039,2782,3851,484,1972,853,2363,511,688 +org.apache.lucene.analysis.de.TestGermanLightStemFilter=823,913,1042,738,2271,1652,1256,1288,1330,2079,1613,579,683,1638,488,709,804,939 +org.apache.lucene.analysis.de.TestGermanMinimalStemFilter=1054,1436,967,1107,899,3604,1153,829,1302,721,1355,1023,4399,1581,610,752,536,4787 +org.apache.lucene.analysis.de.TestGermanNormalizationFilter=655,1079,897,1635,3824,1149,528,930,881,760,1543,3480,476,1270,362,846,414,1526 +org.apache.lucene.analysis.de.TestGermanStemFilter=753,1062,1107,636,547,1604,587,586,3311,593,1613,609,496,1672,2293,546,1420,474 +org.apache.lucene.analysis.el.GreekAnalyzerTest=1042,1517,1506,981,1084,2535,802,1527,4061,2081,1290,4696,1232,2387,904,1273,1127,1261 +org.apache.lucene.analysis.el.TestGreekStemmer=114,101,273,46,102,52,52,42,77,79,69,79,303,68,183,54,89,89 +org.apache.lucene.analysis.en.TestEnglishAnalyzer=1778,637,2113,655,851,1580,573,1113,1314,1055,1890,1037,1057,2389,904,777,668,1228 +org.apache.lucene.analysis.en.TestEnglishMinimalStemFilter=586,816,1155,346,547,1231,554,695,1281,660,469,653,628,1755,3043,765,538,588 +org.apache.lucene.analysis.en.TestKStemmer=1062,1077,1773,1237,904,2055,980,4058,2766,1110,745,3915,1262,1856,4615,1418,958,1199 +org.apache.lucene.analysis.en.TestPorterStemFilter=558,1156,1346,838,804,2269,578,1004,1454,1063,868,667,562,1646,1046,1810,787,1139 +org.apache.lucene.analysis.es.TestSpanishAnalyzer=536,1195,1680,621,733,2331,780,3288,1264,942,621,595,589,2564,404,719,730,728 +org.apache.lucene.analysis.es.TestSpanishLightStemFilter=645,781,1307,621,1187,2517,838,1445,1071,769,879,888,755,2640,1095,1563,1061,537 +org.apache.lucene.analysis.eu.TestBasqueAnalyzer=734,1163,1202,812,848,4775,1390,897,1156,1031,937,3380,862,1927,505,1216,1155,863 +org.apache.lucene.analysis.fa.TestPersianAnalyzer=822,2745,1527,523,1184,4288,580,986,1487,1430,837,869,463,4734,620,824,924,2343 +org.apache.lucene.analysis.fa.TestPersianNormalizationFilter=83,96,311,54,54,55,163,76,310,36,47,44,130,54,62,51,43,44 +org.apache.lucene.analysis.fi.TestFinnishAnalyzer=665,766,1205,488,981,2297,1121,1063,1149,616,679,961,712,1422,513,888,813,2725 +org.apache.lucene.analysis.fi.TestFinnishLightStemFilter=2388,1054,2537,885,1135,1639,4507,1081,1557,2206,884,5243,1062,1631,864,1143,830,1322 +org.apache.lucene.analysis.fr.TestElision=42,45,30,54,54,60,43,29,29,30,227,61,70,44,68,82,53,108 +org.apache.lucene.analysis.fr.TestFrenchAnalyzer=776,934,1174,560,1055,2129,686,1160,2902,1068,1011,768,922,2113,480,1001,740,947 +org.apache.lucene.analysis.fr.TestFrenchLightStemFilter=4429,768,1339,1538,704,1417,911,1401,1396,1181,573,968,1072,1764,2564,1296,792,728 +org.apache.lucene.analysis.fr.TestFrenchMinimalStemFilter=1296,921,955,807,772,5167,803,952,1452,738,679,780,674,1445,896,811,898,1271 +org.apache.lucene.analysis.ga.TestIrishAnalyzer=1125,902,1737,1125,2445,2327,553,2332,2715,973,913,870,1639,1922,1413,819,874,871 +org.apache.lucene.analysis.ga.TestIrishLowerCaseFilter=51,37,38,24,48,37,45,47,49,38,182,46,42,40,29,54,54,48 +org.apache.lucene.analysis.gl.TestGalicianAnalyzer=1445,4111,1450,918,1146,2596,928,1329,1947,2894,2867,981,871,4699,2797,1321,2313,1351 +org.apache.lucene.analysis.gl.TestGalicianMinimalStemFilter=646,775,1146,465,847,2286,528,705,2070,613,423,685,803,1172,541,595,506,645 +org.apache.lucene.analysis.gl.TestGalicianStemFilter=159,393,850,442,1034,291,194,411,261,727,346,150,529,109,264,288,323,420 +org.apache.lucene.analysis.hi.TestHindiAnalyzer=2380,1030,1359,779,934,1692,742,1221,1204,845,900,1052,1060,2297,2239,995,640,956 +org.apache.lucene.analysis.hi.TestHindiNormalizer=52,45,48,38,57,62,43,46,44,62,44,87,117,38,38,77,49,73 +org.apache.lucene.analysis.hi.TestHindiStemmer=70,95,67,55,49,51,45,54,54,70,52,36,74,109,86,47,43,363 +org.apache.lucene.analysis.hu.TestHungarianAnalyzer=915,864,1130,807,1022,1538,1063,785,1171,1146,538,964,3052,1729,1084,1143,803,736 +org.apache.lucene.analysis.hu.TestHungarianLightStemFilter=352,410,250,319,561,413,662,385,381,271,544,486,428,1888,471,463,463,405 +org.apache.lucene.analysis.hunspell.HunspellDictionaryTest=55,150,44,64,68,54,72,137,72,75,45,91,261,55,68,75,78,102 +org.apache.lucene.analysis.hunspell.HunspellStemFilterTest=1144,1277,2202,1554,820,1854,793,1339,1561,1121,1160,2961,765,2217,1196,1145,802,1211 +org.apache.lucene.analysis.hunspell.HunspellStemmerTest=95,56,55,133,85,64,108,103,94,103,61,143,61,218,59,87,108,45 +org.apache.lucene.analysis.hy.TestArmenianAnalyzer=769,1063,1047,547,950,1733,636,931,1346,3302,617,1176,529,2735,2594,2088,694,1705 +org.apache.lucene.analysis.icu.TestICUFoldingFilter=2946,5717,2316,1676,2924,3657,2786,2276,2598,2575,2581,2457,2558,2359,2504,3341,20029,2352 +org.apache.lucene.analysis.icu.TestICUNormalizer2Filter=2100,3368,2639,2083,2419,2361,2543,2822,2542,2496,2332,2632,3072,2611,2627,3518,1274,2267 +org.apache.lucene.analysis.icu.TestICUTransformFilter=3828,6008,3707,3119,4524,3932,4118,3357,4227,3464,3071,4096,3888,3466,3746,4205,12914,3105 +org.apache.lucene.analysis.icu.segmentation.TestCharArrayIterator=355,78,137,144,141,77,227,111,106,224,69,102,235,276,61,85,95,126 +org.apache.lucene.analysis.icu.segmentation.TestICUTokenizer=5499,10025,6936,5127,6878,6483,7212,4877,5808,5223,5205,5908,5877,5230,5995,5924,20409,5380 +org.apache.lucene.analysis.icu.segmentation.TestLaoBreakIterator=128,143,353,211,49,407,84,359,345,54,86,60,210,209,265,406,57,252 +org.apache.lucene.analysis.icu.segmentation.TestWithCJKBigramFilter=449,299,673,741,662,674,386,623,273,715,221,648,333,693,308,710,257,671 +org.apache.lucene.analysis.id.TestIndonesianAnalyzer=791,861,1062,655,3607,3158,510,841,1605,991,771,955,624,1533,660,944,580,690 +org.apache.lucene.analysis.id.TestIndonesianStemmer=63,91,158,52,61,44,85,52,68,46,38,29,61,55,71,43,97,53 +org.apache.lucene.analysis.in.TestIndicNormalizer=71,78,36,55,71,34,45,43,62,270,62,62,37,38,54,71,60,379 +org.apache.lucene.analysis.it.TestItalianAnalyzer=688,728,1233,580,820,1581,638,754,1178,1623,646,702,610,3519,421,517,572,2409 +org.apache.lucene.analysis.it.TestItalianLightStemFilter=1945,755,1396,1319,788,1745,467,946,1152,1072,628,727,632,6523,479,771,585,1120 +org.apache.lucene.analysis.ja.TestExtendedMode=20407,22421,16503,14729,14700,16012,12693,20738,21372,15339,18302,15551,19281,38139,17804,17637,22086,18810 +org.apache.lucene.analysis.ja.TestJapaneseAnalyzer=10935,17207,11204,14273,8595,14118,9742,13924,10351,11626,11290,7372,12891,24728,11077,11717,13058,11572 +org.apache.lucene.analysis.ja.TestJapaneseBaseFormFilter=4580,3317,4512,3712,3423,1703,5313,2509,5429,6656,5017,5552,4739,2536,5052,2913,4139,6194 +org.apache.lucene.analysis.ja.TestJapaneseKatakanaStemFilter=4559,2794,2921,1617,3175,3373,1628,2267,5667,2641,3923,2239,2472,2271,2406,2011,2330,3492 +org.apache.lucene.analysis.ja.TestJapaneseReadingFormFilter=1884,5737,1384,3887,2010,1733,2077,1615,1414,1400,1333,1165,866,3205,2337,1194,1147,3035 +org.apache.lucene.analysis.ja.TestJapaneseTokenizer=28554,31474,27389,26360,22703,25563,22833,29670,28694,27096,27404,22090,26597,48021,28204,24928,27286,27692 +org.apache.lucene.analysis.ja.TestSearchMode=66,117,108,100,83,655,83,463,579,92,75,92,453,103,100,601,76,75 +org.apache.lucene.analysis.ja.dict.TestTokenInfoDictionary=1113,1470,768,629,1100,981,508,925,550,1068,1396,720,633,837,1198,427,1102,778 +org.apache.lucene.analysis.ja.dict.UserDictionaryTest=209,144,227,121,388,158,129,275,92,125,185,177,396,214,117,134,159,160 +org.apache.lucene.analysis.ja.util.TestToStringUtil=263,86,107,102,95,75,90,103,112,138,111,53,153,143,86,96,89,61 +org.apache.lucene.analysis.lv.TestLatvianAnalyzer=591,803,2436,480,622,1498,704,2890,3332,814,1403,822,555,1339,573,3638,530,691 +org.apache.lucene.analysis.lv.TestLatvianStemmer=106,65,79,85,53,62,68,69,94,158,233,54,95,48,49,62,105,341 +org.apache.lucene.analysis.miscellaneous.TestASCIIFoldingFilter=763,659,1634,1992,776,1814,588,3492,1146,845,623,903,549,1428,862,861,687,801 +org.apache.lucene.analysis.miscellaneous.TestCapitalizationFilter=418,3602,786,477,900,735,808,1053,545,845,878,643,525,721,362,517,953,592 +org.apache.lucene.analysis.miscellaneous.TestEmptyTokenStream=38,48,30,55,39,62,46,22,37,33,38,67,119,34,63,41,277,39 +org.apache.lucene.analysis.miscellaneous.TestHyphenatedWordsFilter=595,606,628,2859,940,679,836,621,507,539,1479,761,586,364,611,410,1086,1279 +org.apache.lucene.analysis.miscellaneous.TestKeepWordFilter=528,631,1087,474,529,1369,578,588,1153,486,638,1757,1823,1526,410,601,468,589 +org.apache.lucene.analysis.miscellaneous.TestKeywordMarkerFilter=44,62,47,84,218,37,48,29,56,38,51,45,35,59,53,54,69,52 +org.apache.lucene.analysis.miscellaneous.TestLengthFilter=46,86,29,25,62,52,43,37,28,52,67,104,297,53,44,36,37,87 +org.apache.lucene.analysis.miscellaneous.TestLimitTokenCountAnalyzer=985,130,579,480,604,312,330,428,254,313,107,487,240,275,377,369,140,542 +org.apache.lucene.analysis.miscellaneous.TestPerFieldAnalzyerWrapper=38,53,55,41,65,45,43,30,37,38,62,126,37,49,53,53,47,70 +org.apache.lucene.analysis.miscellaneous.TestPrefixAndSuffixAwareTokenFilter=63,45,45,37,41,51,36,69,54,45,46,75,58,63,55,44,46,61 +org.apache.lucene.analysis.miscellaneous.TestPrefixAwareTokenFilter=288,53,47,41,45,40,35,38,180,38,44,80,42,42,81,67,60,52 +org.apache.lucene.analysis.miscellaneous.TestRemoveDuplicatesTokenFilter=5621,3430,3280,4498,3317,5502,1561,4640,2695,5612,3458,2894,2824,2361,2747,2442,3760,2779 +org.apache.lucene.analysis.miscellaneous.TestSingleTokenTokenFilter=31,43,46,204,39,32,54,55,47,87,56,39,38,64,31,55,133,110 +org.apache.lucene.analysis.miscellaneous.TestStemmerOverrideFilter=35,38,39,41,38,29,51,21,37,59,46,64,44,55,44,35,53,33 +org.apache.lucene.analysis.miscellaneous.TestTrimFilter=1056,1045,3493,4030,796,2173,1560,1480,3274,1571,4020,2101,1197,1789,1093,1356,818,854 +org.apache.lucene.analysis.miscellaneous.TestWordDelimiterFilter=3795,2461,962,2006,3238,1211,5561,1284,1780,1266,1339,2976,2777,2737,2700,4078,977,2493 +org.apache.lucene.analysis.morfologik.TestMorfologikAnalyzer=2073,2023,2189,1886,1830,2281,2460,2218,2828,11280,2339,1894,2117,2085,2195,2549,2413,1994 +org.apache.lucene.analysis.ngram.EdgeNGramTokenFilterTest=1380,2254,6361,1447,2626,2166,2162,2520,2790,2178,1536,2889,1475,2549,1335,2863,2604,2363 +org.apache.lucene.analysis.ngram.EdgeNGramTokenizerTest=6413,4117,4570,2456,2706,4305,3880,8797,4189,4320,4140,7913,2629,4350,2997,3176,3266,2820 +org.apache.lucene.analysis.ngram.NGramTokenFilterTest=3196,2538,1431,1244,1668,3000,1550,3478,2289,2116,1546,3314,1073,2536,1510,2035,2128,4729 +org.apache.lucene.analysis.ngram.NGramTokenizerTest=18482,17658,32199,13972,16080,23818,49923,18961,33438,19117,25742,21599,15754,23035,22815,23015,18742,17599 +org.apache.lucene.analysis.nl.TestDutchStemmer=1379,1430,2274,970,756,1639,983,991,3826,1379,852,1663,614,2147,853,1308,1594,938 +org.apache.lucene.analysis.no.TestNorwegianAnalyzer=679,1147,1199,470,778,1957,561,831,1087,952,644,1061,678,1865,464,3497,539,684 +org.apache.lucene.analysis.no.TestNorwegianLightStemFilter=806,1005,1154,529,1272,1548,853,812,1480,578,437,835,1029,2005,3128,1230,522,929 +org.apache.lucene.analysis.no.TestNorwegianMinimalStemFilter=566,982,1038,571,705,1556,888,710,1126,2384,565,547,670,2070,646,730,594,778 +org.apache.lucene.analysis.path.TestPathHierarchyTokenizer=1092,961,1433,788,793,1847,703,1004,1090,1043,775,2998,1239,4246,881,1069,1404,1936 +org.apache.lucene.analysis.path.TestReversePathHierarchyTokenizer=1803,993,1561,987,811,2011,587,1383,1210,3074,525,553,1770,2160,818,634,1614,787 +org.apache.lucene.analysis.pattern.TestPatternReplaceCharFilter=2411,10035,2763,2699,4101,9087,5997,2343,7843,10410,6634,6967,3033,7792,9375,2012,7051,2408 +org.apache.lucene.analysis.pattern.TestPatternReplaceFilter=851,2507,2147,1205,3588,2228,2479,1569,2290,1624,1206,1467,5606,4433,951,1353,1761,3636 +org.apache.lucene.analysis.pattern.TestPatternTokenizer=2160,2774,2140,1147,1193,2026,1161,1276,2008,1583,723,1352,797,1587,1003,953,1713,2012 +org.apache.lucene.analysis.payloads.DelimitedPayloadTokenFilterTest=55,62,151,46,54,64,45,290,39,71,35,62,63,46,38,229,114,37 +org.apache.lucene.analysis.payloads.NumericPayloadTokenFilterTest=42,46,44,54,62,30,167,26,31,58,35,264,62,39,70,42,37,39 +org.apache.lucene.analysis.payloads.TokenOffsetPayloadTokenFilterTest=39,37,33,242,46,62,37,33,55,73,36,45,41,30,43,37,42,30 +org.apache.lucene.analysis.payloads.TypeAsPayloadTokenFilterTest=38,45,30,30,50,38,46,88,138,30,43,55,47,36,52,79,59,38 +org.apache.lucene.analysis.phonetic.DoubleMetaphoneFilterTest=1973,3131,1989,1777,2381,2294,2748,2097,3404,2360,1980,2246,1827,2316,2203,3319,2276,2176 +org.apache.lucene.analysis.phonetic.TestBeiderMorseFilter=1127,1060,866,1116,852,1001,949,768,919,946,893,999,1040,1124,884,909,1087,814 +org.apache.lucene.analysis.phonetic.TestPhoneticFilter=5125,4914,8289,5082,5336,6064,14273,6869,6763,5715,12587,6772,4981,5149,5395,5049,5020,5223 +org.apache.lucene.analysis.pl.TestPolishAnalyzer=1904,1650,1606,1984,1720,1957,1756,1807,1906,1877,1866,1745,1853,2494,9258,1917,1779,1830 +org.apache.lucene.analysis.position.PositionFilterTest=40,65,63,47,54,70,62,39,47,46,61,61,58,87,37,66,76,53 +org.apache.lucene.analysis.pt.TestPortugueseAnalyzer=683,672,1288,439,659,1588,395,706,939,894,744,1021,496,1391,693,840,800,3403 +org.apache.lucene.analysis.pt.TestPortugueseLightStemFilter=1330,1155,1054,1043,3418,1521,3809,1254,1421,1330,714,2463,642,2971,1035,852,984,764 +org.apache.lucene.analysis.pt.TestPortugueseMinimalStemFilter=740,1436,1119,1047,770,1815,556,1238,2078,899,1153,1055,842,2388,695,1972,3031,1801 +org.apache.lucene.analysis.pt.TestPortugueseStemFilter=1862,1565,4502,997,1082,2054,2630,2171,1619,1374,1129,2061,829,3796,4069,1148,1091,4649 +org.apache.lucene.analysis.query.QueryAutoStopWordAnalyzerTest=253,245,346,162,198,502,340,313,235,432,347,201,509,329,147,304,528,484 +org.apache.lucene.analysis.reverse.TestReverseStringFilter=2526,3056,979,629,696,1201,399,701,939,621,1013,673,528,1638,426,2362,543,878 +org.apache.lucene.analysis.ro.TestRomanianAnalyzer=797,902,1189,675,2291,1439,1048,1056,1476,1172,762,774,828,1847,732,967,539,894 +org.apache.lucene.analysis.ru.TestRussianAnalyzer=1055,944,1085,2960,727,1570,739,1622,1025,743,562,616,485,1602,551,1596,513,688 +org.apache.lucene.analysis.ru.TestRussianLightStemFilter=1038,1013,5276,1005,1007,3024,826,1003,1364,1438,1157,1054,1412,1747,681,1188,948,1236 +org.apache.lucene.analysis.shingle.ShingleAnalyzerWrapperTest=503,198,349,411,456,1069,362,841,965,753,660,769,468,244,619,315,337,725 +org.apache.lucene.analysis.shingle.ShingleFilterTest=6724,4851,7637,5140,5489,5906,7495,6201,11096,18848,4398,5469,6111,13182,5572,5851,6882,6611 +org.apache.lucene.analysis.sinks.DateRecognizerSinkTokenizerTest=38,45,35,34,46,46,28,211,45,54,36,86,179,54,60,215,90,38 +org.apache.lucene.analysis.sinks.TestTeeSinkTokenFilter=133,87,161,87,95,95,61,76,45,102,135,152,151,238,52,85,90,125 +org.apache.lucene.analysis.sinks.TokenRangeSinkTokenizerTest=47,43,54,38,37,36,37,42,46,30,38,77,30,63,31,70,53,33 +org.apache.lucene.analysis.sinks.TokenTypeSinkTokenizerTest=244,37,29,60,37,36,37,31,21,49,37,90,62,46,36,46,37,36 +org.apache.lucene.analysis.snowball.TestSnowball=143,45,79,111,44,120,151,71,111,63,111,69,118,94,63,169,78,221 +org.apache.lucene.analysis.snowball.TestSnowballVocab=4907,4367,4453,4572,4246,6103,5087,4591,4316,5122,4682,4134,4549,5008,3762,4326,3709,4064 +org.apache.lucene.analysis.sv.TestSwedishAnalyzer=611,913,1360,3684,663,1769,639,2205,1374,1497,738,762,564,1523,446,3280,696,621 +org.apache.lucene.analysis.sv.TestSwedishLightStemFilter=815,848,1115,761,1147,4117,690,897,1744,1045,866,720,790,1491,2950,3684,2547,1277 +org.apache.lucene.analysis.synonym.TestSolrSynonymParser=95,471,40,60,55,167,59,151,60,79,69,52,65,359,118,67,87,75 +org.apache.lucene.analysis.synonym.TestSynonymMapFilter=31461,21112,21116,27579,30744,23232,21325,28831,23813,28441,21053,30284,26141,23887,23087,19259,22892,24471 +org.apache.lucene.analysis.synonym.TestWordnetSynonymParser=42,29,46,46,52,47,38,62,55,30,45,46,49,72,30,50,37,57 +org.apache.lucene.analysis.th.TestThaiAnalyzer=2979,3148,2226,2372,2620,4198,3528,2577,2234,2940,2242,2440,10281,4514,2821,2797,2894,3375 +org.apache.lucene.analysis.tokenattributes.TestCharTermAttributeImpl=669,1123,740,809,798,939,899,973,1607,863,746,1283,1199,682,1291,807,815,768 +org.apache.lucene.analysis.tokenattributes.TestSimpleAttributeImpl=56,39,26,39,251,38,38,37,47,39,32,86,21,64,31,65,63,39 +org.apache.lucene.analysis.tr.TestTurkishAnalyzer=3213,1106,1771,816,722,1864,1357,1782,1204,1315,621,743,730,1930,639,887,547,783 +org.apache.lucene.analysis.tr.TestTurkishLowerCaseFilter=38,46,37,37,30,45,36,32,38,54,37,49,51,54,62,54,77,76 +org.apache.lucene.analysis.uima.UIMABaseAnalyzerTest=7971,7461,8084,7818,7802,7794,8256,7919,7919,6791,7610,7310,7133,7758,7467,7844,7149,8111 +org.apache.lucene.analysis.uima.UIMATypeAwareAnalyzerTest=3343,2839,3006,3008,3417,2794,3499,2852,2951,3137,2670,2778,2485,3307,3816,3382,2990,3033 +org.apache.lucene.analysis.uima.ae.BasicAEProviderTest=356,303,77,69,57,337,321,66,74,53,352,283,83,75,302,76,75,72 +org.apache.lucene.analysis.uima.ae.OverridingParamsAEProviderTest=2307,3025,2709,2892,2369,2708,2291,2841,2935,2889,3060,2688,3023,2991,2240,2233,2943,2917 +org.apache.lucene.analysis.util.TestCharArrayIterator=455,267,255,166,221,447,206,379,522,236,246,413,578,431,213,236,380,323 +org.apache.lucene.analysis.util.TestCharArrayMap=103,240,280,110,456,264,337,122,116,329,244,363,145,387,543,114,166,140 +org.apache.lucene.analysis.util.TestCharArraySet=56,129,63,80,297,71,91,62,80,179,71,138,68,108,62,196,100,58 +org.apache.lucene.analysis.util.TestCharTokenizers=1874,1047,1746,1032,4129,4424,5031,1312,1081,2396,4343,12832,1443,4472,14787,1313,1492,2393 +org.apache.lucene.analysis.util.TestCharacterUtils=75,104,45,36,48,58,50,169,54,69,41,55,89,60,55,247,60,52 +org.apache.lucene.analysis.util.TestWordlistLoader=44,120,64,78,38,30,46,62,56,70,30,47,50,81,46,47,63,38 +org.apache.lucene.analysis.wikipedia.WikipediaTokenizerTest=2840,5286,6111,2230,3043,7734,2440,2638,3345,12311,2412,2382,7006,3494,4929,6493,7132,3470 +org.apache.lucene.benchmark.byTask.TestPerfTasksLogic=10263,9883,8898,9260,8519,9387,10046,11465,10741,10708,8708,10442,10152,9084,9666,10014,10048,9866 +org.apache.lucene.benchmark.byTask.TestPerfTasksParse=2177,1224,1235,2185,1694,1678,1899,1509,1455,2135,1597,1814,2246,2101,1303,1841,1324,2040 +org.apache.lucene.benchmark.byTask.feeds.DocMakerTest=436,387,546,334,568,244,402,528,264,453,428,527,397,360,488,378,310,344 +org.apache.lucene.benchmark.byTask.feeds.LineDocSourceTest=2142,2319,1773,2095,1794,1690,2000,2089,2167,1678,1856,2087,1341,1734,1967,1361,2388,1871 +org.apache.lucene.benchmark.byTask.feeds.TrecContentSourceTest=186,317,197,313,177,252,318,245,190,372,207,352,175,361,172,168,315,340 +org.apache.lucene.benchmark.byTask.feeds.demohtml.TestHtmlParser=311,122,330,99,133,118,146,312,285,142,271,185,188,219,125,179,201,174 +org.apache.lucene.benchmark.byTask.tasks.AddIndexesTaskTest=229,514,484,235,499,428,477,227,252,1012,514,219,235,262,488,211,598,229 +org.apache.lucene.benchmark.byTask.tasks.CreateIndexTaskTest=354,460,413,555,508,488,613,524,355,505,523,539,619,598,474,580,379,602 +org.apache.lucene.benchmark.byTask.tasks.PerfTaskTest=343,303,457,541,362,328,360,422,283,345,362,367,423,331,261,286,653,295 +org.apache.lucene.benchmark.byTask.tasks.SearchWithSortTaskTest=171,331,121,146,154,128,255,132,271,308,125,145,112,150,120,143,175,351 +org.apache.lucene.benchmark.byTask.tasks.WriteLineDocTaskTest=706,672,635,795,623,583,594,610,723,749,573,458,496,564,570,841,989,502 +org.apache.lucene.benchmark.byTask.tasks.alt.AltPackageTaskTest=286,212,223,197,194,220,268,244,199,155,218,196,209,139,253,170,223,216 +org.apache.lucene.benchmark.byTask.utils.StreamUtilsTest=170,382,308,138,225,317,241,342,355,142,251,483,331,167,151,374,344,261 +org.apache.lucene.benchmark.byTask.utils.TestConfig=50,144,60,46,75,68,48,42,229,76,65,328,252,46,83,68,115,55 +org.apache.lucene.benchmark.quality.TestQualityRun=3516,3200,2980,3913,3071,3225,3103,3465,3565,4229,3022,3214,3367,3227,3225,3542,4065,3340 +org.apache.lucene.codecs.appending.TestAppendingCodec=98,345,123,69,66,72,65,83,122,97,75,86,40,73,72,58,80,69 +org.apache.lucene.codecs.intblock.TestIntBlockCodec=86,156,64,37,48,47,47,82,48,49,48,73,40,31,48,53,39,47 +org.apache.lucene.codecs.lucene40.TestAllFilesHaveCodecHeader=148,213,140,1065,157,266,447,514,175,173,273,164,78,73,264,243,6883,303 +org.apache.lucene.codecs.lucene40.TestBitVector=3428,990,999,607,731,791,1849,1690,559,573,535,1013,730,899,1233,506,680,811 +org.apache.lucene.codecs.lucene40.TestLucene40PostingsReader=4308,3911,3302,2484,1424,2076,4371,5354,3433,2722,2340,4289,1868,1815,4338,8408,2146,6213 +org.apache.lucene.codecs.lucene40.TestReuseDocsEnum=413,964,415,297,366,439,781,352,789,385,708,574,590,557,1072,904,978,633 +org.apache.lucene.codecs.lucene40.values.TestDocValues=1224,1653,1090,560,1022,983,3297,923,281,632,603,706,565,683,859,345,1755,1200 +org.apache.lucene.codecs.perfield.TestPerFieldPostingsFormat=601,764,698,476,1053,592,1656,688,730,576,906,1868,688,320,1247,2740,386,2249 +org.apache.lucene.codecs.pulsing.Test10KPulsings=36,47,31,30,31,56,54,48,47,47,84,30,52,32,62,37,41,63 +org.apache.lucene.codecs.pulsing.TestPulsingReuse=70,89,65,106,56,63,64,73,55,72,55,146,272,299,60,87,55,32 +org.apache.lucene.collation.TestCollationKeyAnalyzer=1045,613,538,555,1626,685,412,1485,1617,505,661,835,604,1146,508,1009,669,728 +org.apache.lucene.collation.TestICUCollationKeyAnalyzer=1565,665,1385,1555,672,1446,2514,1148,845,1500,1532,724,2065,1199,1648,1875,2131,1532 +org.apache.lucene.demo.TestDemo=978,1559,1341,1095,1045,1203,1172,1321,926,1498,1322,1346,1197,1368,1341,1030,1062,1163 +org.apache.lucene.document.TestBinaryDocument=48,30,73,39,56,62,100,40,421,39,142,56,49,296,46,56,80,37 +org.apache.lucene.document.TestDateTools=35,64,41,56,146,69,54,64,74,106,45,63,97,155,94,81,307,63 +org.apache.lucene.document.TestDocument=163,363,99,82,154,105,56,51,64,107,112,288,169,69,106,221,147,97 +org.apache.lucene.facet.enhancements.EnhancementsPayloadIteratorTest=114,137,97,286,253,261,164,96,171,342,121,259,188,119,216,356,266,146 +org.apache.lucene.facet.enhancements.TwoEnhancementsTest=402,82,612,156,72,114,97,230,224,95,359,80,420,97,418,140,123,61 +org.apache.lucene.facet.enhancements.association.AssociationPropertyTest=33,55,59,199,73,115,43,51,31,68,38,38,64,71,110,63,49,29 +org.apache.lucene.facet.enhancements.association.CustomAssociationPropertyTest=87,71,96,61,90,316,220,89,85,170,123,203,296,577,62,70,96,64 +org.apache.lucene.facet.enhancements.params.DefaultEnhancementsIndexingParamsTest=47,51,40,54,39,30,30,67,30,27,39,53,39,40,43,53,55,36 +org.apache.lucene.facet.example.TestAdaptiveExample=72,446,79,55,65,104,47,141,294,329,153,102,323,131,77,99,55,456 +org.apache.lucene.facet.example.TestAssociationExample=107,124,161,113,71,56,133,138,122,215,254,168,120,337,91,122,72,113 +org.apache.lucene.facet.example.TestMultiCLExample=262,105,88,114,72,149,56,317,121,50,245,172,220,325,254,65,73,466 +org.apache.lucene.facet.example.TestSimpleExample=179,89,123,116,130,261,95,97,139,56,261,170,414,328,334,63,86,138 +org.apache.lucene.facet.index.CategoryContainerTest=131,138,136,69,235,262,120,324,175,104,104,85,102,129,334,102,139,243 +org.apache.lucene.facet.index.CategoryListPayloadStreamTest=38,44,58,38,37,40,31,46,53,72,64,47,55,52,87,66,36,39 +org.apache.lucene.facet.index.FacetsPayloadProcessorProviderTest=880,490,673,482,339,1906,264,671,615,290,914,1153,913,1230,206,474,1590,372 +org.apache.lucene.facet.index.attributes.CategoryAttributeImplTest=112,55,97,61,86,86,72,84,64,63,63,67,64,54,46,62,46,73 +org.apache.lucene.facet.index.attributes.CategoryAttributesIterableTest=45,47,38,40,54,55,35,37,45,73,48,64,79,35,50,29,43,103 +org.apache.lucene.facet.index.categorypolicy.OrdinalPolicyTest=997,521,488,614,198,1014,1119,739,281,159,145,160,540,1019,1072,104,1057,714 +org.apache.lucene.facet.index.categorypolicy.PathPolicyTest=180,629,497,432,470,533,619,312,347,1043,213,536,236,721,928,878,522,281 +org.apache.lucene.facet.index.params.CategoryListParamsTest=50,61,44,41,63,86,53,77,108,34,103,234,62,52,52,226,37,36 +org.apache.lucene.facet.index.params.DefaultFacetIndexingParamsTest=39,39,58,52,319,58,46,87,76,61,307,77,56,78,38,55,70,37 +org.apache.lucene.facet.index.params.PerDimensionIndexingParamsTest=47,50,44,77,138,310,54,72,286,56,65,73,80,64,60,47,41,46 +org.apache.lucene.facet.index.streaming.CategoryAttributesStreamTest=56,35,36,46,70,51,45,63,120,67,40,57,38,49,31,293,32,46 +org.apache.lucene.facet.index.streaming.CategoryParentsStreamTest=167,179,230,441,85,198,238,146,230,111,146,152,201,180,135,151,160,255 +org.apache.lucene.facet.index.streaming.CategoryTokenizerTest=80,62,170,80,466,133,41,63,162,87,129,128,80,110,93,155,491,41 +org.apache.lucene.facet.search.AdaptiveAccumulatorTest=11976,11863,13487,13294,19548,22385,13856,26736,13282,11849,18119,17558,12246,14013,11149,14650,8838,9245 +org.apache.lucene.facet.search.CategoryListIteratorTest=444,72,104,146,159,63,416,129,189,375,87,111,215,232,258,95,82,100 +org.apache.lucene.facet.search.DrillDownTest=207,217,618,229,205,63,53,157,609,139,357,250,99,149,168,241,104,275 +org.apache.lucene.facet.search.SamplingWrapperTest=18079,13150,11485,10790,28589,9409,9605,24060,10688,22423,12843,10351,14688,10586,10535,7890,25812,15091 +org.apache.lucene.facet.search.TestCategoryListCache=88,140,230,80,112,169,298,181,113,137,113,109,183,253,119,385,89,79 +org.apache.lucene.facet.search.TestFacetArrays=70,47,57,92,63,223,39,22,33,157,266,61,37,41,45,38,51,46 +org.apache.lucene.facet.search.TestFacetsAccumulatorWithComplement=309,305,135,293,246,353,233,201,152,87,61,129,352,129,260,225,203,144 +org.apache.lucene.facet.search.TestMultipleCategoryLists=205,629,667,496,516,533,446,894,338,169,450,305,256,218,147,256,280,130 +org.apache.lucene.facet.search.TestScoredDocIdCollector=101,118,88,70,71,213,84,78,129,83,110,163,141,241,80,389,54,104 +org.apache.lucene.facet.search.TestTopKInEachNodeResultHandler=262,1185,832,480,341,290,931,614,687,335,294,552,521,876,705,1039,951,639 +org.apache.lucene.facet.search.TestTopKResultsHandler=220,652,329,760,256,224,703,587,570,214,280,417,353,241,137,316,735,337 +org.apache.lucene.facet.search.TestTopKResultsHandlerRandom=2813,5890,5543,7166,6559,6507,6949,6539,7156,7838,6009,9889,6392,7269,5869,9269,3741,4166 +org.apache.lucene.facet.search.TestTotalFacetCounts=536,466,604,278,391,606,707,173,279,120,214,403,223,227,337,395,163,273 +org.apache.lucene.facet.search.TestTotalFacetCountsCache=2561,2781,5499,4409,2875,3866,3503,4369,6189,1813,5712,2204,2798,3043,1864,2029,5806,2729 +org.apache.lucene.facet.search.association.AssociationsFacetRequestTest=139,212,358,232,136,72,85,266,152,130,274,178,330,108,152,267,255,241 +org.apache.lucene.facet.search.params.FacetRequestTest=88,354,78,364,114,98,313,163,96,868,96,110,155,113,138,95,114,69 +org.apache.lucene.facet.search.params.FacetSearchParamsTest=80,105,130,161,102,106,237,227,102,146,322,431,348,121,205,80,72,474 +org.apache.lucene.facet.search.params.MultiIteratorsPerCLParamsTest=238,178,95,171,352,71,448,139,500,229,120,287,138,194,662,77,205,129 +org.apache.lucene.facet.search.sampling.SamplingAccumulatorTest=19331,12434,11593,13130,29853,12263,12800,25545,13233,30789,27586,10720,19314,9704,10376,13703,8858,24151 +org.apache.lucene.facet.taxonomy.TestCategoryPath=410,230,352,410,268,285,582,377,564,213,1045,322,836,520,415,427,837,275 +org.apache.lucene.facet.taxonomy.TestTaxonomyCombined=1526,1783,1671,1815,2149,1739,1489,1721,1359,1407,1555,2093,2092,1386,2110,2181,1548,1772 +org.apache.lucene.facet.taxonomy.directory.TestAddTaxonomy=6708,5974,4050,4675,4008,5674,7283,5833,3441,7061,5643,11475,4637,8407,11251,3716,2632,3388 +org.apache.lucene.facet.taxonomy.directory.TestDirectoryTaxonomyReader=554,413,1041,435,711,403,454,772,450,197,535,1516,219,305,582,250,604,396 +org.apache.lucene.facet.taxonomy.directory.TestDirectoryTaxonomyWriter=2417,2588,4756,7556,2011,3831,7097,4354,2056,1812,1992,3372,3334,5113,3966,4319,2729,2202 +org.apache.lucene.facet.taxonomy.writercache.cl2o.TestCharBlockArray=1603,1889,1664,1820,2053,2723,1751,1604,2455,2231,2108,2309,1506,2514,1347,1530,2457,1222 +org.apache.lucene.facet.taxonomy.writercache.cl2o.TestCompactLabelToOrdinal=814,714,912,748,917,987,1338,556,806,2022,1041,1359,1540,985,1411,908,843,697 +org.apache.lucene.facet.util.TestScoredDocIDsUtils=1446,138,877,349,313,588,459,425,1092,1020,487,251,1723,1130,271,466,3560,354 +org.apache.lucene.index.Test2BDocs=1676,3303,1940,866,923,899,805,997,2051,2487,859,1077,1207,2423,916,1024,1558,1594 +org.apache.lucene.index.Test2BPostings=86,87,94,40,86,91,88,31,62,30,23,368,31,48,31,105,121,31 +org.apache.lucene.index.Test2BTerms=54,70,55,28,27,110,29,33,86,103,52,96,44,45,49,112,62,122 +org.apache.lucene.index.TestAddIndexes=4730,3867,4389,4337,3551,3142,6350,2757,6835,4215,3312,5814,4273,3226,6174,9340,5983,3357 +org.apache.lucene.index.TestAtomicUpdate=7138,1801,5422,6219,4077,4576,5329,1616,2059,1579,2434,18547,1983,14455,2340,1834,3737,4364 +org.apache.lucene.index.TestBackwardsCompatibility=2178,1405,2517,2838,2451,2977,8382,2049,6345,2890,2088,2676,1765,1171,3307,3365,5489,1555 +org.apache.lucene.index.TestBalancedSegmentMergePolicy=11603,1626,2545,1975,2755,1786,1726,2112,2866,1851,3519,1920,3050,1594,3497,2627,6177,25620 +org.apache.lucene.index.TestBinaryTerms=175,59,156,107,39,105,48,81,189,247,48,72,310,712,113,148,89,279 +org.apache.lucene.index.TestByteSlices=4758,5063,4460,3906,5816,4531,4209,4369,3621,4876,4491,3935,4118,4685,5621,3807,3977,5369 +org.apache.lucene.index.TestCheckIndex=98,69,132,71,153,70,48,189,165,148,48,145,64,528,79,231,94,170 +org.apache.lucene.index.TestCodecs=1189,523,523,376,498,879,833,607,492,589,381,871,613,614,803,1565,984,857 +org.apache.lucene.index.TestCompoundFile=2412,2777,2347,1418,2421,1123,3250,1225,915,993,1750,958,1348,1649,1061,1315,2547,2877 +org.apache.lucene.index.TestConcurrentMergeScheduler=1891,2268,1854,551,2273,4744,4541,704,1190,1301,601,1838,1061,873,1208,1588,685,1109 +org.apache.lucene.index.TestConsistentFieldNumbers=1816,2679,1369,772,757,2037,1706,722,676,609,865,1289,1481,555,981,1197,567,1138 +org.apache.lucene.index.TestCrash=240,505,598,666,535,377,427,322,736,622,240,1030,523,861,629,972,372,391 +org.apache.lucene.index.TestCrashCausesCorruptIndex=1219,1038,1182,840,750,904,855,627,930,1209,769,2036,1183,1448,1276,929,916,1106 +org.apache.lucene.index.TestCustomNorms=3181,3097,4698,1363,1371,1829,1554,617,1968,456,1018,1580,1680,842,1867,647,929,2956 +org.apache.lucene.index.TestDeletionPolicy=4345,4418,9208,4814,5439,4291,5346,3373,4396,4427,2954,4215,3428,4540,3505,4154,3503,7441 +org.apache.lucene.index.TestDirectoryReader=1935,1529,1165,2272,719,1735,2422,4171,1089,2050,738,1334,571,708,619,1914,1920,1969 +org.apache.lucene.index.TestDirectoryReaderReopen=1733,2911,3057,2226,1972,2897,2093,1467,2214,1858,2239,2490,2137,2701,1708,2356,1839,1561 +org.apache.lucene.index.TestDoc=667,281,1996,97,148,178,887,111,120,483,1157,104,820,587,103,616,113,973 +org.apache.lucene.index.TestDocCount=57,140,57,39,123,132,164,148,284,163,156,95,44,55,96,108,347,109 +org.apache.lucene.index.TestDocTermOrds=514,322,273,151,483,108,214,228,185,290,322,297,103,248,139,183,107,253 +org.apache.lucene.index.TestDocValuesIndexing=2267,1323,744,1596,626,2656,3363,1263,1005,2788,846,977,993,1532,731,2275,8625,1196 +org.apache.lucene.index.TestDocValuesTypeCompatibility=1415,212,468,347,374,991,347,323,792,292,281,1181,582,1800,339,973,620,473 +org.apache.lucene.index.TestDocsAndPositions=997,477,215,5575,131,922,433,157,205,943,197,187,2092,226,654,228,314,172 +org.apache.lucene.index.TestDocumentWriter=1348,348,138,445,212,312,691,488,238,699,363,229,105,509,212,246,388,503 +org.apache.lucene.index.TestDocumentsWriterDeleteQueue=194,197,448,380,506,764,302,383,471,232,186,385,260,439,731,409,461,151 +org.apache.lucene.index.TestDocumentsWriterStallControl=1983,1236,1842,688,1236,2226,857,1457,356,714,648,1663,14403,5337,1039,1493,1198,2344 +org.apache.lucene.index.TestDuelingCodecs=7002,2311,4441,6554,1625,2035,2124,3258,3482,3309,4349,4524,19389,6498,5503,5305,23152,2466 +org.apache.lucene.index.TestFieldInfos=56,39,44,60,65,65,39,310,48,38,39,42,55,30,35,170,46,61 +org.apache.lucene.index.TestFieldsReader=420,1457,840,842,656,1331,2024,1242,2515,723,376,522,1362,346,2657,580,705,2458 +org.apache.lucene.index.TestFilterAtomicReader=128,42,123,121,47,69,80,66,72,72,100,80,52,138,89,140,98,81 +org.apache.lucene.index.TestFlex=491,457,157,264,92,290,134,391,300,502,122,647,596,282,91,123,257,365 +org.apache.lucene.index.TestFlushByRamOrCountsPolicy=2748,1906,1616,6356,2445,4021,3041,1364,4124,3597,3230,1145,5283,1973,6238,1930,2798,4697 +org.apache.lucene.index.TestForTooMuchCloning=119,511,285,584,124,380,267,98,129,550,246,415,148,265,867,321,586,116 +org.apache.lucene.index.TestForceMergeForever=343,729,398,456,373,1807,135,873,465,175,236,456,341,448,2557,805,1963,296 +org.apache.lucene.index.TestIndexCommit=49,56,31,30,47,22,28,30,31,56,30,47,79,40,289,45,39,39 +org.apache.lucene.index.TestIndexFileDeleter=231,63,74,66,147,230,130,48,130,352,131,48,81,146,71,72,97,106 +org.apache.lucene.index.TestIndexInput=443,797,497,600,254,347,451,389,566,919,319,1497,1160,649,464,817,832,522 +org.apache.lucene.index.TestIndexSplitter=1730,1756,1641,1731,1583,1592,1547,1562,779,1750,879,1484,1395,1676,838,1174,2135,1030 +org.apache.lucene.index.TestIndexWriter=6621,3913,3256,2532,5080,8052,9191,3755,4995,3281,3925,3919,3537,6911,3494,6761,3684,5211 +org.apache.lucene.index.TestIndexWriterCommit=3318,1553,867,1253,3177,1157,16888,1259,1146,1059,1946,1424,2372,1791,9540,2155,1531,1789 +org.apache.lucene.index.TestIndexWriterConfig=179,62,46,58,104,129,71,114,129,67,70,66,60,216,104,118,87,145 +org.apache.lucene.index.TestIndexWriterDelete=3695,1580,2223,2315,2007,1266,1380,2495,680,1512,9093,1535,26794,2832,4788,4258,1546,2221 +org.apache.lucene.index.TestIndexWriterExceptions=3313,9350,6952,11043,6355,6056,6953,3649,3176,2912,6786,7107,4994,3555,4196,5594,5372,5827 +org.apache.lucene.index.TestIndexWriterForceMerge=3808,2166,1951,2475,3256,1296,907,3623,2837,1765,1924,750,1298,1085,1241,1191,2111,3022 +org.apache.lucene.index.TestIndexWriterLockRelease=38,61,323,40,48,48,38,64,64,43,360,39,40,40,40,31,283,46 +org.apache.lucene.index.TestIndexWriterMergePolicy=5314,2789,4680,3085,5590,1740,5193,4722,3283,2244,1925,1707,2517,12030,3101,1595,3849,4460 +org.apache.lucene.index.TestIndexWriterMerging=4885,2289,3682,2324,2908,2324,2529,4840,2809,3914,10627,2175,7026,4516,6048,3652,2236,2381 +org.apache.lucene.index.TestIndexWriterNRTIsCurrent=2062,1575,1090,682,575,527,8962,996,1082,2908,1358,1447,857,1212,1836,1419,6524,1634 +org.apache.lucene.index.TestIndexWriterOnDiskFull=601,909,2577,281,825,1599,464,321,494,461,1020,232,639,535,531,982,236,999 +org.apache.lucene.index.TestIndexWriterOnJRECrash=95,59,47,57,245,65,37,87,56,38,21,154,36,35,56,39,68,145 +org.apache.lucene.index.TestIndexWriterReader=10360,5229,9807,5995,5888,13504,8979,5070,9485,14115,8220,4899,7396,6423,7873,5262,14695,6256 +org.apache.lucene.index.TestIndexWriterUnicode=2221,2765,3072,1883,2509,2063,2239,2227,2693,2285,2847,2383,2761,2557,2014,2228,2119,2289 +org.apache.lucene.index.TestIndexWriterWithThreads=16205,23039,16398,23726,24971,17148,25202,15817,17964,17948,46912,19876,26882,14150,16498,20122,27555,29099 +org.apache.lucene.index.TestIndexableField=333,499,200,131,199,191,189,359,297,273,250,257,348,198,297,236,181,154 +org.apache.lucene.index.TestIsCurrent=65,80,45,43,88,56,53,54,47,64,39,31,38,39,48,56,48,39 +org.apache.lucene.index.TestLazyProxSkipping=663,488,398,581,573,742,144,807,597,457,58,522,493,414,37,60,536,1014 +org.apache.lucene.index.TestLongPostings=1432,5430,3307,1575,603,1051,6323,3637,1866,2173,4890,1155,737,1148,3584,1403,1077,5349 +org.apache.lucene.index.TestMaxTermFrequency=206,195,182,165,906,973,761,201,994,281,173,1155,668,231,861,204,697,580 +org.apache.lucene.index.TestMixedCodecs=548,561,8388,608,539,290,906,524,890,390,432,1206,1174,555,624,1293,740,359 +org.apache.lucene.index.TestMultiFields=256,124,181,273,231,189,338,697,2905,356,141,1853,85,473,190,564,139,192 +org.apache.lucene.index.TestMultiLevelSkipList=231,479,346,294,280,213,721,172,237,156,95,213,220,98,255,405,287,153 +org.apache.lucene.index.TestMultiPassIndexSplitter=354,349,354,337,346,281,405,237,774,263,655,329,772,279,716,1401,271,760 +org.apache.lucene.index.TestNRTReaderWithThreads=1450,2348,1524,1375,1112,1140,1224,1333,1198,1221,1549,1343,1963,1274,1245,1515,1112,1431 +org.apache.lucene.index.TestNRTThreads=4542,4976,5631,5885,3420,5113,14755,3351,4458,5316,3924,4759,3163,3629,5326,12921,6789,4235 +org.apache.lucene.index.TestNeverDelete=3128,2746,3391,2521,1232,2277,10120,1106,1767,4558,3641,11553,1320,1156,3083,1954,4622,9993 +org.apache.lucene.index.TestNewestSegment=56,51,39,56,39,56,40,31,73,31,47,45,41,157,35,48,31,40 +org.apache.lucene.index.TestNoDeletionPolicy=73,148,106,264,149,64,336,96,155,122,64,83,79,55,228,80,90,56 +org.apache.lucene.index.TestNoMergePolicy=46,40,41,94,62,31,98,414,46,23,116,72,55,40,39,213,41,56 +org.apache.lucene.index.TestNoMergeScheduler=53,161,40,23,31,80,60,31,299,106,45,38,40,29,150,38,34,73 +org.apache.lucene.index.TestNorms=35629,5423,15039,6486,8045,9957,2992,7857,13156,24080,4554,6620,9622,7868,1861,21117,23353,6073 +org.apache.lucene.index.TestOmitNorms=780,570,1285,472,340,844,1260,604,1098,450,2366,474,1147,412,739,2740,883,1365 +org.apache.lucene.index.TestOmitPositions=239,255,98,95,321,340,94,82,204,222,132,331,206,89,292,99,123,214 +org.apache.lucene.index.TestOmitTf=257,584,294,484,487,528,1016,548,466,317,338,443,349,457,217,907,219,529 +org.apache.lucene.index.TestPKIndexSplitter=677,810,779,733,686,1299,618,717,914,519,894,766,832,626,715,856,561,888 +org.apache.lucene.index.TestParallelAtomicReader=115,256,356,435,264,89,248,88,189,163,86,325,222,726,171,108,480,238 +org.apache.lucene.index.TestParallelCompositeReader=2558,539,371,507,552,258,259,240,531,771,3245,333,556,1041,459,481,257,578 +org.apache.lucene.index.TestParallelReaderEmptyIndex=68,53,185,98,157,106,58,88,113,275,108,155,48,173,150,147,65,97 +org.apache.lucene.index.TestParallelTermEnum=39,108,22,73,64,454,90,202,48,55,32,56,56,48,81,102,55,131 +org.apache.lucene.index.TestPayloadProcessorProvider=355,236,2371,221,449,1764,195,176,589,1056,522,548,519,201,389,436,371,271 +org.apache.lucene.index.TestPayloads=185,323,146,370,138,114,124,130,232,388,321,189,272,182,512,214,197,172 +org.apache.lucene.index.TestPerSegmentDeletes=122,162,162,54,115,82,99,123,97,157,116,115,138,56,63,295,60,122 +org.apache.lucene.index.TestPersistentSnapshotDeletionPolicy=2886,2657,2415,2330,2541,2557,3021,2811,2663,3297,2876,4116,2699,2514,2622,2836,2522,3324 +org.apache.lucene.index.TestPostingsOffsets=2630,854,528,350,1729,392,823,300,719,747,361,1838,794,424,2638,1026,757,2087 +org.apache.lucene.index.TestPrefixCodedTerms=317,449,488,739,1106,140,1930,346,332,316,204,298,481,274,215,195,249,1349 +org.apache.lucene.index.TestRandomStoredFields=1148,784,481,304,339,499,893,716,1411,1303,1195,415,770,2455,656,443,750,2081 +org.apache.lucene.index.TestReaderClosed=98,57,35,68,64,411,56,86,57,41,33,64,48,47,65,165,82,40 +org.apache.lucene.index.TestRollback=353,36,31,32,39,48,65,31,448,73,58,47,47,220,29,45,233,73 +org.apache.lucene.index.TestRollingUpdates=1274,1093,833,2460,668,1437,899,634,501,526,1963,525,431,725,473,2337,1241,562 +org.apache.lucene.index.TestSameTokenSamePosition=47,48,138,204,62,48,86,27,64,152,65,58,55,46,56,64,106,71 +org.apache.lucene.index.TestSegmentMerger=815,552,965,822,796,449,812,415,1214,726,1707,1057,888,1696,1746,2069,667,423 +org.apache.lucene.index.TestSegmentReader=589,807,1314,535,4116,373,1170,773,642,408,783,1031,7940,545,608,459,512,1457 +org.apache.lucene.index.TestSegmentTermDocs=669,3885,430,1677,525,802,457,472,635,470,852,467,467,532,532,928,892,385 +org.apache.lucene.index.TestSegmentTermEnum=106,55,115,59,56,56,54,77,105,130,55,56,246,73,48,217,89,42 +org.apache.lucene.index.TestSizeBoundedForceMerge=272,78,164,172,100,154,123,109,106,104,205,164,202,501,380,744,142,118 +org.apache.lucene.index.TestSnapshotDeletionPolicy=1522,1563,1690,1481,1460,1399,1366,1505,1590,1262,1226,2024,1516,1760,1489,1681,1273,1882 +org.apache.lucene.index.TestStressAdvance=3840,606,473,2286,792,3928,975,573,530,558,1023,5349,932,602,2922,1181,1407,2359 +org.apache.lucene.index.TestStressIndexing=1722,1310,1624,1942,1474,1250,1679,1384,1796,1630,1490,1137,1590,1351,1441,1534,1144,2056 +org.apache.lucene.index.TestStressIndexing2=385,490,1514,327,506,880,867,589,519,1230,340,1249,381,598,636,3242,983,573 +org.apache.lucene.index.TestStressNRT=1671,1231,576,6967,231,1924,756,425,473,179,2124,531,2667,890,4538,1228,1632,1774 +org.apache.lucene.index.TestSumDocFreq=214,180,334,103,423,107,81,42,51,122,89,415,123,65,373,90,239,135 +org.apache.lucene.index.TestTerm=156,28,31,26,39,38,40,31,30,89,56,32,36,31,385,57,31,32 +org.apache.lucene.index.TestTermVectorsReader=359,247,124,835,170,248,183,482,1302,485,97,221,72,585,85,155,323,296 +org.apache.lucene.index.TestTermVectorsWriter=127,596,641,490,555,927,261,165,164,103,180,694,223,273,305,588,238,505 +org.apache.lucene.index.TestTermdocPerf=39,30,22,21,47,39,91,39,31,46,45,39,31,130,49,38,35,29 +org.apache.lucene.index.TestTermsEnum=2061,3268,3001,5088,6544,7486,1118,1827,4580,2789,7189,3987,5674,6029,3437,1120,2864,3941 +org.apache.lucene.index.TestTermsEnum2=424,302,405,519,416,1747,581,247,713,361,369,329,455,576,489,1797,720,480 +org.apache.lucene.index.TestThreadedForceMerge=2647,583,443,666,465,1783,1032,850,1584,3698,290,382,782,1922,724,674,365,1816 +org.apache.lucene.index.TestTieredMergePolicy=2454,1442,948,1342,879,1406,1503,998,3747,499,1331,2484,832,1220,840,594,1125,1865 +org.apache.lucene.index.TestTransactionRollback=466,281,386,1053,187,222,299,238,93,123,852,273,305,339,288,152,1496,173 +org.apache.lucene.index.TestTransactions=800,793,766,750,649,1031,800,897,894,892,854,831,757,714,602,896,758,765 +org.apache.lucene.index.TestTypePromotion=13112,1464,1192,3310,1085,1166,14210,1373,1522,26213,2387,1219,1691,21747,2064,20616,1274,761 +org.apache.lucene.index.TestUniqueTermCount=206,97,80,450,181,273,157,97,64,68,123,55,75,140,98,454,184,73 +org.apache.lucene.index.memory.MemoryIndexTest=4413,4349,4425,4549,4415,4641,4317,4516,4366,4556,4615,4423,4641,4553,4494,4652,4505,4557 +org.apache.lucene.misc.SweetSpotSimilarityTest=332,426,352,353,309,303,336,317,223,330,284,305,318,310,160,184,335,269 +org.apache.lucene.misc.TestHighFreqTerms=620,864,686,726,649,738,804,695,508,809,404,733,653,808,290,334,744,390 +org.apache.lucene.queries.BooleanFilterTest=557,606,709,1001,592,951,782,747,1009,525,689,879,588,808,791,810,2328,915 +org.apache.lucene.queries.BoostingQueryTest=60,45,130,197,28,193,53,43,63,59,45,42,61,71,132,45,90,314 +org.apache.lucene.queries.ChainedFilterTest=1840,1731,2022,2393,2270,2031,2135,1508,2268,1994,2554,1587,2288,2875,2124,2380,2067,2517 +org.apache.lucene.queries.TermsFilterTest=195,461,158,102,188,163,258,160,144,519,447,170,175,169,178,509,175,170 +org.apache.lucene.queries.TestCustomScoreQuery=3109,2753,2621,2503,3110,3090,3472,2413,2978,2555,3520,3421,3171,3253,3505,3596,2601,4078 +org.apache.lucene.queries.function.TestBoostedQuery=476,499,151,499,84,67,77,436,113,411,69,67,69,481,441,70,59,84 +org.apache.lucene.queries.function.TestFieldScoreQuery=476,934,277,611,853,636,1140,688,1069,615,1121,519,643,1211,1008,675,606,768 +org.apache.lucene.queries.function.TestOrdValues=631,236,694,444,585,427,252,257,385,691,354,306,728,523,549,310,188,804 +org.apache.lucene.queries.function.TestValueSources=816,814,546,473,396,470,440,653,649,479,447,795,337,365,871,395,459,401 +org.apache.lucene.queries.mlt.TestMoreLikeThis=112,103,432,146,426,284,145,441,108,120,85,113,195,170,96,142,104,120 +org.apache.lucene.queryparser.analyzing.TestAnalyzingQueryParser=170,331,101,144,158,304,67,104,47,120,41,88,52,49,121,136,44,75 +org.apache.lucene.queryparser.classic.TestMultiAnalyzer=91,58,139,317,49,71,70,41,55,313,68,292,39,112,110,316,116,138 +org.apache.lucene.queryparser.classic.TestMultiFieldQueryParser=110,526,253,203,450,597,239,261,183,95,163,185,73,120,114,261,154,82 +org.apache.lucene.queryparser.classic.TestMultiPhraseQueryParsing=165,60,337,53,32,68,57,83,78,43,55,38,150,61,62,42,38,32 +org.apache.lucene.queryparser.classic.TestQueryParser=879,708,1153,778,404,677,580,784,754,350,536,777,709,938,409,426,637,602 +org.apache.lucene.queryparser.complexPhrase.TestComplexPhraseQuery=120,220,583,170,287,269,159,88,260,73,73,260,365,119,123,130,95,109 +org.apache.lucene.queryparser.ext.TestExtendableQueryParser=841,846,615,594,486,567,400,1176,397,650,790,768,942,471,705,715,371,342 +org.apache.lucene.queryparser.ext.TestExtensions=54,45,53,70,44,43,99,70,78,76,239,61,376,70,249,36,89,78 +org.apache.lucene.queryparser.flexible.core.builders.TestQueryTreeBuilder=44,37,54,29,252,52,46,272,53,46,40,45,83,35,49,37,71,46 +org.apache.lucene.queryparser.flexible.core.nodes.TestQueryNode=45,51,51,52,221,42,44,52,51,35,46,50,56,48,54,52,37,86 +org.apache.lucene.queryparser.flexible.messages.TestNLS=64,61,136,104,66,70,93,139,119,71,272,74,376,102,248,54,149,96 +org.apache.lucene.queryparser.flexible.precedence.TestPrecedenceQueryParser=695,391,340,305,329,254,337,342,438,190,288,512,312,580,753,461,356,662 +org.apache.lucene.queryparser.flexible.spans.TestSpanQueryParser=95,62,144,135,92,55,76,64,62,116,73,34,185,115,79,78,94,94 +org.apache.lucene.queryparser.flexible.spans.TestSpanQueryParserSimpleSample=112,37,76,44,245,81,54,70,35,32,39,69,73,86,54,54,42,68 +org.apache.lucene.queryparser.flexible.standard.TestMultiAnalyzerQPHelper=75,350,113,113,81,291,144,58,94,136,187,88,170,99,155,38,439,95 +org.apache.lucene.queryparser.flexible.standard.TestMultiFieldQPHelper=149,340,383,282,244,291,158,610,252,117,345,376,376,155,125,686,134,205 +org.apache.lucene.queryparser.flexible.standard.TestNumericQueryParser=764,260,705,524,466,476,811,473,810,899,565,921,483,699,400,474,786,310 +org.apache.lucene.queryparser.flexible.standard.TestQPHelper=643,599,641,524,798,510,491,737,453,1151,779,877,595,630,388,1133,530,632 +org.apache.lucene.queryparser.surround.query.SrndQueryTest=405,107,207,117,143,143,390,165,355,440,291,367,102,362,110,166,441,86 +org.apache.lucene.queryparser.surround.query.Test01Exceptions=88,35,120,61,69,254,29,79,54,33,221,88,364,35,53,72,55,66 +org.apache.lucene.queryparser.surround.query.Test02Boolean=679,588,429,828,425,341,429,329,404,617,452,444,553,537,747,458,611,594 +org.apache.lucene.queryparser.surround.query.Test03Distance=1512,1708,1733,1665,1192,1369,2213,1260,1692,1953,1766,1427,1269,1672,1183,1236,1206,1570 +org.apache.lucene.queryparser.xml.TestParser=3580,2742,1896,2351,2117,1588,2431,1793,1694,3509,2043,3261,3192,2454,2976,2190,2996,2584 +org.apache.lucene.queryparser.xml.TestQueryTemplateManager=479,495,419,497,585,619,534,325,396,532,480,634,626,722,357,548,480,453 +org.apache.lucene.queryparser.xml.builders.TestNumericRangeFilterBuilder=168,197,183,277,95,119,307,377,72,262,142,180,198,123,227,202,112,437 +org.apache.lucene.queryparser.xml.builders.TestNumericRangeQueryBuilder=91,210,224,135,155,186,119,284,85,61,252,226,58,74,129,152,74,293 +org.apache.lucene.sandbox.queries.DuplicateFilterTest=140,262,175,508,350,260,295,219,459,225,251,267,286,358,384,269,643,632 +org.apache.lucene.sandbox.queries.FuzzyLikeThisQueryTest=250,648,616,536,499,367,194,558,619,583,547,536,655,227,564,642,544,203 +org.apache.lucene.sandbox.queries.TestSlowCollationMethods=5193,1973,4078,2314,2098,1479,4452,3739,12152,5379,2101,2487,2343,4490,8055,2151,1896,4607 +org.apache.lucene.sandbox.queries.TestSlowFuzzyQuery=936,383,326,651,398,724,855,360,299,425,348,359,342,975,342,451,466,839 +org.apache.lucene.sandbox.queries.TestSlowFuzzyQuery2=1164,1285,1479,786,970,945,901,1229,1079,1502,1153,1241,1134,1120,1572,1358,816,786 +org.apache.lucene.sandbox.queries.regex.TestJakartaRegexpCapabilities=70,70,149,238,55,61,229,68,227,43,66,72,242,62,75,48,60,79 +org.apache.lucene.sandbox.queries.regex.TestRegexQuery=878,568,471,291,580,744,748,595,335,450,392,470,482,689,965,376,346,677 +org.apache.lucene.sandbox.queries.regex.TestSpanRegexQuery=393,104,74,140,450,504,297,110,142,78,553,60,106,407,227,67,232,98 +org.apache.lucene.search.MultiCollectorTest=47,70,40,40,48,47,80,30,62,69,75,31,71,25,40,57,55,32 +org.apache.lucene.search.TestAutomatonQuery=97,165,152,123,140,214,164,156,428,545,114,372,108,240,134,105,154,238 +org.apache.lucene.search.TestAutomatonQueryUnicode=45,124,188,138,104,197,39,43,63,55,34,116,31,45,64,63,73,47 +org.apache.lucene.search.TestBoolean2=7528,1288,5484,1082,7284,1452,7993,6547,6042,4165,5968,7748,1430,4879,3929,3743,6707,7908 +org.apache.lucene.search.TestBooleanMinShouldMatch=1056,541,851,1183,784,2067,649,572,671,598,1267,1189,583,606,701,693,575,418 +org.apache.lucene.search.TestBooleanOr=457,140,131,232,98,132,171,198,331,139,347,115,141,780,447,936,139,172 +org.apache.lucene.search.TestBooleanQuery=272,139,164,272,106,347,247,180,81,150,929,237,179,214,216,380,320,70 +org.apache.lucene.search.TestBooleanScorer=47,68,143,48,194,236,239,93,68,57,56,348,105,72,56,56,264,57 +org.apache.lucene.search.TestCachingCollector=64,55,57,48,71,42,73,56,37,39,80,73,63,365,40,66,56,30 +org.apache.lucene.search.TestCachingWrapperFilter=120,75,64,87,71,79,88,122,80,47,62,163,105,218,81,95,97,209 +org.apache.lucene.search.TestComplexExplanations=2204,1445,1940,1052,2485,825,959,982,940,969,1811,1853,1042,1040,1614,956,999,1156 +org.apache.lucene.search.TestComplexExplanationsOfNonMatches=81,264,71,504,81,69,121,230,255,227,331,73,155,357,1161,130,115,94 +org.apache.lucene.search.TestConstantScoreQuery=223,72,79,55,71,147,54,73,51,146,62,72,54,97,61,122,64,88 +org.apache.lucene.search.TestCustomSearcherSort=708,496,496,555,431,776,464,883,446,338,313,457,630,306,952,6762,781,613 +org.apache.lucene.search.TestDateFilter=90,79,153,45,73,123,65,48,115,39,171,105,80,48,97,177,64,60 +org.apache.lucene.search.TestDateSort=127,89,55,55,57,72,59,56,143,73,70,129,56,39,33,56,31,101 +org.apache.lucene.search.TestDisjunctionMaxQuery=933,461,249,286,721,738,1105,431,363,339,248,252,1198,704,251,380,622,1252 +org.apache.lucene.search.TestDocBoost=99,55,113,56,31,48,31,70,59,82,74,55,48,40,50,73,40,55 +org.apache.lucene.search.TestDocIdSet=35,224,211,47,94,181,39,222,64,45,93,88,36,88,45,63,80,114 +org.apache.lucene.search.TestDocValuesScoring=81,98,89,89,71,150,55,92,306,83,296,298,146,69,96,299,246,72 +org.apache.lucene.search.TestElevationComparator=97,73,74,50,56,56,97,105,40,113,56,37,47,180,64,169,63,63 +org.apache.lucene.search.TestExplanations=54,103,64,65,41,48,74,38,150,146,89,48,38,50,55,32,71,88 +org.apache.lucene.search.TestFieldCache=556,1365,2983,666,722,405,523,6054,380,404,2242,1894,639,832,405,353,3206,1324 +org.apache.lucene.search.TestFieldCacheRangeFilter=662,836,204,153,1077,342,171,1690,454,721,1286,196,941,774,220,305,190,775 +org.apache.lucene.search.TestFieldCacheRewriteMethod=921,806,1606,740,2472,4229,846,1079,1740,1082,2593,539,2552,1091,773,1922,721,1273 +org.apache.lucene.search.TestFieldCacheTermsFilter=48,71,47,64,73,76,49,175,47,106,47,45,64,48,73,57,89,41 +org.apache.lucene.search.TestFieldValueFilter=80,64,81,40,56,115,238,105,245,355,105,234,81,65,67,72,48,89 +org.apache.lucene.search.TestFilteredQuery=164,196,212,288,254,875,560,405,515,205,152,729,1641,369,417,214,262,581 +org.apache.lucene.search.TestFilteredSearch=81,114,98,97,190,107,139,164,56,110,73,65,80,64,48,196,80,56 +org.apache.lucene.search.TestFuzzyQuery=455,455,130,147,389,121,191,97,148,181,697,145,88,241,214,133,104,446 +org.apache.lucene.search.TestMatchAllDocsQuery=206,64,48,48,47,47,56,64,65,104,67,122,398,43,148,114,54,55 +org.apache.lucene.search.TestMultiPhraseQuery=177,282,126,433,262,139,231,419,239,348,493,588,277,202,328,179,117,133 +org.apache.lucene.search.TestMultiTermConstantScore=448,1352,157,284,324,211,364,364,262,521,229,147,1160,297,316,415,213,1452 +org.apache.lucene.search.TestMultiTermQueryRewrites=122,156,54,52,57,106,78,73,117,186,198,79,90,64,70,72,72,47 +org.apache.lucene.search.TestMultiThreadTermVectors=339,295,589,867,1154,580,501,588,326,356,648,311,365,542,306,458,681,774 +org.apache.lucene.search.TestMultiValuedNumericRangeQuery=798,3980,1242,415,1573,389,572,640,1010,452,4082,2375,601,414,1043,1058,399,883 +org.apache.lucene.search.TestNGramPhraseQuery=49,42,40,63,37,28,47,73,31,56,265,163,48,160,45,164,48,65 +org.apache.lucene.search.TestNRTManager=4097,3480,2558,5199,2611,2666,2389,4042,2903,2338,2863,2605,1664,2485,1996,2001,5206,2294 +org.apache.lucene.search.TestNot=57,52,48,55,90,80,81,194,56,157,74,28,39,32,64,86,64,47 +org.apache.lucene.search.TestNumericRangeQuery32=1438,7492,2280,1680,9379,6513,2146,5276,3790,2481,2988,5486,8881,5436,7925,2130,6096,3491 +org.apache.lucene.search.TestNumericRangeQuery64=4797,8844,3363,4695,15803,14810,4123,4813,2801,9418,3871,7193,4442,5772,9701,12345,10858,3272 +org.apache.lucene.search.TestPhrasePrefixQuery=66,48,122,179,35,48,47,88,187,96,74,72,64,23,30,339,114,56 +org.apache.lucene.search.TestPhraseQuery=593,786,558,1137,825,1527,1404,1086,1411,5397,4262,688,1007,1414,2552,3821,1177,644 +org.apache.lucene.search.TestPositionIncrement=72,104,130,139,65,80,48,43,54,72,64,63,39,274,129,80,63,78 +org.apache.lucene.search.TestPositiveScoresOnlyCollector=30,97,63,74,466,47,58,33,39,34,40,39,56,24,89,53,48,39 +org.apache.lucene.search.TestPrefixFilter=47,56,47,30,30,59,71,40,40,21,70,49,80,55,44,48,31,85 +org.apache.lucene.search.TestPrefixInBooleanQuery=799,256,2736,623,657,4185,1101,1744,601,262,6441,373,1275,564,499,247,2989,790 +org.apache.lucene.search.TestPrefixQuery=52,78,71,48,234,158,56,85,110,63,41,31,47,39,47,56,39,83 +org.apache.lucene.search.TestPrefixRandom=1657,389,707,283,194,1040,373,736,625,144,265,147,336,287,98,117,1289,1337 +org.apache.lucene.search.TestQueryWrapperFilter=193,148,356,1245,240,489,249,1114,266,319,489,1108,1138,239,314,456,136,698 +org.apache.lucene.search.TestRegexpQuery=125,305,82,105,106,81,73,74,258,363,91,255,81,53,474,157,229,240 +org.apache.lucene.search.TestRegexpRandom=156,206,421,316,208,273,105,143,165,136,66,74,81,274,447,198,148,248 +org.apache.lucene.search.TestRegexpRandom2=2833,638,1487,1347,912,923,1203,3630,2115,837,774,2294,639,632,1822,811,1578,1908 +org.apache.lucene.search.TestScoreCachingWrappingScorer=73,69,88,31,73,39,42,61,48,47,27,28,30,47,40,39,47,136 +org.apache.lucene.search.TestScorerPerf=424,355,450,563,508,713,2239,522,754,491,291,438,2666,1489,307,303,816,366 +org.apache.lucene.search.TestSearchAfter=2632,1859,1778,2815,3396,3701,2051,4054,2886,3006,1967,5365,5017,21488,2365,1218,1574,2770 +org.apache.lucene.search.TestSearchWithThreads=1551,5941,1657,11448,1731,1835,2761,2173,2352,2207,2109,1930,1591,3790,3818,1958,2058,2300 +org.apache.lucene.search.TestSearcherManager=3781,2876,5306,7841,3642,8873,9252,22287,7997,8663,2497,2942,4686,2556,6268,5513,3768,6816 +org.apache.lucene.search.TestShardSearching=11949,4931,4588,5030,5666,7098,4667,7329,4245,6592,6670,8107,8495,5210,7130,9084,8687,4495 +org.apache.lucene.search.TestSimilarity=69,64,86,225,33,28,46,67,85,86,69,52,38,33,54,46,46,78 +org.apache.lucene.search.TestSimilarityProvider=64,138,41,256,49,73,48,89,185,40,89,48,56,71,39,87,46,38 +org.apache.lucene.search.TestSimpleExplanations=1073,1612,1705,3775,3637,1057,2520,948,1609,3671,3128,1248,1827,1372,961,2740,1206,872 +org.apache.lucene.search.TestSimpleExplanationsOfNonMatches=865,151,326,153,160,161,153,669,195,183,198,401,192,134,161,178,352,452 +org.apache.lucene.search.TestSimpleSearchEquivalence=356,159,291,279,2268,2495,321,764,2415,816,162,832,173,504,1674,448,446,272 +org.apache.lucene.search.TestSloppyPhraseQuery=3975,5134,2755,1399,1846,2889,2498,3802,2535,3544,4114,1438,2133,2649,1629,4725,1937,3046 +org.apache.lucene.search.TestSloppyPhraseQuery2=692,2988,3955,925,1557,489,2083,1267,433,1203,400,606,1739,696,1542,1988,822,1245 +org.apache.lucene.search.TestSort=1898,3374,2010,4651,7486,1230,3327,2203,4075,5959,2537,1257,2826,2314,1706,5677,2411,1244 +org.apache.lucene.search.TestSubScorerFreqs=47,65,121,251,40,185,89,488,504,240,46,64,38,65,85,76,98,73 +org.apache.lucene.search.TestTermRangeFilter=962,129,396,1476,147,214,557,2042,371,314,947,281,141,397,1071,1101,213,150 +org.apache.lucene.search.TestTermRangeQuery=164,296,452,95,70,281,458,223,104,132,174,216,89,296,103,620,110,1972 +org.apache.lucene.search.TestTermScorer=164,55,138,46,56,97,138,65,40,56,64,97,87,47,64,498,215,248 +org.apache.lucene.search.TestTermVectors=300,525,314,140,1095,197,448,247,180,306,257,1800,264,430,531,264,204,847 +org.apache.lucene.search.TestTimeLimitingCollector=1561,5562,2047,6779,2022,1920,6393,5023,9654,3235,3671,2892,2108,3111,2414,2606,3187,1608 +org.apache.lucene.search.TestTopDocsCollector=100,73,57,90,124,54,112,216,533,81,278,253,55,69,74,123,70,121 +org.apache.lucene.search.TestTopDocsMerge=1799,2176,1516,3507,2711,1699,4903,3436,4281,2767,2153,4373,3727,3913,2995,2836,3622,2743 +org.apache.lucene.search.TestTopScoreDocCollector=40,35,39,46,48,189,91,507,75,121,50,68,33,41,63,56,73,30 +org.apache.lucene.search.TestWildcard=263,107,105,227,589,163,180,165,172,122,188,836,123,79,100,147,107,477 +org.apache.lucene.search.TestWildcardRandom=156,122,81,247,171,99,180,785,351,132,58,215,339,123,91,216,74,56 +org.apache.lucene.search.grouping.AllGroupHeadsCollectorTest=5382,3124,5386,5141,5887,3641,6603,5935,6445,5246,5118,9220,6577,4748,6670,5183,4792,7589 +org.apache.lucene.search.grouping.AllGroupsCollectorTest=137,212,460,129,152,242,486,538,479,542,186,507,112,195,127,496,516,134 +org.apache.lucene.search.grouping.DistinctValuesCollectorTest=3369,2008,3730,2641,2808,2408,5334,2001,4235,8182,3492,4081,3833,4042,3906,2907,3406,4211 +org.apache.lucene.search.grouping.GroupFacetCollectorTest=3286,4513,4261,2885,4096,3329,3997,7489,3711,983,2443,6096,3889,4117,4010,6003,3474,5194 +org.apache.lucene.search.grouping.GroupingSearchTest=218,568,158,184,560,210,134,139,135,211,517,161,144,550,498,244,228,159 +org.apache.lucene.search.grouping.TestGrouping=8494,7303,8497,7378,6909,8634,9043,9821,8771,7226,9320,11361,8606,9078,9084,9912,7104,9275 +org.apache.lucene.search.highlight.HighlighterPhraseTest=247,234,114,113,171,593,636,151,664,100,609,133,548,597,600,134,579,282 +org.apache.lucene.search.highlight.HighlighterTest=1424,1624,2098,1704,1521,1518,2149,1786,2161,2151,1149,2279,1419,2080,1731,2596,1289,1137 +org.apache.lucene.search.highlight.OffsetLimitTokenFilterTest=46,64,48,45,64,27,58,60,45,66,281,78,60,80,54,71,102,68 +org.apache.lucene.search.highlight.TokenSourcesTest=196,511,128,140,146,217,187,170,168,551,161,449,517,562,528,578,517,334 +org.apache.lucene.search.highlight.custom.HighlightCustomQueryTest=111,69,79,86,54,63,78,55,53,47,72,60,91,46,77,97,67,320 +org.apache.lucene.search.join.TestBlockJoin=3292,3294,4588,4589,3936,3445,4222,3494,3460,4482,4184,3753,3382,5191,4278,3801,4182,4878 +org.apache.lucene.search.join.TestJoinUtil=22759,50368,24051,268199,53788,47014,17275,30591,180860,60593,41343,13515,7358,59730,15905,49900,89896,214761 +org.apache.lucene.search.payloads.TestPayloadExplanations=164,313,205,1177,497,356,795,994,704,599,273,331,221,361,565,254,219,401 +org.apache.lucene.search.payloads.TestPayloadNearQuery=542,164,177,143,377,222,310,3615,429,437,1020,430,196,443,415,186,395,301 +org.apache.lucene.search.payloads.TestPayloadTermQuery=383,2196,478,236,269,420,357,1121,152,545,526,321,975,641,182,1411,869,492 +org.apache.lucene.search.similarities.TestSimilarity2=164,173,239,262,279,111,381,274,322,793,239,130,248,224,107,380,153,283 +org.apache.lucene.search.similarities.TestSimilarityBase=512,1147,476,592,670,225,674,506,272,305,2717,523,873,439,587,323,409,914 +org.apache.lucene.search.spans.TestBasics=2635,2648,3270,6363,3425,3580,3831,5440,3086,5298,5369,2424,4636,5853,7226,9254,4813,7600 +org.apache.lucene.search.spans.TestFieldMaskingSpanQuery=264,526,596,431,405,412,338,272,264,266,263,466,313,752,565,388,438,1088 +org.apache.lucene.search.spans.TestNearSpansOrdered=159,324,478,106,122,313,554,87,123,114,97,192,189,96,124,386,95,98 +org.apache.lucene.search.spans.TestPayloadSpans=990,440,406,603,1833,589,848,3110,501,413,420,629,602,693,1684,3031,1942,581 +org.apache.lucene.search.spans.TestSpanExplanations=2221,410,308,772,516,680,1524,1007,539,428,1866,1565,276,419,365,535,834,713 +org.apache.lucene.search.spans.TestSpanExplanationsOfNonMatches=99,122,89,89,347,268,254,120,80,614,248,148,80,281,253,163,99,236 +org.apache.lucene.search.spans.TestSpanFirstQuery=39,56,32,81,39,71,86,40,31,60,64,105,23,31,28,56,49,96 +org.apache.lucene.search.spans.TestSpanMultiTermQueryWrapper=89,246,140,181,89,499,214,264,63,157,73,65,189,207,354,130,64,96 +org.apache.lucene.search.spans.TestSpanSearchEquivalence=574,203,617,549,1425,281,588,295,1332,196,256,1570,204,2305,195,225,140,1250 +org.apache.lucene.search.spans.TestSpans=675,845,280,1580,215,265,1097,1638,1060,507,261,664,662,541,473,422,351,494 +org.apache.lucene.search.spans.TestSpansAdvanced=65,73,63,64,89,53,82,156,52,41,395,254,64,57,81,58,86,293 +org.apache.lucene.search.spans.TestSpansAdvanced2=200,106,256,297,215,165,114,651,154,98,115,179,262,170,142,464,157,136 +org.apache.lucene.search.spell.TestDirectSpellChecker=415,612,844,719,779,1233,672,562,440,1017,922,1285,723,893,1010,782,878,1571 +org.apache.lucene.search.spell.TestJaroWinklerDistance=144,112,162,68,46,64,62,57,86,43,50,59,69,86,59,56,51,45 +org.apache.lucene.search.spell.TestLevenshteinDistance=49,52,84,207,208,51,52,78,334,68,51,183,395,240,50,78,52,43 +org.apache.lucene.search.spell.TestLuceneDictionary=260,1351,587,660,295,727,509,709,1582,735,327,514,505,353,416,286,428,275 +org.apache.lucene.search.spell.TestNGramDistance=69,190,152,86,254,93,71,161,62,79,69,79,87,91,87,117,61,70 +org.apache.lucene.search.spell.TestPlainTextDictionary=403,254,141,120,113,104,918,121,89,163,404,447,395,421,459,219,92,100 +org.apache.lucene.search.spell.TestSpellChecker=5196,3063,5928,4004,3643,3795,4082,3245,4117,4065,3880,4755,4298,4336,4240,3545,4518,4204 +org.apache.lucene.search.spell.TestWordBreakSpellChecker=510,907,222,646,576,509,653,726,594,312,412,845,676,395,528,857,1120,925 +org.apache.lucene.search.suggest.LookupBenchmarkTest=33,34,41,32,23,90,33,27,31,31,32,24,46,27,35,31,40,44 +org.apache.lucene.search.suggest.PersistenceTest=335,118,177,203,208,85,307,338,168,366,70,101,155,152,89,390,84,328 +org.apache.lucene.search.suggest.TestBytesRefList=98,201,96,354,170,140,128,400,245,220,129,186,258,136,303,186,207,120 +org.apache.lucene.search.suggest.TestHighFrequencyDictionary=128,375,277,224,81,319,169,144,34,117,312,151,452,183,112,211,119,152 +org.apache.lucene.search.suggest.TestTermFreqIterator=810,1330,649,966,1280,936,854,613,931,1220,892,944,1263,912,569,1053,1588,804 +org.apache.lucene.search.suggest.fst.BytesRefSortersTest=240,176,256,267,262,270,79,110,99,76,83,75,440,102,273,76,283,109 +org.apache.lucene.search.suggest.fst.FSTCompletionTest=2786,1696,2842,2112,3551,2452,2288,3261,3085,1864,2502,2157,2362,2441,3632,1987,2172,3079 +org.apache.lucene.search.suggest.fst.TestSort=6711,6323,6170,5952,6742,6394,5155,6658,6399,6520,5761,6643,5785,6764,7735,6364,7463,6454 +org.apache.lucene.search.suggest.fst.WFSTCompletionTest=1153,876,837,994,1871,1345,830,854,1720,840,1302,671,903,1103,1897,1478,1579,641 +org.apache.lucene.search.vectorhighlight.BreakIteratorBoundaryScannerTest=312,128,153,254,92,61,261,70,87,71,70,222,84,79,120,80,120,126 +org.apache.lucene.search.vectorhighlight.FieldPhraseListTest=686,368,265,460,348,540,506,374,320,414,449,183,334,235,302,307,393,602 +org.apache.lucene.search.vectorhighlight.FieldQueryTest=469,366,334,333,198,645,638,308,1233,238,394,370,561,356,403,381,305,472 +org.apache.lucene.search.vectorhighlight.FieldTermStackTest=316,240,667,496,743,486,832,606,177,486,332,441,295,433,327,442,251,326 +org.apache.lucene.search.vectorhighlight.IndexTimeSynonymTest=585,502,568,587,351,385,495,462,599,417,851,675,563,652,365,428,753,412 +org.apache.lucene.search.vectorhighlight.ScoreOrderFragmentsBuilderTest=62,108,69,95,85,495,78,81,95,108,127,93,113,140,139,87,76,53 +org.apache.lucene.search.vectorhighlight.SimpleBoundaryScannerTest=250,45,63,293,268,37,75,210,251,54,41,214,235,59,63,62,70,87 +org.apache.lucene.search.vectorhighlight.SimpleFragListBuilderTest=726,444,611,554,446,378,504,462,796,478,703,502,543,412,479,305,707,410 +org.apache.lucene.search.vectorhighlight.SimpleFragmentsBuilderTest=219,719,660,285,531,283,292,294,390,622,277,295,187,278,662,353,433,652 +org.apache.lucene.search.vectorhighlight.SingleFragListBuilderTest=129,188,187,125,161,161,129,324,137,101,503,404,184,532,180,484,108,108 +org.apache.lucene.search.vectorhighlight.WeightedFragListBuilderTest=119,493,87,516,387,77,62,869,94,70,103,66,185,69,61,67,106,438 +org.apache.lucene.spatial.TestTestFramework=70,272,189,53,222,203,221,266,202,258,196,29,236,238,211,37,249,269 +org.apache.lucene.spatial.prefix.TestRecursivePrefixTreeStrategy=3329,4032,3419,3023,3959,4499,3417,4240,3252,3536,3102,3460,3406,4153,4557,4768,4621,4646 +org.apache.lucene.spatial.prefix.TestSpatialPrefixField=36,285,194,52,200,213,199,280,201,233,195,81,202,289,209,35,249,225 +org.apache.lucene.spatial.prefix.TestTermQueryPrefixGridStrategy=381,94,60,568,93,81,60,98,77,94,262,506,127,82,82,422,84,108 +org.apache.lucene.spatial.prefix.tree.SpatialPrefixTreeTest=239,52,50,201,72,41,59,56,58,50,75,243,50,58,75,285,66,59 +org.apache.lucene.spatial.vector.TestTwoDoublesStrategy=3717,3129,2399,2794,3135,3154,2813,3657,2500,3682,3621,3363,3430,2567,3481,2450,3663,3450 +org.apache.lucene.store.TestBufferedIndexInput=1663,1777,1692,2301,2365,1896,2247,3707,1837,2544,1663,1823,2109,2229,1991,2361,1768,2452 +org.apache.lucene.store.TestByteArrayDataInput=29,52,205,56,65,39,49,23,57,40,48,38,29,33,56,29,56,31 +org.apache.lucene.store.TestCopyBytes=265,380,414,338,340,339,428,647,315,399,291,232,398,468,730,1265,514,323 +org.apache.lucene.store.TestDirectory=228,262,67,63,81,82,40,71,49,39,106,40,109,45,64,90,49,51 +org.apache.lucene.store.TestFileSwitchDirectory=68,256,86,90,204,81,50,59,72,72,89,56,62,64,128,137,121,55 +org.apache.lucene.store.TestHugeRamFile=995,1663,912,1200,1065,2950,1783,1774,1303,1451,927,1680,1256,1115,994,1008,1118,1731 +org.apache.lucene.store.TestLock=48,55,56,40,74,48,40,47,182,92,48,61,40,307,337,59,39,64 +org.apache.lucene.store.TestLockFactory=1147,1092,1162,1221,1236,1068,1068,1063,1301,1083,1046,1144,1122,1057,1224,1169,1126,1096 +org.apache.lucene.store.TestMultiMMap=3008,3978,3092,4050,3463,2670,3434,4938,2482,3790,3663,4215,3623,3829,5545,2927,3505,2926 +org.apache.lucene.store.TestNRTCachingDirectory=383,1634,2532,2043,5717,1266,1749,704,944,2618,824,979,2840,1963,3365,4048,776,5744 +org.apache.lucene.store.TestRAMDirectory=293,555,965,1205,762,1286,1712,1291,273,1451,567,1974,1174,225,440,407,263,1431 +org.apache.lucene.store.TestWindowsMMap=932,885,687,677,333,702,495,827,729,568,584,893,650,700,415,714,648,1202 +org.apache.lucene.util.TestArrayUtil=2266,1750,2561,1980,1279,2208,3548,6830,2539,3088,1613,2130,1366,1529,1889,2457,3201,3279 +org.apache.lucene.util.TestAttributeSource=72,65,56,31,202,63,39,62,57,80,48,56,72,42,30,81,63,72 +org.apache.lucene.util.TestBitUtil=206,55,37,50,25,71,32,39,38,47,31,64,47,30,67,97,181,31 +org.apache.lucene.util.TestByteBlockPool=39,48,39,32,45,55,39,38,45,40,39,198,33,153,79,67,53,56 +org.apache.lucene.util.TestBytesRef=40,69,62,40,127,64,32,55,36,40,56,64,56,95,81,56,48,54 +org.apache.lucene.util.TestBytesRefHash=448,1133,806,639,1280,1314,604,466,2740,888,998,466,739,431,515,899,453,421 +org.apache.lucene.util.TestCharsRef=53,281,81,47,105,81,44,196,113,56,65,63,52,284,64,81,52,47 +org.apache.lucene.util.TestCloseableThreadLocal=47,40,179,57,46,39,33,168,39,32,105,48,56,31,40,31,64,31 +org.apache.lucene.util.TestCollectionUtil=1858,1574,2051,1900,1408,1259,2799,2492,1424,1600,4968,2166,1648,1718,2498,2292,2436,1823 +org.apache.lucene.util.TestDoubleBarrelLRUCache=1052,1073,1039,1055,1052,1055,1058,1035,1043,1346,1041,1063,1067,1093,1097,1046,1063,1029 +org.apache.lucene.util.TestFieldCacheSanityChecker=317,5164,515,515,2963,456,397,297,882,531,773,648,475,415,1358,276,421,894 +org.apache.lucene.util.TestFixedBitSet=1600,1576,2391,1503,1700,1203,1324,3405,1927,1780,2218,1964,1731,1490,1101,1797,1920,1635 +org.apache.lucene.util.TestIOUtils=60,136,22,29,39,39,30,38,90,43,52,47,39,32,39,56,31,31 +org.apache.lucene.util.TestIdentityHashSet=214,98,164,147,653,117,127,108,495,221,142,198,131,95,231,240,116,348 +org.apache.lucene.util.TestIntsRef=56,38,324,62,225,29,65,48,31,45,56,40,31,56,33,349,172,32 +org.apache.lucene.util.TestNamedSPILoader=46,65,64,31,52,171,53,61,95,46,52,31,31,49,31,40,43,30 +org.apache.lucene.util.TestNumericUtils=664,939,913,853,931,912,989,972,773,814,728,1072,656,791,866,740,1190,1133 +org.apache.lucene.util.TestOpenBitSet=3785,3143,3103,2980,4094,3289,2226,2789,3334,2233,3670,3659,4834,2465,3888,2849,3577,1902 +org.apache.lucene.util.TestPagedBytes=2540,840,1140,2589,4246,5485,1177,3268,4551,4684,2185,1340,1892,1506,1484,1186,1890,1707 +org.apache.lucene.util.TestPriorityQueue=113,64,55,54,50,48,64,81,54,47,97,56,48,146,73,74,48,56 +org.apache.lucene.util.TestRamUsageEstimator=32,68,204,36,39,55,39,54,61,64,31,55,34,36,40,55,39,56 +org.apache.lucene.util.TestRamUsageEstimatorOnWildAnimals=876,800,1344,741,557,517,1667,582,1040,524,923,525,532,713,614,541,893,516 +org.apache.lucene.util.TestRecyclingByteBlockAllocator=55,63,34,51,57,88,96,67,95,72,206,93,331,59,58,88,56,80 +org.apache.lucene.util.TestRollingBuffer=98,104,322,438,156,180,363,365,217,106,112,103,256,176,155,107,98,256 +org.apache.lucene.util.TestRollingCharBuffer=1684,896,1090,1374,1122,1595,1320,1273,2616,1133,1237,969,2160,1740,1707,1015,2456,1465 +org.apache.lucene.util.TestSentinelIntSet=262,155,158,282,273,161,164,120,569,423,148,256,734,249,281,199,173,1315 +org.apache.lucene.util.TestSetOnce=72,64,98,105,106,214,86,55,56,156,157,64,66,132,127,181,55,98 +org.apache.lucene.util.TestSmallFloat=370,106,64,273,73,63,290,69,81,354,93,213,81,123,248,156,115,106 +org.apache.lucene.util.TestSortedVIntList=60,261,58,63,55,163,96,84,168,64,63,77,226,46,72,105,121,179 +org.apache.lucene.util.TestTwoPhaseCommitTool=56,47,31,73,56,86,51,125,63,80,47,56,32,58,29,90,81,31 +org.apache.lucene.util.TestUnicodeUtil=269,224,341,281,235,275,306,256,239,289,414,284,289,535,297,364,356,505 +org.apache.lucene.util.TestVersion=41,62,40,40,47,27,40,47,48,46,48,74,32,69,38,47,59,48 +org.apache.lucene.util.TestVersionComparator=31,75,52,33,25,65,181,23,56,46,31,57,30,55,31,176,46,32 +org.apache.lucene.util.TestVirtualMethod=47,55,56,23,64,234,31,63,125,359,31,62,47,73,57,31,47,39 +org.apache.lucene.util.TestWeakIdentityMap=2057,1204,2643,1548,2049,1999,5951,1390,1398,1128,1441,1385,1708,1766,1718,1149,3778,1426 +org.apache.lucene.util.UnsafeByteArrayInputStreamTest=233,92,70,43,47,105,43,55,64,63,112,79,291,56,60,75,80,88 +org.apache.lucene.util.UnsafeByteArrayOutputStreamTest=83,95,94,63,57,93,94,180,37,81,111,56,104,88,77,74,61,55 +org.apache.lucene.util.Vint8Test=86,87,86,116,88,47,146,64,88,95,79,69,71,88,78,36,88,61 +org.apache.lucene.util.automaton.TestBasicOperations=572,165,388,232,554,239,458,131,131,117,173,121,151,123,331,190,499,473 +org.apache.lucene.util.automaton.TestCompiledAutomaton=250,281,223,390,315,206,226,431,606,257,65,515,151,690,110,239,267,95 +org.apache.lucene.util.automaton.TestDeterminism=215,291,870,816,272,291,924,3140,1090,816,1264,491,1034,1112,948,769,434,1556 +org.apache.lucene.util.automaton.TestDeterminizeLexicon=2282,931,1893,535,542,1606,973,1260,485,818,564,1498,2425,3047,1882,1781,1815,595 +org.apache.lucene.util.automaton.TestLevenshteinAutomata=1107,1458,3474,610,759,640,766,948,651,690,714,889,2478,2013,1690,859,2191,570 +org.apache.lucene.util.automaton.TestMinimize=2553,8637,2299,2804,1395,2021,2194,2399,2504,2793,3746,3974,2955,2854,4211,3650,2300,3065 +org.apache.lucene.util.automaton.TestSpecialOperations=107,836,124,117,79,107,108,106,89,340,214,324,98,258,248,215,109,306 +org.apache.lucene.util.automaton.TestUTF32ToUTF8=1589,1235,782,2624,1396,802,1046,684,668,1140,723,906,669,763,1506,767,3098,726 +org.apache.lucene.util.collections.ArrayHashMapTest=146,88,272,294,62,102,101,151,172,111,122,80,160,169,118,163,101,112 +org.apache.lucene.util.collections.FloatToObjectMapTest=216,75,104,302,216,76,130,105,63,72,72,114,96,172,219,74,210,55 +org.apache.lucene.util.collections.IntArrayTest=38,87,61,87,38,37,93,56,70,46,68,46,47,55,134,80,39,124 +org.apache.lucene.util.collections.IntHashSetTest=184,182,200,218,133,192,121,185,146,96,133,326,286,328,113,139,97,105 +org.apache.lucene.util.collections.IntToDoubleMapTest=119,96,105,113,76,61,201,54,64,63,124,130,176,152,83,89,163,76 +org.apache.lucene.util.collections.IntToIntMapTest=209,218,97,114,72,144,145,66,174,179,71,74,122,80,72,105,64,84 +org.apache.lucene.util.collections.IntToObjectMapTest=104,89,113,155,68,102,312,288,171,240,272,218,270,79,120,112,156,121 +org.apache.lucene.util.collections.ObjectToFloatMapTest=115,63,102,155,105,87,96,104,204,78,156,149,102,330,110,148,68,64 +org.apache.lucene.util.collections.ObjectToIntMapTest=120,53,98,90,68,128,62,181,178,71,88,80,128,92,124,80,54,81 +org.apache.lucene.util.collections.TestLRUHashMap=41,38,65,40,32,209,39,44,77,77,35,31,74,55,75,256,29,28 +org.apache.lucene.util.encoding.EncodingTest=291,237,369,357,313,309,236,343,236,253,250,371,302,372,475,220,244,282 +org.apache.lucene.util.fst.TestFSTs=15370,10584,14699,9921,7974,6679,10243,12694,12394,7107,10337,14788,16280,16884,10418,8874,13186,7153 +org.apache.lucene.util.junitcompat.TestBeforeAfterOverrides=91,82,149,123,83,91,84,74,103,77,104,99,174,66,116,99,95,99 +org.apache.lucene.util.junitcompat.TestCodecReported=60,48,65,78,61,255,76,39,61,89,66,65,66,48,107,77,60,33 +org.apache.lucene.util.junitcompat.TestExceptionInBeforeClassHooks=409,149,227,299,199,256,250,160,248,209,133,217,221,124,295,158,261,149 +org.apache.lucene.util.junitcompat.TestJUnitRuleOrder=86,32,41,44,45,245,99,53,47,48,349,49,66,128,56,86,187,70 +org.apache.lucene.util.junitcompat.TestReproduceMessage=487,274,275,293,434,648,563,476,276,400,256,381,457,510,414,624,260,266 +org.apache.lucene.util.junitcompat.TestSameRandomnessLocalePassedOrNot=92,104,45,90,49,74,88,69,58,49,91,101,87,40,136,65,101,74 +org.apache.lucene.util.junitcompat.TestSeedFromUncaught=91,51,72,91,66,57,76,60,61,65,121,83,93,60,99,158,74,121 +org.apache.lucene.util.junitcompat.TestSetupTeardownChaining=73,104,94,74,74,150,134,176,86,91,104,323,85,87,90,58,174,78 +org.apache.lucene.util.junitcompat.TestSystemPropertiesInvariantRule=150,318,153,166,174,124,116,157,255,141,166,259,150,157,186,124,176,175 +org.apache.lucene.util.packed.TestPackedInts=13025,14248,13139,16450,13648,25217,15462,11274,17179,15497,13267,19049,13573,12274,20538,12830,14621,17156 +org.apache.solr.BasicFunctionalityTest=2178,2132,2386,3185,2496,2893,3296,1992,2586,2320,2667,3252,2437,2121,2998,2811,2571,1904 +org.apache.solr.ConvertedLegacyTest=3190,2982,4889,3308,3462,2846,3037,2574,6927,3435,2241,2650,2766,3031,2390,6628,3481,2297 +org.apache.solr.DisMaxRequestHandlerTest=1052,1615,1366,1502,1779,1694,1141,1317,2088,1958,1109,1140,1769,1181,2233,2278,1367,1484 +org.apache.solr.EchoParamsTest=218,224,214,241,190,512,295,201,590,183,257,181,218,254,605,220,690,763 +org.apache.solr.MinimalSchemaTest=842,731,1015,1547,830,1106,1646,1927,1507,1796,1426,1429,2071,1123,1510,1189,1365,1838 +org.apache.solr.OutputWriterTest=389,264,410,305,1016,480,476,643,422,379,423,312,281,636,531,383,412,283 +org.apache.solr.SampleTest=1809,2710,2326,2523,2664,1990,1455,2362,4081,2280,1985,2064,3209,1500,2627,1600,2422,2520 +org.apache.solr.SolrInfoMBeanTest=1045,1600,1395,1168,1880,1328,915,857,1826,1121,1578,1790,1657,1083,1702,1439,1506,940 +org.apache.solr.TestDistributedGrouping=15350,17874,14119,24952,17999,18188,16127,24858,21026,16509,18723,12726,19154,19500,18176,20077,17431,18114 +org.apache.solr.TestDistributedSearch=15105,14932,16537,15159,22035,12915,16683,15596,14759,19820,15258,13088,13675,15355,19230,13944,17473,13735 +org.apache.solr.TestGroupingSearch=3222,3811,3825,4295,7585,3192,3429,3099,3172,4803,2546,4163,3175,2994,4462,4071,4296,3741 +org.apache.solr.TestJoin=6561,5615,7336,8767,8553,7255,6018,8765,6276,13057,6970,5724,7066,14003,5617,9158,6590,6415 +org.apache.solr.TestPluginEnable=130,155,197,189,110,379,131,96,131,317,106,116,250,182,157,146,322,131 +org.apache.solr.TestRandomFaceting=7514,27142,19366,16020,22394,17368,22152,11660,13387,23532,10689,15407,20399,9268,15628,15383,17217,25492 +org.apache.solr.TestSolrCoreProperties=254,567,342,484,295,314,566,256,242,328,268,283,449,282,218,325,216,220 +org.apache.solr.TestTrie=1645,2058,2307,1633,1982,2635,1229,1407,1819,1543,1603,2270,1337,1941,2247,1284,2261,1624 +org.apache.solr.analysis.CommonGramsFilterFactoryTest=45,81,63,37,37,90,64,64,59,222,280,154,102,46,47,43,54,55 +org.apache.solr.analysis.CommonGramsQueryFilterFactoryTest=130,46,33,86,355,37,30,55,272,55,36,38,330,88,61,36,56,69 +org.apache.solr.analysis.DoubleMetaphoneFilterFactoryTest=47,109,69,40,80,54,52,145,46,105,35,80,60,79,35,121,329,211 +org.apache.solr.analysis.LegacyHTMLStripCharFilterTest=353,120,149,229,153,154,146,167,175,201,400,100,276,134,324,116,147,121 +org.apache.solr.analysis.LengthFilterTest=31,38,297,64,41,48,48,58,64,61,48,57,67,47,72,65,46,38 +org.apache.solr.analysis.SnowballPorterFilterFactoryTest=56,55,80,48,47,42,55,54,48,82,254,26,65,39,132,172,40,186 +org.apache.solr.analysis.TestArabicFilters=63,338,43,48,63,41,45,154,54,54,154,29,37,71,95,111,37,29 +org.apache.solr.analysis.TestBeiderMorseFilterFactory=320,264,548,635,561,261,346,413,731,498,337,280,289,459,266,281,333,332 +org.apache.solr.analysis.TestBrazilianStemFilterFactory=92,164,60,23,39,55,60,64,63,26,40,79,59,80,39,54,38,72 +org.apache.solr.analysis.TestBulgarianStemFilterFactory=40,30,282,30,46,196,136,53,63,189,29,29,31,37,55,51,72,50 +org.apache.solr.analysis.TestCJKBigramFilterFactory=102,31,55,334,231,38,46,35,79,54,30,41,54,38,58,40,155,441 +org.apache.solr.analysis.TestCJKWidthFilterFactory=38,73,30,52,72,77,51,56,56,44,288,45,90,39,94,48,321,69 +org.apache.solr.analysis.TestCapitalizationFilterFactory=60,346,86,57,155,87,97,52,43,336,75,37,103,53,79,46,53,37 +org.apache.solr.analysis.TestCzechStemFilterFactory=48,82,197,239,47,116,180,39,30,73,239,39,73,48,210,127,37,65 +org.apache.solr.analysis.TestDelimitedPayloadTokenFilterFactory=340,63,62,46,70,144,154,39,381,72,80,55,47,164,39,33,38,147 +org.apache.solr.analysis.TestDictionaryCompoundWordTokenFilterFactory=54,43,60,36,37,57,136,35,82,37,70,29,43,37,54,314,574,79 +org.apache.solr.analysis.TestElisionFilterFactory=46,38,308,38,55,54,54,67,69,60,155,262,156,67,77,63,44,38 +org.apache.solr.analysis.TestEnglishMinimalStemFilterFactory=440,41,64,63,81,281,55,312,280,28,30,46,37,37,29,51,481,47 +org.apache.solr.analysis.TestFinnishLightStemFilterFactory=219,212,79,31,44,40,45,42,81,31,74,39,38,94,30,39,340,40 +org.apache.solr.analysis.TestFoldingMultitermExtrasQuery=1679,1758,1573,2029,1998,1444,1603,1870,1748,2241,1666,1463,1732,1948,2100,1306,1432,1534 +org.apache.solr.analysis.TestFrenchLightStemFilterFactory=180,43,119,43,39,165,28,55,29,38,59,81,46,49,63,30,166,68 +org.apache.solr.analysis.TestFrenchMinimalStemFilterFactory=80,60,64,53,47,155,131,40,439,25,155,30,101,448,41,41,63,33 +org.apache.solr.analysis.TestGalicianMinimalStemFilterFactory=72,47,61,40,64,84,38,362,71,55,359,30,64,180,63,55,173,439 +org.apache.solr.analysis.TestGalicianStemFilterFactory=76,47,59,80,35,62,46,59,107,178,38,39,215,82,208,106,62,82 +org.apache.solr.analysis.TestGermanLightStemFilterFactory=55,55,261,322,30,80,30,54,100,112,157,53,31,81,283,56,38,67 +org.apache.solr.analysis.TestGermanMinimalStemFilterFactory=96,32,40,30,258,72,227,60,49,36,53,30,273,29,33,31,89,63 +org.apache.solr.analysis.TestGermanNormalizationFilterFactory=43,65,55,30,297,163,57,178,317,404,35,27,39,56,38,44,30,36 +org.apache.solr.analysis.TestGermanStemFilterFactory=330,50,31,38,71,48,64,43,55,32,92,29,45,55,271,356,30,55 +org.apache.solr.analysis.TestGreekLowerCaseFilterFactory=138,135,80,146,180,55,36,239,278,56,88,62,29,45,525,25,41,63 +org.apache.solr.analysis.TestGreekStemFilterFactory=63,37,56,61,39,46,46,56,51,56,66,284,54,92,56,155,35,372 +org.apache.solr.analysis.TestHTMLStripCharFilterFactory=193,63,88,114,63,394,163,104,71,134,255,104,238,71,140,239,39,351 +org.apache.solr.analysis.TestHindiFilters=71,38,55,55,47,47,51,72,78,81,153,31,82,55,62,245,348,61 +org.apache.solr.analysis.TestHungarianLightStemFilterFactory=61,72,33,32,39,30,64,131,373,47,48,53,57,85,55,475,376,65 +org.apache.solr.analysis.TestHunspellStemFilterFactory=46,146,325,28,55,54,45,56,79,33,53,37,63,54,117,69,38,68 +org.apache.solr.analysis.TestHyphenationCompoundWordTokenFilterFactory=124,142,181,276,123,318,123,116,147,120,123,264,163,522,195,155,126,104 +org.apache.solr.analysis.TestICUFoldingFilterFactory=151,49,300,112,290,298,151,143,272,110,47,56,270,101,66,102,45,321 +org.apache.solr.analysis.TestICUNormalizer2FilterFactory=117,64,69,254,358,90,67,308,79,320,77,75,83,80,251,48,78,92 +org.apache.solr.analysis.TestICUTokenizerFactory=659,200,542,352,208,517,413,255,217,298,507,553,203,247,256,576,440,493 +org.apache.solr.analysis.TestICUTransformFilterFactory=1106,590,639,794,619,556,615,634,696,993,1069,834,593,600,822,840,664,633 +org.apache.solr.analysis.TestIndonesianStemFilterFactory=64,47,257,139,30,63,38,31,63,181,67,55,64,47,47,43,38,63 +org.apache.solr.analysis.TestIrishLowerCaseFilterFactory=87,30,57,47,91,45,54,64,130,63,230,30,36,30,55,30,65,164 +org.apache.solr.analysis.TestItalianLightStemFilterFactory=30,397,63,31,40,55,33,47,39,64,69,41,48,145,64,44,215,61 +org.apache.solr.analysis.TestJapaneseBaseFormFilterFactory=341,465,252,120,453,347,560,59,264,255,615,233,280,385,233,244,87,42 +org.apache.solr.analysis.TestJapaneseKatakanaStemFilterFactory=412,474,264,315,302,557,237,406,222,214,495,66,219,510,330,121,266,229 +org.apache.solr.analysis.TestJapanesePartOfSpeechStopFilterFactory=48,31,258,480,30,39,255,30,632,338,64,46,39,63,287,49,238,773 +org.apache.solr.analysis.TestJapaneseReadingFormFilterFactory=332,146,156,69,338,39,63,226,40,324,30,430,30,522,61,64,223,63 +org.apache.solr.analysis.TestJapaneseTokenizerFactory=421,258,54,231,80,266,167,50,38,40,992,38,281,438,57,573,63,259 +org.apache.solr.analysis.TestKStemFilterFactory=147,440,190,246,495,157,418,136,175,180,272,130,265,229,197,382,416,122 +org.apache.solr.analysis.TestKeepFilterFactory=55,49,30,62,45,47,63,30,40,64,64,48,72,69,31,31,72,65 +org.apache.solr.analysis.TestKeywordMarkerFilterFactory=65,78,123,41,63,54,42,38,63,80,46,31,30,38,106,57,47,114 +org.apache.solr.analysis.TestLatvianStemFilterFactory=37,38,38,47,55,53,35,30,30,54,55,39,34,31,55,73,38,72 +org.apache.solr.analysis.TestLuceneMatchVersion=1463,535,2180,862,1725,1622,633,586,596,1307,744,646,1030,1614,718,650,2438,790 +org.apache.solr.analysis.TestMappingCharFilterFactory=31,42,198,30,63,57,48,227,351,53,297,73,229,81,95,48,58,30 +org.apache.solr.analysis.TestMorfologikFilterFactory=296,129,144,88,94,80,261,117,76,89,102,76,76,117,127,109,257,108 +org.apache.solr.analysis.TestMultiWordSynonyms=338,142,84,63,37,79,31,43,46,303,91,39,289,80,46,37,54,164 +org.apache.solr.analysis.TestNGramFilters=138,59,53,71,43,52,153,80,346,53,74,311,78,119,313,472,92,53 +org.apache.solr.analysis.TestNorwegianLightStemFilterFactory=398,267,47,41,41,46,126,64,55,47,64,55,39,48,64,56,98,55 +org.apache.solr.analysis.TestNorwegianMinimalStemFilterFactory=26,46,43,46,31,37,211,39,65,61,48,305,54,49,180,47,242,46 +org.apache.solr.analysis.TestPatternReplaceCharFilterFactory=38,30,127,42,84,37,46,53,65,47,56,68,47,38,88,56,221,155 +org.apache.solr.analysis.TestPatternReplaceFilterFactory=38,47,263,30,109,42,30,151,30,58,37,39,48,37,29,72,48,47 +org.apache.solr.analysis.TestPatternTokenizerFactory=49,113,34,142,30,336,57,31,46,73,30,58,73,56,63,39,71,30 +org.apache.solr.analysis.TestPersianNormalizationFilterFactory=67,30,322,90,39,154,45,64,27,71,334,37,48,60,37,84,171,68 +org.apache.solr.analysis.TestPhoneticFilterFactory=12850,9095,9779,9497,10255,9767,9048,17129,10086,14364,8453,9241,9021,9291,9919,8574,10031,8509 +org.apache.solr.analysis.TestPorterStemFilterFactory=49,43,56,48,42,53,42,55,80,67,30,281,71,411,55,38,62,56 +org.apache.solr.analysis.TestPortugueseLightStemFilterFactory=31,44,39,254,115,48,30,131,213,55,52,58,30,52,73,385,30,46 +org.apache.solr.analysis.TestPortugueseMinimalStemFilterFactory=38,55,165,349,30,205,89,31,65,71,60,37,288,72,263,35,108,239 +org.apache.solr.analysis.TestPortugueseStemFilterFactory=29,64,81,63,72,72,38,56,156,56,43,69,55,73,72,32,72,89 +org.apache.solr.analysis.TestRemoveDuplicatesTokenFilterFactory=46,55,46,304,38,183,56,69,30,54,37,30,67,51,69,25,25,46 +org.apache.solr.analysis.TestReverseStringFilterFactory=347,44,29,63,67,101,29,70,63,63,182,68,39,72,39,44,239,63 +org.apache.solr.analysis.TestReversedWildcardFilterFactory=889,2853,942,897,1429,1105,1749,1273,1140,1661,1245,775,1960,961,1350,1379,1656,1141 +org.apache.solr.analysis.TestRussianLightStemFilterFactory=31,28,30,29,38,72,29,56,31,36,30,30,60,59,30,70,30,30 +org.apache.solr.analysis.TestShingleFilterFactory=70,78,63,246,88,63,86,45,111,361,79,59,85,96,102,63,96,94 +org.apache.solr.analysis.TestSmartChineseFactories=519,758,846,510,498,852,688,505,709,483,522,624,806,716,559,967,905,718 +org.apache.solr.analysis.TestSpanishLightStemFilterFactory=64,88,76,421,55,221,46,58,26,64,64,74,60,44,55,47,47,54 +org.apache.solr.analysis.TestStandardFactories=82,48,46,118,52,367,87,50,54,68,68,66,62,629,47,74,147,331 +org.apache.solr.analysis.TestStemmerOverrideFilterFactory=71,43,67,276,39,46,46,237,54,30,30,44,39,29,87,64,45,29 +org.apache.solr.analysis.TestStempelPolishStemFilterFactory=210,215,195,381,204,192,297,386,400,463,176,169,375,400,223,398,309,184 +org.apache.solr.analysis.TestStopFilterFactory=29,63,64,43,33,79,38,105,55,122,464,38,64,112,39,66,81,137 +org.apache.solr.analysis.TestSwedishLightStemFilterFactory=30,131,58,33,49,318,30,43,98,198,190,130,58,47,74,237,63,30 +org.apache.solr.analysis.TestSynonymFilterFactory=36,30,46,56,47,68,41,74,30,106,181,46,41,97,49,40,30,92 +org.apache.solr.analysis.TestThaiWordFilterFactory=82,83,63,54,46,309,65,491,73,133,97,198,63,100,366,104,65,239 +org.apache.solr.analysis.TestTrimFilterFactory=39,33,48,29,55,55,60,73,72,64,29,54,163,38,74,72,50,48 +org.apache.solr.analysis.TestTurkishLowerCaseFilterFactory=30,47,56,39,55,39,28,56,47,29,45,61,39,39,248,214,38,55 +org.apache.solr.analysis.TestTypeTokenFilterFactory=71,63,46,45,61,46,38,62,220,48,66,37,65,99,48,162,45,63 +org.apache.solr.analysis.TestUAX29URLEmailTokenizerFactory=62,59,85,89,77,72,68,302,72,63,54,199,63,80,104,66,161,82 +org.apache.solr.analysis.TestWikipediaTokenizerFactory=38,50,63,46,48,58,39,59,56,55,70,47,68,39,46,136,29,76 +org.apache.solr.analysis.TestWordDelimiterFilterFactory=1669,1184,1243,1090,1515,2506,2052,1262,1484,1538,1619,2115,1462,1955,2216,1690,1661,1670 +org.apache.solr.client.solrj.SolrExampleBinaryTest=6846,5536,4778,4980,4822,5025,5023,4774,5656,5167,5252,5011,5705,5055,5214,5021,7935,5005 +org.apache.solr.client.solrj.SolrExampleXMLTest=5436,5850,7790,5661,5342,6706,6261,5429,4879,4876,5985,4894,5166,5093,6100,5237,4970,5597 +org.apache.solr.client.solrj.SolrExceptionTest=30,31,30,48,61,417,30,31,390,44,39,197,31,36,341,43,36,29 +org.apache.solr.client.solrj.SolrQueryTest=47,45,71,60,63,80,185,75,111,61,53,62,46,146,68,110,44,363 +org.apache.solr.client.solrj.TestBatchUpdate=1829,1870,1460,1961,2345,1926,1802,2182,1503,4423,1692,3931,4047,1629,1558,1851,1780,4363 +org.apache.solr.client.solrj.TestLBHttpSolrServer=34841,35283,36867,36407,35319,35334,35727,35388,35161,35630,35314,37133,35054,36825,36665,35321,35728,35438 +org.apache.solr.client.solrj.beans.TestDocumentObjectBinder=431,61,49,63,72,62,61,70,54,53,53,55,59,66,94,103,207,46 +org.apache.solr.client.solrj.embedded.JettyWebappTest=1381,2854,903,2786,1500,992,1133,1049,2999,880,1082,1206,957,1074,1061,2964,838,1362 +org.apache.solr.client.solrj.embedded.LargeVolumeBinaryJettyTest=984,1180,1434,1037,1328,4099,1533,3217,1678,1721,1062,1115,1182,1039,1090,3298,1412,1147 +org.apache.solr.client.solrj.embedded.LargeVolumeEmbeddedTest=1424,1064,897,1324,1366,1214,3377,1037,863,1757,822,1380,992,1154,1513,1344,801,3013 +org.apache.solr.client.solrj.embedded.LargeVolumeJettyTest=3272,1364,1069,1408,1134,1256,1209,1537,1554,1442,1212,1159,1121,1013,3364,1944,1204,1038 +org.apache.solr.client.solrj.embedded.MergeIndexesEmbeddedTest=5696,6190,4437,6882,5668,4651,3639,3698,4222,3900,5743,4010,6061,3930,4356,4158,4192,3625 +org.apache.solr.client.solrj.embedded.MultiCoreEmbeddedTest=576,553,562,516,2085,499,449,509,582,485,2161,594,846,450,570,537,469,489 +org.apache.solr.client.solrj.embedded.MultiCoreExampleJettyTest=1886,1390,2971,1298,1389,1077,1428,1472,1212,1273,1361,1130,977,1650,1920,3322,1323,1286 +org.apache.solr.client.solrj.embedded.SolrExampleEmbeddedTest=4942,4936,5078,4297,4527,4820,7044,6487,4343,4448,4388,4776,4592,4565,4654,4431,4736,4620 +org.apache.solr.client.solrj.embedded.SolrExampleJettyTest=5689,5027,5369,7834,5465,4782,5032,5715,4954,4954,4920,5336,5157,6780,4806,5226,7507,7997 +org.apache.solr.client.solrj.embedded.SolrExampleStreamingBinaryTest=11701,14065,11821,11904,14465,11982,12600,11967,12265,15060,12846,14788,12037,11987,12107,12757,12388,11717 +org.apache.solr.client.solrj.embedded.SolrExampleStreamingTest=12164,11653,12219,12220,12473,11790,12116,14355,12779,12454,11861,12388,12204,14698,11727,11635,13437,11756 +org.apache.solr.client.solrj.embedded.TestEmbeddedSolrServer=1809,514,664,713,581,700,735,680,1061,669,1076,696,932,671,756,770,667,782 +org.apache.solr.client.solrj.embedded.TestSolrProperties=1306,880,827,1214,881,3337,2492,830,883,1079,1491,972,946,1163,1221,912,2635,1005 +org.apache.solr.client.solrj.impl.BasicHttpSolrServerTest=3251,3203,2824,2992,2835,3426,2897,3182,4812,2863,2888,2914,3503,5707,4667,3379,3348,2920 +org.apache.solr.client.solrj.impl.HttpClientUtilTest=33,47,78,255,46,226,34,48,108,48,34,33,42,456,31,33,43,34 +org.apache.solr.client.solrj.request.TestUpdateRequestCodec=76,43,35,37,43,45,38,315,36,39,39,64,50,43,285,68,64,137 +org.apache.solr.client.solrj.response.AnlysisResponseBaseTest=46,51,29,46,45,30,53,64,38,27,29,44,59,99,71,28,299,62 +org.apache.solr.client.solrj.response.DocumentAnalysisResponseTest=62,28,25,33,37,86,29,29,31,46,30,30,28,63,37,29,58,139 +org.apache.solr.client.solrj.response.FacetFieldTest=31,23,29,45,42,37,30,46,31,79,36,103,30,284,45,23,36,47 +org.apache.solr.client.solrj.response.FieldAnalysisResponseTest=28,53,29,69,38,117,22,35,30,29,30,63,30,108,255,50,72,55 +org.apache.solr.client.solrj.response.QueryResponseTest=88,62,38,80,72,54,58,80,62,52,53,53,58,73,185,118,123,51 +org.apache.solr.client.solrj.response.TermsResponseTest=721,982,664,775,648,788,772,697,2660,2593,965,670,1072,1006,628,803,672,705 +org.apache.solr.client.solrj.response.TestSpellCheckResponse=880,823,1066,1025,1042,3498,971,1229,985,1196,3012,1111,2831,1181,818,806,904,1247 +org.apache.solr.client.solrj.util.ClientUtilsTest=28,29,25,28,44,102,25,257,243,39,29,30,42,30,66,29,31,30 +org.apache.solr.cloud.BasicDistributedZkTest=158063,187348,166707,169820,175011,176064,136591,157491,158872,155306,171078,164210,175323,170921,169196,192507,178206,164074 +org.apache.solr.cloud.BasicZkTest=10134,11637,10069,9620,11038,9904,10770,9446,7167,10664,10233,8105,10107,10284,9283,10663,11560,9329 +org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest=23,37,67,71,24,362,53,58,205,129,29,148,40,49,304,23,39,26 +org.apache.solr.cloud.ChaosMonkeySafeLeaderTest=58,31,29,74,34,58,67,38,49,328,58,49,23,37,208,39,68,66 +org.apache.solr.cloud.CloudStateTest=166,39,274,158,63,348,65,63,374,69,39,58,54,65,483,80,85,38 +org.apache.solr.cloud.CloudStateUpdateTest=16015,16138,15156,21394,17073,17035,15831,14966,17541,14700,15208,16665,20844,20075,19790,16846,10203,13519 +org.apache.solr.cloud.FullSolrCloudDistribCmdsTest=49230,49961,52750,49180,46979,47453,53916,47038,52147,45376,47460,51946,48853,46243,51459,52993,53168,45617 +org.apache.solr.cloud.FullSolrCloudTest=100983,102628,92624,92651,94950,97958,96623,96319,95924,102573,92303,93321,101803,96877,96693,76449,97479,98265 +org.apache.solr.cloud.LeaderElectionIntegrationTest=39326,40625,38980,33630,39185,41088,43998,47671,43187,50718,33565,49096,42612,36818,49763,47892,48774,45147 +org.apache.solr.cloud.LeaderElectionTest=28926,45419,31092,41123,43529,30312,37276,30548,24988,36648,28780,35835,37456,34124,35072,35142,27943,23574 +org.apache.solr.cloud.OverseerTest=93124,97404,83171,97215,88779,95633,89009,106302,107801,84474,82868,101908,99123,96326,82965,101190,108840,108284 +org.apache.solr.cloud.RecoveryZkTest=77692,74436,72581,110319,87727,84502,80572,75682,72155,75985,80095,115504,63800,69803,95874,84736,93077,78089 +org.apache.solr.cloud.TestHashPartitioner=8708,8735,6645,7232,11150,7591,7867,6879,7449,22686,11315,9921,7469,8126,10803,10557,8123,7056 +org.apache.solr.cloud.TestMultiCoreConfBootstrap=6128,8570,5936,3619,4564,4063,5859,3090,5085,3402,4667,4027,7410,8073,4036,4229,7718,5471 +org.apache.solr.cloud.ZkControllerTest=19644,20393,40140,20797,30546,20476,22256,24027,21384,26111,29106,25672,32456,45590,16446,43198,16728,26342 +org.apache.solr.cloud.ZkNodePropsTest=64,324,63,52,72,47,50,30,39,50,93,46,42,41,64,63,37,55 +org.apache.solr.cloud.ZkSolrClientTest=8987,9218,8862,7845,9034,8733,8652,8995,10334,9944,9137,10847,8370,9779,10487,10431,8787,8778 +org.apache.solr.common.SolrDocumentTest=118,39,92,24,52,430,38,229,84,39,46,39,54,94,69,36,111,39 +org.apache.solr.common.params.ModifiableSolrParamsTest=36,42,37,44,219,227,91,38,38,243,38,55,61,103,52,37,187,44 +org.apache.solr.common.params.SolrParamTest=189,43,110,35,77,39,36,43,145,54,108,62,233,38,45,43,44,53 +org.apache.solr.common.util.ContentStreamTest=1388,1116,1328,1016,1915,1868,1339,1375,1183,1664,1415,1468,1489,1871,1048,1954,1130,1214 +org.apache.solr.common.util.IteratorChainTest=118,38,94,39,47,71,197,78,139,46,38,53,38,144,54,38,132,47 +org.apache.solr.common.util.NamedListTest=31,118,80,52,31,27,28,61,30,31,50,25,27,30,39,30,85,41 +org.apache.solr.common.util.TestFastInputStream=61,36,83,30,38,29,29,30,37,46,30,175,30,80,44,39,42,45 +org.apache.solr.common.util.TestHash=128,253,332,129,148,186,395,122,163,115,220,96,130,144,97,304,114,130 +org.apache.solr.common.util.TestJavaBinCodec=206,224,567,306,231,415,255,154,264,352,204,526,213,588,264,573,205,198 +org.apache.solr.common.util.TestNamedListCodec=674,535,646,862,453,847,932,512,347,654,431,855,472,594,463,908,996,454 +org.apache.solr.common.util.TestXMLEscaping=131,38,39,46,46,39,160,44,33,38,39,38,39,72,47,48,39,45 +org.apache.solr.core.AlternateDirectoryTest=1099,915,455,510,600,2073,576,551,628,331,595,410,605,1066,807,384,669,835 +org.apache.solr.core.IndexReaderFactoryTest=356,515,433,377,759,450,340,584,382,440,459,426,408,1069,967,722,380,456 +org.apache.solr.core.PluginInfoTest=82,87,87,195,64,196,241,104,97,63,80,56,267,89,72,115,97,97 +org.apache.solr.core.RAMDirectoryFactoryTest=30,31,38,106,56,73,39,47,156,252,29,193,139,82,55,40,64,50 +org.apache.solr.core.RequestHandlersTest=1240,1130,1030,916,1027,1262,1069,1078,1628,1982,1540,1894,1920,859,1939,889,1034,1686 +org.apache.solr.core.ResourceLoaderTest=419,271,38,42,64,115,366,39,49,86,87,72,51,51,64,79,73,139 +org.apache.solr.core.SOLR749Test=511,687,813,364,350,2110,383,516,393,383,514,573,500,617,605,499,409,417 +org.apache.solr.core.SolrCoreCheckLockOnStartupTest=913,757,774,1116,631,710,2463,1343,884,899,760,936,1373,897,1068,942,848,671 +org.apache.solr.core.SolrCoreTest=5620,6564,6151,7424,7177,5779,5939,5699,5893,6640,5869,6182,7301,5712,6982,6362,5513,5043 +org.apache.solr.core.TestArbitraryIndexDir=1042,1192,1147,1262,1842,1394,855,827,1366,1127,1263,2268,1029,1017,1544,1143,1928,1016 +org.apache.solr.core.TestBadConfig=47,72,65,68,47,291,367,31,47,33,83,64,48,91,52,66,199,99 +org.apache.solr.core.TestCodecSupport=146,1121,121,93,523,133,88,172,165,386,102,107,86,328,123,148,141,114 +org.apache.solr.core.TestConfig=178,204,198,331,245,357,210,238,571,306,332,246,218,171,164,413,488,217 +org.apache.solr.core.TestCoreContainer=3382,4314,3672,3242,4557,4229,4689,5459,4487,4833,4677,3255,6350,6606,5111,4475,4962,3767 +org.apache.solr.core.TestJmxIntegration=2845,2445,3427,2637,2366,2389,2359,2193,3593,3096,2060,1579,2768,2334,2253,2674,2029,2834 +org.apache.solr.core.TestJmxMonitoredMap=713,317,276,454,358,323,237,318,255,241,209,248,245,240,273,240,202,504 +org.apache.solr.core.TestMergePolicyConfig=549,489,483,1956,400,487,340,386,454,495,415,639,450,430,561,461,831,769 +org.apache.solr.core.TestPropInject=703,716,715,671,754,1024,1209,919,2326,1541,994,780,700,848,740,735,904,673 +org.apache.solr.core.TestPropInjectDefaults=464,630,402,499,823,383,422,1862,496,473,564,456,531,366,717,596,449,461 +org.apache.solr.core.TestQuerySenderListener=354,581,564,381,398,2046,592,488,377,449,472,400,644,422,827,409,499,409 +org.apache.solr.core.TestQuerySenderNoQuery=493,385,531,385,442,431,399,445,824,606,638,606,498,699,400,440,463,421 +org.apache.solr.core.TestSolrDeletionPolicy1=883,1007,730,2822,737,746,791,820,1078,1045,795,2211,1101,809,779,798,941,1851 +org.apache.solr.core.TestSolrDeletionPolicy2=472,865,446,616,427,551,464,377,365,414,522,442,381,638,515,692,777,740 +org.apache.solr.core.TestSolrXMLSerializer=58,174,290,58,106,712,65,80,71,72,56,63,40,41,97,282,147,72 +org.apache.solr.core.TestXIncludeConfig=417,513,649,358,818,401,564,414,725,484,458,607,619,461,687,380,398,440 +org.apache.solr.handler.BinaryUpdateRequestHandlerTest=1484,1577,939,1202,855,1580,1505,1983,794,958,2290,1055,1793,1683,1220,2586,1062,1083 +org.apache.solr.handler.CSVRequestHandlerTest=1006,914,1345,963,895,861,2394,837,831,2651,1372,802,2101,1124,1111,818,1081,778 +org.apache.solr.handler.DocumentAnalysisRequestHandlerTest=1784,1552,1059,1439,1514,1245,1397,809,1628,1043,1050,1328,1319,1322,990,1264,1328,2829 +org.apache.solr.handler.FieldAnalysisRequestHandlerTest=925,935,1842,1685,1662,1151,1713,1082,1400,911,1567,1027,1122,2038,1358,1021,2014,1087 +org.apache.solr.handler.JsonLoaderTest=1041,1320,1035,829,1257,1000,1548,2931,1180,1534,2524,1337,868,2001,1185,2057,1796,1837 +org.apache.solr.handler.MoreLikeThisHandlerTest=1154,2200,1857,1999,1473,1175,1849,1679,1191,950,1071,1006,1224,1610,1756,1650,944,2070 +org.apache.solr.handler.PingRequestHandlerTest=1815,989,1982,2245,1213,803,910,1678,1414,863,911,944,837,2333,1349,1040,1815,1576 +org.apache.solr.handler.StandardRequestHandlerTest=2006,1986,881,2977,1010,1312,1855,1284,1757,1587,1414,1324,1879,1115,1110,1053,1838,1226 +org.apache.solr.handler.TestCSVLoader=1117,2611,2178,1243,1295,1180,1659,1279,2380,1895,1309,1446,1554,1472,2158,1634,2777,2481 +org.apache.solr.handler.TestReplicationHandler=44312,48253,45742,46244,44003,43344,41645,47807,47490,45499,44596,44714,48037,45409,43868,47371,44319,43678 +org.apache.solr.handler.XmlUpdateRequestHandlerTest=1585,1595,1304,1056,1427,988,929,1371,2659,811,1858,875,2769,1069,1597,1738,1268,965 +org.apache.solr.handler.XsltUpdateRequestHandlerTest=1550,2463,2118,2163,3065,1336,1214,1459,1329,1202,977,1318,2172,1435,1121,1606,1089,985 +org.apache.solr.handler.admin.CoreAdminHandlerTest=2032,2140,1839,2061,1935,1568,1833,2893,2278,2582,3788,3363,1595,1998,1811,1299,2151,2146 +org.apache.solr.handler.admin.LoggingHandlerTest=1421,1003,1022,1463,1181,824,1064,994,1237,1398,1418,1161,1304,928,1371,1473,941,992 +org.apache.solr.handler.admin.LukeRequestHandlerTest=3934,5307,3663,3262,4242,4172,3203,5540,3947,3783,6473,4077,5601,4357,4461,4535,5517,4010 +org.apache.solr.handler.admin.MBeansHandlerTest=1134,1272,874,1017,1813,1963,1759,1451,2568,1144,937,2060,2199,1354,1076,1570,1844,1582 +org.apache.solr.handler.admin.ShowFileRequestHandlerTest=804,717,1077,1125,1133,1030,697,888,826,714,2313,1901,715,983,1376,2783,837,811 +org.apache.solr.handler.admin.SystemInfoHandlerTest=99,65,31,38,165,56,31,31,22,41,61,30,39,225,58,231,65,31 +org.apache.solr.handler.clustering.ClusteringComponentTest=2822,2809,2884,2784,2654,2740,2954,2555,2538,2638,2680,2727,2776,3000,2788,2617,2663,2757 +org.apache.solr.handler.clustering.DistributedClusteringComponentTest=6592,6541,6474,6490,6658,6547,6694,6426,6547,6506,7370,6557,6640,6661,6828,6516,6345,6516 +org.apache.solr.handler.clustering.carrot2.CarrotClusteringEngineTest=3491,3489,3483,3434,3469,3377,3784,3370,3383,3476,3758,3476,3397,3531,3498,3478,3544,3437 +org.apache.solr.handler.component.BadComponentTest=358,431,622,560,398,398,1025,457,607,333,648,451,752,373,474,1216,813,1772 +org.apache.solr.handler.component.DebugComponentTest=1065,1410,1185,1081,1165,1099,1331,1323,1685,1422,1546,1265,1034,1635,1261,947,1490,997 +org.apache.solr.handler.component.DistributedQueryElevationComponentTest=1445,2175,2216,2250,1383,1542,1194,1122,1249,1727,2111,1254,1847,1347,1417,1486,4619,3820 +org.apache.solr.handler.component.DistributedSpellCheckComponentTest=23605,31664,31332,34497,31853,33868,29377,30676,36914,23687,32494,26549,33282,30368,27633,31247,32761,30572 +org.apache.solr.handler.component.DistributedTermsComponentTest=6983,7050,8061,9223,8922,7329,9315,8096,7678,8299,8936,8273,8023,9227,6736,7892,8568,7761 +org.apache.solr.handler.component.QueryElevationComponentTest=2886,4083,5369,3383,5317,2815,2680,3125,2802,3105,2722,2851,2864,2951,2714,2655,3719,2522 +org.apache.solr.handler.component.SearchHandlerTest=1926,1318,1335,1047,1264,712,921,1572,1029,852,1622,807,1961,883,893,1535,844,1239 +org.apache.solr.handler.component.SpellCheckComponentTest=38798,65545,79995,68639,54686,47625,60538,46303,72630,75637,85575,82471,80329,79350,71373,65007,62858,46032 +org.apache.solr.handler.component.StatsComponentTest=7935,7262,9011,8404,9364,6804,9325,7335,7208,6230,8728,6933,7890,9065,7051,6833,6444,7989 +org.apache.solr.handler.component.TermVectorComponentTest=1334,1347,1202,1108,1937,997,1624,1255,2703,917,1416,1360,1395,1443,1226,1181,1136,1734 +org.apache.solr.handler.component.TermsComponentTest=993,1730,965,1594,1276,2028,1809,1241,1896,1484,1433,952,1173,2418,1005,1457,1352,1478 +org.apache.solr.handler.dataimport.TestCachedSqlEntityProcessor=194,220,217,581,241,347,112,404,163,163,162,330,242,454,537,161,172,163 +org.apache.solr.handler.dataimport.TestClobTransformer=63,55,115,55,330,62,59,55,103,111,63,71,62,428,54,61,67,95 +org.apache.solr.handler.dataimport.TestContentStreamDataSource=1739,1730,1606,1648,2690,1646,1712,2030,1678,1647,1938,2162,1716,1699,2797,1614,1659,1914 +org.apache.solr.handler.dataimport.TestContextImpl=167,159,146,439,274,146,145,151,146,130,138,138,151,137,522,456,165,142 +org.apache.solr.handler.dataimport.TestDIHEndToEnd=430,341,491,514,354,401,380,338,637,354,394,486,404,456,355,437,355,404 +org.apache.solr.handler.dataimport.TestDataConfig=248,272,200,338,280,515,230,964,230,205,345,212,205,247,254,223,364,208 +org.apache.solr.handler.dataimport.TestDateFormatTransformer=88,139,63,347,86,80,129,105,280,130,79,86,77,123,85,154,87,100 +org.apache.solr.handler.dataimport.TestDocBuilder=253,375,278,596,280,248,199,243,535,271,388,303,221,714,305,294,289,590 +org.apache.solr.handler.dataimport.TestDocBuilder2=937,824,1931,726,851,1853,720,1308,756,713,774,997,889,853,1032,965,1932,907 +org.apache.solr.handler.dataimport.TestEntityProcessorBase=62,70,59,121,75,78,268,63,73,71,66,63,135,62,63,61,58,54 +org.apache.solr.handler.dataimport.TestEphemeralCache=262,504,255,312,307,198,330,391,204,267,439,278,330,337,204,1013,195,462 +org.apache.solr.handler.dataimport.TestErrorHandling=456,448,772,767,683,730,1367,489,784,2047,1082,466,548,864,545,555,1641,964 +org.apache.solr.handler.dataimport.TestEvaluatorBag=164,425,125,143,356,201,129,136,137,146,143,171,137,137,162,130,130,145 +org.apache.solr.handler.dataimport.TestFieldReader=138,80,71,67,63,149,82,74,70,90,153,70,62,62,73,72,64,504 +org.apache.solr.handler.dataimport.TestFileListEntityProcessor=130,139,147,114,376,114,94,430,105,136,122,120,339,348,335,114,111,331 +org.apache.solr.handler.dataimport.TestJdbcDataSource=146,147,143,201,239,293,145,277,146,134,144,139,554,156,172,138,138,159 +org.apache.solr.handler.dataimport.TestLineEntityProcessor=137,144,87,207,137,148,196,129,135,160,136,301,130,229,163,230,130,203 +org.apache.solr.handler.dataimport.TestMailEntityProcessor=249,238,215,255,254,226,224,230,228,266,226,289,228,319,229,285,292,247 +org.apache.solr.handler.dataimport.TestNumberFormatTransformer=1062,213,264,294,313,300,293,288,281,321,289,679,247,259,277,261,262,270 +org.apache.solr.handler.dataimport.TestPlainTextEntityProcessor=248,72,70,63,158,81,62,71,148,314,80,71,83,123,180,69,72,71 +org.apache.solr.handler.dataimport.TestRegexTransformer=153,154,162,155,580,163,137,161,156,170,156,171,156,156,185,164,204,176 +org.apache.solr.handler.dataimport.TestScriptTransformer=228,186,172,179,187,171,178,364,522,170,171,229,181,239,211,188,205,170 +org.apache.solr.handler.dataimport.TestSolrEntityProcessorEndToEnd=6993,7029,7040,6606,6217,7209,6711,7029,7087,6802,6240,5279,7036,6800,6689,6865,6296,6454 +org.apache.solr.handler.dataimport.TestSolrEntityProcessorUnit=130,113,136,113,145,113,183,113,163,128,111,392,113,126,117,146,254,121 +org.apache.solr.handler.dataimport.TestSortedMapBackedCache=139,105,108,153,115,107,108,117,115,105,105,107,99,357,198,127,101,106 +org.apache.solr.handler.dataimport.TestSqlEntityProcessor=168,144,379,212,136,138,129,398,142,206,137,506,530,138,155,138,138,137 +org.apache.solr.handler.dataimport.TestSqlEntityProcessor2=815,512,530,554,677,572,546,476,748,488,681,1364,690,1490,566,483,1189,477 +org.apache.solr.handler.dataimport.TestSqlEntityProcessorDelta=1213,2423,1834,1461,2034,1197,1193,1207,1551,1476,2018,1184,2241,1135,2206,1336,1208,1610 +org.apache.solr.handler.dataimport.TestSqlEntityProcessorDelta2=862,1472,905,974,905,883,825,875,836,949,1022,795,935,790,831,1005,855,836 +org.apache.solr.handler.dataimport.TestSqlEntityProcessorDelta3=905,672,598,1520,838,801,696,666,723,1894,655,680,956,664,754,654,799,638 +org.apache.solr.handler.dataimport.TestSqlEntityProcessorDeltaPrefixedPk=1523,1125,387,496,430,422,471,373,349,455,504,394,382,472,441,405,494,1966 +org.apache.solr.handler.dataimport.TestTemplateString=171,53,54,73,63,54,60,61,54,55,55,72,93,63,76,55,277,89 +org.apache.solr.handler.dataimport.TestTemplateTransformer=55,53,53,89,55,55,55,54,55,82,54,267,106,338,68,55,55,54 +org.apache.solr.handler.dataimport.TestTikaEntityProcessor=2085,2076,2183,2108,2137,2110,2130,2103,2121,2142,2107,2153,2075,3116,2148,2126,2207,2120 +org.apache.solr.handler.dataimport.TestURLDataSource=53,55,62,314,163,55,54,82,62,62,58,130,114,58,70,63,63,62 +org.apache.solr.handler.dataimport.TestVariableResolver=303,286,260,289,279,262,262,812,321,270,254,388,284,334,671,311,270,261 +org.apache.solr.handler.dataimport.TestXPathEntityProcessor=734,751,712,684,653,638,930,745,737,1154,737,680,746,760,631,751,631,685 +org.apache.solr.handler.dataimport.TestXPathRecordReader=620,838,1111,727,922,1296,871,720,755,596,1139,978,623,795,1104,2148,1164,693 +org.apache.solr.handler.extraction.ExtractingRequestHandlerTest=3834,4047,4010,3800,3862,3700,3888,3848,4243,3942,4013,3679,3811,4064,4065,3842,4029,3757 +org.apache.solr.highlight.FastVectorHighlighterTest=1645,1233,1415,921,1033,1736,1088,1618,2878,1263,1201,1043,2781,909,1408,1453,1287,1211 +org.apache.solr.highlight.HighlighterConfigTest=516,400,579,447,517,598,566,650,605,424,409,400,741,483,2237,475,2167,455 +org.apache.solr.highlight.HighlighterTest=1646,2913,2357,2172,3504,2143,1623,2688,1530,1849,2278,2388,2508,2621,4061,2121,3700,2142 +org.apache.solr.internal.csv.CSVParserTest=102,260,46,77,61,56,53,39,51,201,31,83,52,43,66,99,67,110 +org.apache.solr.internal.csv.CSVPrinterTest=492,501,509,573,624,605,2029,509,533,500,504,536,636,551,587,541,793,482 +org.apache.solr.internal.csv.CSVStrategyTest=66,29,67,54,70,406,67,55,49,58,354,23,32,33,141,22,71,24 +org.apache.solr.internal.csv.CSVUtilsTest=97,87,60,70,32,25,36,56,60,26,32,70,31,52,52,49,55,36 +org.apache.solr.internal.csv.CharBufferTest=37,37,48,23,29,66,121,18,29,57,41,32,23,57,58,82,31,180 +org.apache.solr.internal.csv.ExtendedBufferedReaderTest=24,131,65,49,268,50,26,23,53,66,115,57,33,84,41,75,91,65 +org.apache.solr.internal.csv.writer.CSVConfigGuesserTest=90,39,41,358,32,32,31,44,66,815,24,55,41,33,51,36,54,24 +org.apache.solr.internal.csv.writer.CSVConfigTest=24,36,275,232,324,32,96,24,44,64,15,37,33,51,42,358,32,90 +org.apache.solr.internal.csv.writer.CSVFieldTest=45,76,99,16,58,41,94,250,25,111,24,47,29,41,42,24,25,83 +org.apache.solr.internal.csv.writer.CSVWriterTest=30,69,155,133,37,27,369,41,29,28,27,62,35,57,46,46,55,33 +org.apache.solr.request.JSONWriterTest=918,1213,1324,910,2122,786,1331,2243,1198,2088,1339,1264,848,1124,2173,1213,2030,923 +org.apache.solr.request.SimpleFacetsTest=4343,5863,4175,5416,4183,4137,4500,4567,6300,3905,4230,3520,5516,4769,4878,4820,3689,5050 +org.apache.solr.request.TestBinaryResponseWriter=3052,2294,2625,2244,2790,2910,1628,1633,2464,1550,3137,1796,1629,2118,4576,1802,3123,2574 +org.apache.solr.request.TestFaceting=8928,7825,7676,11315,8566,11331,8195,7573,7511,18737,19713,8453,7808,11747,8939,9112,6748,8503 +org.apache.solr.request.TestRemoteStreaming=1158,2447,2394,2124,2303,1290,1120,1456,2673,1774,1648,975,2411,1764,1717,1639,1486,1325 +org.apache.solr.request.TestWriterPerf=787,908,724,824,706,689,874,950,809,694,725,749,779,1065,668,841,642,746 +org.apache.solr.response.TestCSVResponseWriter=1236,2221,1704,1035,1021,1992,858,884,968,1612,1323,1888,1658,847,3065,879,3070,1860 +org.apache.solr.response.TestPHPSerializedResponseWriter=945,1094,1148,868,1410,896,831,1573,1443,1559,1043,2079,1059,1651,1372,2414,871,1581 +org.apache.solr.schema.BadIndexSchemaTest=906,767,867,1119,1184,828,820,833,778,1272,1080,1035,927,1206,1006,789,1152,748 +org.apache.solr.schema.CopyFieldTest=1670,1213,1107,839,1837,2490,1967,2213,898,1204,959,1077,1468,1097,977,776,1364,2201 +org.apache.solr.schema.CurrencyFieldTest=1740,1401,1532,1529,1188,1995,1071,1377,1190,1717,1781,1097,1716,1191,1797,1575,1205,1247 +org.apache.solr.schema.DateFieldTest=62,71,55,57,87,81,47,54,63,60,89,48,328,81,64,175,70,79 +org.apache.solr.schema.IndexSchemaRuntimeFieldTest=1593,1481,1030,1394,950,2224,1258,1102,2027,1501,1127,860,1321,1824,1975,1044,812,1131 +org.apache.solr.schema.IndexSchemaTest=1057,981,1491,887,1510,1715,885,974,1068,1753,1464,1231,1338,2334,2951,1764,3108,1773 +org.apache.solr.schema.MultiTermTest=296,364,242,305,214,369,399,229,284,256,380,256,297,238,285,315,431,238 +org.apache.solr.schema.NotRequiredUniqueKeyTest=695,693,1144,1443,746,1469,779,1117,942,855,809,1444,586,1165,1143,1342,705,768 +org.apache.solr.schema.NumericFieldsTest=288,965,413,364,273,437,305,420,515,633,342,1186,391,401,255,700,386,701 +org.apache.solr.schema.OpenExchangeRatesOrgProviderTest=54,56,80,83,61,88,71,340,54,246,46,39,72,354,79,88,78,53 +org.apache.solr.schema.PolyFieldTest=2267,1466,1622,1235,1221,1118,2302,1904,1298,1998,1487,1654,1571,1519,2490,2987,2198,1580 +org.apache.solr.schema.PreAnalyzedFieldTest=34,31,182,24,43,36,55,66,54,61,40,57,49,53,32,40,149,57 +org.apache.solr.schema.PrimitiveFieldTypeTest=622,955,564,662,638,854,667,634,659,877,1039,837,659,776,709,638,946,614 +org.apache.solr.schema.RequiredFieldsTest=1672,1040,1107,1881,1597,1197,2422,1480,762,913,1142,2938,2072,1583,1230,864,1074,909 +org.apache.solr.schema.TestBinaryField=553,985,1676,396,370,388,574,339,311,533,486,1123,343,665,675,636,395,325 +org.apache.solr.schema.TestCollationField=448,544,391,298,632,459,784,486,294,351,470,339,489,1045,474,436,547,322 +org.apache.solr.schema.TestICUCollationField=2261,2248,2269,2178,2172,2188,2056,1912,2020,2159,2575,2492,2133,2187,2359,2010,2153,2196 +org.apache.solr.schema.TestICUCollationFieldOptions=1341,1948,2114,1518,1647,2076,1841,1486,1693,1600,1487,1630,1689,2022,1449,1278,1890,1786 +org.apache.solr.schema.TestOmitPositions=837,891,1026,2072,1240,1220,1412,1152,1061,1044,2051,1538,1689,3137,1628,921,1033,961 +org.apache.solr.schema.UUIDFieldTest=67,39,64,74,56,40,30,37,165,23,75,57,135,90,73,38,44,55 +org.apache.solr.search.QueryEqualityTest=2653,1706,2793,2324,1349,1157,1141,1597,1519,1731,2071,1781,1683,1500,1379,2099,1345,2579 +org.apache.solr.search.QueryParsingTest=1016,2541,927,1149,1198,1530,1839,1859,1039,1482,1100,1950,718,957,2103,1798,1888,1462 +org.apache.solr.search.ReturnFieldsTest=947,888,1128,1211,1162,1139,1544,963,1263,1368,1504,2081,1260,1385,1368,866,928,1822 +org.apache.solr.search.SpatialFilterTest=1762,2225,2200,1187,2240,1443,1353,1479,1753,1619,1906,2838,1390,1436,1570,1784,1424,1489 +org.apache.solr.search.TestDocSet=454,816,467,821,458,573,507,571,509,473,469,425,410,620,429,482,464,633 +org.apache.solr.search.TestExtendedDismaxParser=9897,12578,9325,10728,11951,11387,11342,10675,8410,9860,13259,12635,12418,13435,11259,8825,11831,10866 +org.apache.solr.search.TestFastLRUCache=80,328,154,223,129,114,80,81,196,71,131,104,207,102,63,47,155,77 +org.apache.solr.search.TestFiltering=2803,2444,3767,3328,3696,3471,2562,2449,2641,2677,3522,4921,2930,3020,4913,3251,3379,3134 +org.apache.solr.search.TestFoldingMultitermQuery=948,1195,1062,728,904,1331,930,820,1847,777,760,2937,586,1162,645,600,646,865 +org.apache.solr.search.TestIndexSearcher=1495,2586,3113,2125,2177,1761,1763,1992,2085,1647,1665,2660,2328,1532,2128,2340,2617,1872 +org.apache.solr.search.TestLFUCache=530,529,431,463,435,472,471,404,409,562,532,539,420,888,579,475,546,411 +org.apache.solr.search.TestLRUCache=38,89,93,241,44,64,67,39,39,29,62,47,33,106,59,47,123,32 +org.apache.solr.search.TestPseudoReturnFields=1205,1458,1622,2203,1380,1356,3324,2015,1354,1194,1354,1210,1849,1404,1527,1952,1765,1636 +org.apache.solr.search.TestQueryTypes=1462,1999,977,1407,1877,1235,1720,2219,1357,894,1173,2643,2102,1346,1292,1468,982,1258 +org.apache.solr.search.TestQueryUtils=1720,1406,1224,1731,1002,1837,3260,1094,938,973,1131,1189,2182,1594,1078,918,1063,1078 +org.apache.solr.search.TestRangeQuery=6592,6757,8142,5240,6688,4838,5813,6621,5248,5027,4724,6412,6650,4577,4690,5827,4778,5244 +org.apache.solr.search.TestRealTimeGet=28644,24838,24503,31623,28665,29011,34946,33181,27714,39594,33107,29472,33491,34193,34233,29263,29499,25572 +org.apache.solr.search.TestRecovery=6981,7186,5854,7969,6152,9011,6163,7001,5986,6295,5984,15318,9648,8770,17090,8698,7603,6734 +org.apache.solr.search.TestSearchPerf=1581,664,1922,959,966,635,698,921,1849,680,664,1164,938,857,1471,1379,1458,737 +org.apache.solr.search.TestSolrQueryParser=1625,1312,1396,1493,876,758,1039,1020,1730,997,1465,1222,1498,2033,1699,1071,1182,1264 +org.apache.solr.search.TestSort=3388,6508,4693,4247,3210,3955,3916,4553,3540,3792,3547,5120,4199,4588,3817,3958,3682,3004 +org.apache.solr.search.TestSurroundQueryParser=1273,864,1615,1383,894,2702,886,1036,1484,2169,1071,1793,1329,874,1623,1154,1369,1135 +org.apache.solr.search.TestValueSourceCache=1303,1688,1426,1474,1447,1019,1556,1789,1171,1054,1003,2138,894,2206,1191,1208,1032,895 +org.apache.solr.search.function.SortByFunctionTest=4286,4785,2763,2197,2056,2016,2902,2278,2564,2304,2393,3986,2811,3161,4265,2396,3330,3290 +org.apache.solr.search.function.TestFunctionQuery=2029,1483,1506,2221,1823,1869,1789,1828,2127,1510,1543,1468,1640,2121,12474,2406,2230,1615 +org.apache.solr.search.function.distance.DistanceFunctionTest=1541,2404,1109,1015,1104,1019,1556,1271,1434,1456,1569,2064,1663,1142,1026,998,2585,1595 +org.apache.solr.search.similarities.TestBM25SimilarityFactory=121,243,246,405,185,142,114,246,191,118,325,108,138,190,248,139,488,299 +org.apache.solr.search.similarities.TestDFRSimilarityFactory=122,152,996,132,154,106,146,139,238,158,106,214,136,156,114,105,95,156 +org.apache.solr.search.similarities.TestDefaultSimilarityFactory=157,106,139,133,80,121,315,140,196,108,308,111,112,140,162,136,439,159 +org.apache.solr.search.similarities.TestIBSimilarityFactory=194,149,140,90,176,133,105,133,347,171,146,155,130,129,156,148,106,104 +org.apache.solr.search.similarities.TestLMDirichletSimilarityFactory=97,256,173,181,301,180,90,114,174,147,105,255,161,158,171,156,175,176 +org.apache.solr.search.similarities.TestLMJelinekMercerSimilarityFactory=172,457,123,164,131,114,381,148,133,381,131,154,247,210,142,383,113,130 +org.apache.solr.search.similarities.TestPerFieldSimilarity=137,134,127,271,179,289,328,149,106,146,113,171,158,155,230,423,241,131 +org.apache.solr.servlet.CacheHeaderTest=2448,3114,1424,1126,1977,1225,1623,1289,1725,1237,1804,1569,1710,2065,1790,1361,3400,1394 +org.apache.solr.servlet.DirectSolrConnectionTest=1333,1578,1328,1390,2128,1721,2128,1519,2157,1725,2741,1883,1359,1975,1144,2123,1234,1764 +org.apache.solr.servlet.NoCacheHeaderTest=903,645,503,497,1118,553,714,1030,1424,498,863,531,598,676,664,928,983,562 +org.apache.solr.servlet.SolrRequestParserTest=1620,1783,1524,1548,1343,1379,2645,1761,1963,1611,1972,1744,1869,2269,1274,2877,1498,2169 +org.apache.solr.spelling.DirectSolrSpellCheckerTest=2362,2904,4544,3689,3924,3229,4531,1940,3885,3056,2736,2598,2227,3402,3728,3358,4662,3238 +org.apache.solr.spelling.FileBasedSpellCheckerTest=3154,3135,2363,2330,2451,3144,1901,2132,2869,3534,3452,5497,3250,2852,3426,4473,4380,4117 +org.apache.solr.spelling.IndexBasedSpellCheckerTest=4945,7311,5995,4681,5769,3741,5004,4646,6129,6719,7175,6587,5162,7481,7254,5755,7117,5187 +org.apache.solr.spelling.SpellCheckCollatorTest=18996,13214,17662,13501,12552,8530,13829,12629,12289,18114,17711,13238,5068,14202,15478,8395,16481,9340 +org.apache.solr.spelling.SpellPossibilityIteratorTest=205,216,330,309,292,229,205,161,159,214,364,138,199,206,549,557,339,216 +org.apache.solr.spelling.SpellingQueryConverterTest=80,64,106,162,89,68,497,70,58,106,47,69,247,45,64,47,156,54 +org.apache.solr.spelling.TestSuggestSpellingConverter=28,122,155,80,71,47,64,61,51,68,290,170,62,111,79,66,63,44 +org.apache.solr.spelling.WordBreakSolrSpellCheckerTest=4204,8154,10269,9016,5102,6362,10598,7725,8071,8129,8857,11005,7249,12071,11250,8282,8542,8125 +org.apache.solr.spelling.suggest.SuggesterFSTTest=969,946,918,857,1341,1029,1907,1202,2129,806,2354,2751,1847,1939,1297,756,1031,1217 +org.apache.solr.spelling.suggest.SuggesterTSTTest=690,619,1180,908,1900,1727,845,1621,2849,978,3146,4058,797,2269,1039,713,1006,1250 +org.apache.solr.spelling.suggest.SuggesterTest=2142,1071,685,1542,1339,1047,981,1096,832,804,3950,851,1514,672,984,778,740,1798 +org.apache.solr.spelling.suggest.SuggesterWFSTTest=2438,3104,718,1339,1009,980,1194,1114,1392,1161,3671,588,595,867,1830,1377,1481,1643 +org.apache.solr.spelling.suggest.TestPhraseSuggestions=148,148,167,122,297,182,154,333,239,172,132,273,133,308,228,190,147,421 +org.apache.solr.uima.analysis.UIMAAnnotationsTokenizerFactoryTest=9146,8920,9003,8409,8900,8591,8889,7883,9055,9206,8659,8978,8588,8345,9308,9047,8156,8381 +org.apache.solr.uima.analysis.UIMATypeAwareAnnotationsTokenizerFactoryTest=8549,8864,8763,8952,8970,8591,9393,8745,8687,9393,8540,9107,8881,8859,8897,8957,8837,8917 +org.apache.solr.uima.processor.UIMAUpdateRequestProcessorTest=12029,12017,12807,11762,12246,12297,12296,11766,12165,12286,12054,11787,12067,11751,12059,12322,11135,12111 +org.apache.solr.update.AutoCommitTest=11822,11242,10241,11798,10946,10977,10916,9802,10019,11581,10398,9228,11355,11433,10861,11416,9795,11060 +org.apache.solr.update.DirectUpdateHandlerOptimizeTest=944,1051,1221,1306,2117,1445,1140,1326,1495,1703,1603,1004,992,1619,914,1913,2023,1022 +org.apache.solr.update.DirectUpdateHandlerTest=4910,5385,5345,6409,4519,3801,6140,3749,4986,8126,4672,6846,6323,5526,3483,5277,5342,5233 +org.apache.solr.update.DocumentBuilderTest=1902,1710,1796,786,819,1449,1191,1582,1997,920,1090,1469,980,1489,1962,1894,2184,1706 +org.apache.solr.update.PeerSyncTest=2380,3101,2968,3331,2298,4407,4377,4840,3180,4043,2880,2909,2778,2525,2712,2779,2356,3305 +org.apache.solr.update.SoftAutoCommitTest=11066,10876,11329,11611,10384,10270,11216,11905,11344,11706,10422,9906,12370,11575,11571,12756,10199,10313 +org.apache.solr.update.SolrCmdDistributorTest=1320,1043,900,957,1328,1132,2602,1031,1113,2486,1481,970,1148,1363,1698,1488,970,1157 +org.apache.solr.update.SolrIndexConfigTest=1687,587,371,389,373,531,1850,377,469,472,345,369,595,365,550,781,393,458 +org.apache.solr.update.TestIndexingPerformance=1901,907,392,677,455,458,476,493,519,739,489,777,633,633,967,744,716,725 +org.apache.solr.update.TestUpdate=514,472,674,598,731,508,521,463,489,573,430,929,746,555,447,555,725,659 +org.apache.solr.update.UpdateParamsTest=1493,1152,927,938,2510,952,837,1620,903,981,1053,974,2174,962,1671,1013,1443,1661 +org.apache.solr.update.processor.DefaultValueUpdateProcessorTest=421,864,426,420,417,409,554,492,461,489,437,379,678,760,591,534,383,580 +org.apache.solr.update.processor.FieldMutatingUpdateProcessorTest=553,739,589,596,545,446,466,671,508,856,545,612,721,760,599,523,565,552 +org.apache.solr.update.processor.LangDetectLanguageIdentifierUpdateProcessorFactoryTest=1910,2018,2022,1998,1892,2007,1950,1878,2075,1905,1955,1962,2128,1837,1812,2024,1803,1925 +org.apache.solr.update.processor.SignatureUpdateProcessorFactoryTest=5071,1541,1637,1866,2662,2516,4711,1349,2970,1407,2306,2234,3025,2793,4764,3446,1681,1765 +org.apache.solr.update.processor.TikaLanguageIdentifierUpdateProcessorFactoryTest=2075,2223,2111,2089,2045,2044,2120,2206,2302,2039,2205,2218,2235,2168,2069,2213,2075,2071 +org.apache.solr.update.processor.URLClassifyProcessorTest=102,105,97,196,86,209,97,84,113,129,5098,190,72,131,430,100,72,97 +org.apache.solr.update.processor.UniqFieldsUpdateProcessorFactoryTest=883,910,1106,2168,936,1082,1055,1248,2273,1314,976,1129,1119,852,1496,1436,1300,1297 +org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest=711,800,745,1118,697,855,3160,719,934,769,817,797,689,674,697,825,1397,706 +org.apache.solr.util.CircularListTest=37,24,16,94,32,45,61,177,91,69,312,33,41,52,228,29,45,34 +org.apache.solr.util.DOMUtilTest=67,67,64,73,81,47,38,48,64,39,40,118,39,72,39,50,54,311 +org.apache.solr.util.DateMathParserTest=89,101,105,126,149,90,112,397,239,163,114,105,143,115,92,89,113,167 +org.apache.solr.util.FileUtilsTest=148,323,73,55,88,30,82,65,73,39,81,73,46,46,30,91,48,39 +org.apache.solr.util.PrimUtilsTest=98,71,89,72,65,98,87,372,147,56,97,47,41,63,60,81,56,89 +org.apache.solr.util.SolrPluginUtilsTest=3441,1309,1842,959,1141,1662,3778,1781,955,1712,971,1107,759,1241,904,1715,1088,1538 +org.apache.solr.util.TestNumberUtils=317,226,405,213,567,272,206,212,241,317,618,264,431,247,482,225,393,440 +org.apache.solr.util.TestSystemIdResolver=204,57,41,73,39,31,133,22,506,139,224,223,68,59,268,81,56,45 +org.apache.solr.util.TestUtils=69,41,64,46,56,248,38,31,31,243,39,45,34,98,31,56,56,199 +org.apache.solr.util.TimeZoneUtilsTest=131,157,115,467,105,149,104,156,80,348,81,164,115,131,122,103,140,156 +org.apache.solr.velocity.VelocityResponseWriterTest=1057,1086,1071,1052,1056,1043,1146,1062,1120,1131,1208,1125,1060,1039,1068,1036,1067,1095 +org.egothor.stemmer.TestCompile=252,223,287,301,219,325,198,245,253,260,244,248,288,229,247,228,264,222 +org.egothor.stemmer.TestStemmer=282,262,233,286,253,337,274,242,256,233,230,264,283,255,250,314,245,303 diff --git a/lucene/tools/src/java/org/apache/lucene/validation/LicenseCheckTask.java b/lucene/tools/src/java/org/apache/lucene/validation/LicenseCheckTask.java index 5474665..0704ea7 100644 --- a/lucene/tools/src/java/org/apache/lucene/validation/LicenseCheckTask.java +++ b/lucene/tools/src/java/org/apache/lucene/validation/LicenseCheckTask.java @@ -1,6 +1,6 @@ package org.apache.lucene.validation; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 2f52813..13c85c5 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -20,7 +20,49 @@ See the tutorial at http://lucene.apache.org/solr/tutorial.html $Id$ -================== 4.0.0-dev ================== +================== 5.0.0 ================== + +(No changes) + +================== 4.0.0-BETA =================== + +New Features + +* SOLR-1856: In Solr Cell, literals should override Tika-parsed values. + Patch adds a param "literalsOverride" which defaults to true, but can be set + to "false" to let Tika-parsed values be appended to literal values (Chris Harris, janhoy) + +* SOLR-3488: Added a Collection management API for SolrCloud. + (Tommaso Teofili, Sami Siren, yonik, Mark Miller) + +* SOLR-3559: Full deleteByQuery support with SolrCloud distributed indexing. + All replicas of a shard will be consistent, even if updates arrive in a + different order on different replicas. (yonik) + +* SOLR-1929: Index encrypted documents with ExtractingUpdateRequestHandler. + By supplying resource.password= or specifying an external file with regular + expressions matching file names, Solr will decrypt and index PDFs and DOCX formats. + (janhoy, Yiannis Pericleous) + +Bug Fixes + +* SOLR-3582: Our ZooKeeper watchers respond to session events as if they are change events, + creating undesirable side effects. (Trym R. Møller, Mark Miller) + +* SOLR-3467: ExtendedDismax escaping is missing several reserved characters + (Michael Dodsworth via janhoy) + +* SOLR-3587: After reloading a SolrCore, the original Analyzer is still used rather than a new + one. (Alexey Serba, yonik, rmuir, Mark Miller) + +Other Changes + +* SOLR-1770: Move the default core instance directory into a collection1 folder. + (Mark Miller) + +* SOLR-3355: Add shard and collection to SolrCore statistics. (Michael Garski, Mark Miller) + +================== 4.0.0-ALPHA ================== More information about this release, including any errata related to the release notes, upgrade instructions, or other changes may be found online at: https://wiki.apache.org/solr/Solr4.0 @@ -85,6 +127,14 @@ Upgrading from Solr 3.6-dev paths have been fixed to be resolved against the data dir. See the example solrconfig.xml and SOLR-1258 for more details. +* Due to low level changes to support SolrCloud, the uniqueKey field can no + longer be populated via or in the + schema.xml. Users wishing to have Solr automaticly generate a uniqueKey + value when adding documents should instead use an instance of + solr.UUIDUpdateProcessorFactory in their update processor chain. See + SOLR-2796 for more details. + + Detailed Change List ---------------------- @@ -240,6 +290,8 @@ New Features LastFieldValueUpdateProcessorFactory MinFieldValueUpdateProcessorFactory MaxFieldValueUpdateProcessorFactory + TruncateFieldUpdateProcessorFactory + IgnoreFieldUpdateProcessorFactory (hossman, janhoy) * SOLR-3120: Optional post filtering for spatial queries bbox and geofilt @@ -252,6 +304,9 @@ New Features (Russell Black, ryan) * SOLR-2898: Support grouped faceting. (Martijn van Groningen) + Additional Work: + SOLR-3406: Extended grouped faceting support to facet.query and facet.range parameters. + (David Boychuck, Martijn van Groningen) * SOLR-2949: QueryElevationComponent is now supported with distributed search. (Mark Miller, yonik) @@ -316,6 +371,43 @@ New Features prior "FieldName^boost" syntax is still accepted. In such cases the value on the "ps" parameter serves as the default slop. (Ron Mayer via James Dyer) +* SOLR-3495: New UpdateProcessors have been added to create default values for + configured fields. These works similarly to the + option in schema.xml, but are applied in the UpdateProcessorChain, so they + may be used prior to other UpdateProcessors, or to generate a uniqueKey field + value when using the DistributedUpdateProcessor (ie: SolrCloud) + TimestampUpdateProcessorFactory + UUIDUpdateProcessorFactory + DefaultValueUpdateProcessorFactory + (hossman) + +* SOLR-2993: Add WordBreakSolrSpellChecker to offer suggestions by combining adjacent + query terms and/or breaking terms into multiple words. This spellchecker can be + configured with a traditional checker (ie: DirectSolrSpellChecker). The results + are combined and collations can contain a mix of corrections from both spellcheckers. + (James Dyer) + +* SOLR-3508: Simplify JSON update format for deletes as well as allow + version specification for optimistic locking. Examples: + {"delete":"myid"} + {"delete":["id1","id2","id3"]} + {"delete":{"id":"myid", "_version_":123456789}} + (yonik) + +* SOLR-3211: Allow parameter overrides in conjunction with "spellcheck.maxCollationTries". + To do so, use parameters starting with "spellcheck.collateParam." For instance, to + override the "mm" parameter, specify "spellcheck.collateParam.mm". This is helpful + in cases where testing spellcheck collations for result counts should use different + parameters from the main query (James Dyer) + +* SOLR-2599: CloneFieldUpdateProcessorFactory provides similar functionality + to schema.xml's declaration but as an update processor that can + be combined with other processors in any order. (Jan Høydahl & hossman) + +* SOLR-3351: eDismax: ps2 and ps3 params (janhoy) + +* SOLR-3542: Add WeightedFragListBuilder for FVH and set it to default fragListBuilder + in example solrconfig.xml. (Sebastian Lutze, koji) Optimizations ---------------------- @@ -451,6 +543,19 @@ Bug Fixes * SOLR-2923: IllegalArgumentException when using useFilterForSortedQuery on an empty index. (Adrien Grand via Mark Miller) +* SOLR-2352: Fixed TermVectorComponent so that it will not fail if the fl + param contains globs or psuedo-fields (hossman) + +* SOLR-3541: add missing solrj dependencies to binary packages. + (Thijs Vonk via siren) + +* SOLR-3522: fixed parsing of the 'literal()' function (hossman) + +* SOLR-3548: Fixed a bug in the cachability of queries using the {!join} + parser or the strdist() function, as well as some minor improvements to + the hashCode implementation of {!bbox} and {!geofilt} queries. + (hossman) + Other Changes ---------------------- @@ -596,7 +701,14 @@ Other Changes * SOLR-3083: JMX beans now report Numbers as numeric values rather then String (Tagged Siteops, Greg Bowyer via ryan) +* SOLR-2796: Due to low level changes to support SolrCloud, the uniqueKey + field can no longer be populated via or + in the schema.xml. +* SOLR-3534: The Dismax and eDismax query parsers will fall back on the 'df' parameter + when 'qf' is absent. And if neither is present nor the schema default search field + then an exception will be thrown now. (dsmiley) + Documentation ---------------------- @@ -643,6 +755,9 @@ Upgrading from Solr 3.5 are strongly advised that they should re-index as document signatures may have now changed. (see SOLR-3200 & SOLR-3226 for details) +* SOLR-2724: Specifying and in + schema.xml is now considered deprecated. Instead you are encouraged to specify these via the "df" + and "q.op" parameters in your request handler definition. (David Smiley) New Features ---------------------- diff --git a/solr/common-build.xml b/solr/common-build.xml index 143d31f..ae1a462 100644 --- a/solr/common-build.xml +++ b/solr/common-build.xml @@ -24,7 +24,7 @@ - + @@ -76,7 +76,7 @@ By default, this should be set to "X.Y.M.${dateversion}" where X.Y.M is the last version released (on this branch). --> - + diff --git a/solr/contrib/analysis-extras/CHANGES.txt b/solr/contrib/analysis-extras/CHANGES.txt index b1d043a..6c72eff 100644 --- a/solr/contrib/analysis-extras/CHANGES.txt +++ b/solr/contrib/analysis-extras/CHANGES.txt @@ -11,7 +11,11 @@ analyzers for Chinese and Polish. $Id$ -================== 4.0.0-dev ============== +================== 5.0.0 ============== + + (No changes) + +================== 4.0.0-ALPHA ============== * SOLR-2396: Add ICUCollationField, which is much more efficient than the Solr 3.x ICUCollationKeyFilterFactory, and also supports diff --git a/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUCollationKeyFilterFactory.java b/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUCollationKeyFilterFactory.java deleted file mode 100644 index dfa9c7a..0000000 --- a/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUCollationKeyFilterFactory.java +++ /dev/null @@ -1,194 +0,0 @@ -package org.apache.solr.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.InputStream; - -import org.apache.commons.io.IOUtils; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.util.*; -import org.apache.lucene.collation.ICUCollationKeyFilter; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrException.ErrorCode; - -import com.ibm.icu.text.Collator; -import com.ibm.icu.text.RuleBasedCollator; -import com.ibm.icu.util.ULocale; - -/** - * - * Factory for ICUCollationKeyFilter. - *

      - * This factory can be created in two ways: - *

        - *
      • Based upon a system collator associated with a Locale. - *
      • Based upon a tailored ruleset. - *
      - *

      - * Using a System collator: - *

        - *
      • locale: RFC 3066 locale ID (mandatory) - *
      • strength: 'primary','secondary','tertiary', 'quaternary', or 'identical' (optional) - *
      • decomposition: 'no', or 'canonical' (optional) - *
      - *

      - * Using a Tailored ruleset: - *

        - *
      • custom: UTF-8 text file containing rules supported by RuleBasedCollator (mandatory) - *
      • strength: 'primary','secondary','tertiary', 'quaternary', or 'identical' (optional) - *
      • decomposition: 'no' or 'canonical' (optional) - *
      - *

      - * Expert options: - *

        - *
      • alternate: 'shifted' or 'non-ignorable'. Can be used to ignore punctuation/whitespace. - *
      • caseLevel: 'true' or 'false'. Useful with strength=primary to ignore accents but not case. - *
      • caseFirst: 'lower' or 'upper'. Useful to control which is sorted first when case is not ignored. - *
      • numeric: 'true' or 'false'. Digits are sorted according to numeric value, e.g. foobar-9 sorts before foobar-10 - *
      • variableTop: single character or contraction. Controls what is variable for 'alternate' - *
      - * - * @see Collator - * @see ULocale - * @see RuleBasedCollator - * @deprecated use {@link org.apache.solr.schema.ICUCollationField} instead. - */ -@Deprecated -public class ICUCollationKeyFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent, ResourceLoaderAware { - private Collator collator; - - public void inform(ResourceLoader loader) { - String custom = args.get("custom"); - String localeID = args.get("locale"); - String strength = args.get("strength"); - String decomposition = args.get("decomposition"); - - String alternate = args.get("alternate"); - String caseLevel = args.get("caseLevel"); - String caseFirst = args.get("caseFirst"); - String numeric = args.get("numeric"); - String variableTop = args.get("variableTop"); - - if (custom == null && localeID == null) - throw new SolrException(ErrorCode.SERVER_ERROR, "Either custom or locale is required."); - - if (custom != null && localeID != null) - throw new SolrException(ErrorCode.SERVER_ERROR, "Cannot specify both locale and custom. " - + "To tailor rules for a built-in language, see the javadocs for RuleBasedCollator. " - + "Then save the entire customized ruleset to a file, and use with the custom parameter"); - - if (localeID != null) { - // create from a system collator, based on Locale. - collator = createFromLocale(localeID); - } else { - // create from a custom ruleset - collator = createFromRules(custom, loader); - } - - // set the strength flag, otherwise it will be the default. - if (strength != null) { - if (strength.equalsIgnoreCase("primary")) - collator.setStrength(Collator.PRIMARY); - else if (strength.equalsIgnoreCase("secondary")) - collator.setStrength(Collator.SECONDARY); - else if (strength.equalsIgnoreCase("tertiary")) - collator.setStrength(Collator.TERTIARY); - else if (strength.equalsIgnoreCase("quaternary")) - collator.setStrength(Collator.QUATERNARY); - else if (strength.equalsIgnoreCase("identical")) - collator.setStrength(Collator.IDENTICAL); - else - throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid strength: " + strength); - } - - // set the decomposition flag, otherwise it will be the default. - if (decomposition != null) { - if (decomposition.equalsIgnoreCase("no")) - collator.setDecomposition(Collator.NO_DECOMPOSITION); - else if (decomposition.equalsIgnoreCase("canonical")) - collator.setDecomposition(Collator.CANONICAL_DECOMPOSITION); - else - throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid decomposition: " + decomposition); - } - - // expert options: concrete subclasses are always a RuleBasedCollator - RuleBasedCollator rbc = (RuleBasedCollator) collator; - if (alternate != null) { - if (alternate.equalsIgnoreCase("shifted")) { - rbc.setAlternateHandlingShifted(true); - } else if (alternate.equalsIgnoreCase("non-ignorable")) { - rbc.setAlternateHandlingShifted(false); - } else { - throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid alternate: " + alternate); - } - } - if (caseLevel != null) { - rbc.setCaseLevel(Boolean.parseBoolean(caseLevel)); - } - if (caseFirst != null) { - if (caseFirst.equalsIgnoreCase("lower")) { - rbc.setLowerCaseFirst(true); - } else if (caseFirst.equalsIgnoreCase("upper")) { - rbc.setUpperCaseFirst(true); - } else { - throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid caseFirst: " + caseFirst); - } - } - if (numeric != null) { - rbc.setNumericCollation(Boolean.parseBoolean(numeric)); - } - if (variableTop != null) { - rbc.setVariableTop(variableTop); - } - } - - public TokenStream create(TokenStream input) { - return new ICUCollationKeyFilter(input, collator); - } - - /* - * Create a locale from localeID. - * Then return the appropriate collator for the locale. - */ - private Collator createFromLocale(String localeID) { - return Collator.getInstance(new ULocale(localeID)); - } - - /* - * Read custom rules from a file, and create a RuleBasedCollator - * The file cannot support comments, as # might be in the rules! - */ - private Collator createFromRules(String fileName, ResourceLoader loader) { - InputStream input = null; - try { - input = loader.openResource(fileName); - String rules = IOUtils.toString(input, "UTF-8"); - return new RuleBasedCollator(rules); - } catch (Exception e) { - // io error or invalid rules - throw new RuntimeException(e); - } finally { - IOUtils.closeQuietly(input); - } - } - - @Override - public AbstractAnalysisFactory getMultiTermComponent() { - return this; - } -} diff --git a/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUFoldingFilterFactory.java b/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUFoldingFilterFactory.java index ad89b94..0fdf711 100644 --- a/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUFoldingFilterFactory.java +++ b/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUFoldingFilterFactory.java @@ -6,7 +6,7 @@ import org.apache.lucene.analysis.util.AbstractAnalysisFactory; import org.apache.lucene.analysis.util.MultiTermAwareComponent; import org.apache.lucene.analysis.util.TokenFilterFactory; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUNormalizer2FilterFactory.java b/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUNormalizer2FilterFactory.java index 93d1ae9..0aef42f 100644 --- a/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUNormalizer2FilterFactory.java +++ b/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUNormalizer2FilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUTokenizerFactory.java b/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUTokenizerFactory.java index 3b4c7fd..061bc3c 100644 --- a/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUTokenizerFactory.java +++ b/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUTokenizerFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUTransformFilterFactory.java b/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUTransformFilterFactory.java index 001750b..695c1ba 100644 --- a/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUTransformFilterFactory.java +++ b/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUTransformFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/MorfologikFilterFactory.java b/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/MorfologikFilterFactory.java index f18ba35..764d622 100644 --- a/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/MorfologikFilterFactory.java +++ b/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/MorfologikFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/SmartChineseSentenceTokenizerFactory.java b/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/SmartChineseSentenceTokenizerFactory.java index 94c605d..0df634f 100644 --- a/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/SmartChineseSentenceTokenizerFactory.java +++ b/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/SmartChineseSentenceTokenizerFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/SmartChineseWordTokenFilterFactory.java b/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/SmartChineseWordTokenFilterFactory.java index 3395111..3742fec 100644 --- a/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/SmartChineseWordTokenFilterFactory.java +++ b/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/SmartChineseWordTokenFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/StempelPolishStemFilterFactory.java b/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/StempelPolishStemFilterFactory.java index e8c6089..3340692 100644 --- a/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/StempelPolishStemFilterFactory.java +++ b/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/StempelPolishStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java b/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java index 82c5181..88e1ce5 100644 --- a/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java +++ b/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java @@ -1,6 +1,6 @@ package org.apache.solr.schema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-folding-extra.xml b/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-folding-extra.xml new file mode 100644 index 0000000..5fa8e4e --- /dev/null +++ b/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-folding-extra.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + id + + diff --git a/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-icucollate.xml b/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-icucollate.xml new file mode 100644 index 0000000..3ec19c6 --- /dev/null +++ b/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-icucollate.xml @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text + id + + + + + + + + diff --git a/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-icucollateoptions.xml b/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-icucollateoptions.xml new file mode 100644 index 0000000..2413fc2 --- /dev/null +++ b/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-icucollateoptions.xml @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text + id + + + + + + + + diff --git a/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-icucollate.xml b/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-icucollate.xml new file mode 100644 index 0000000..2ae6c7f --- /dev/null +++ b/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-icucollate.xml @@ -0,0 +1,24 @@ + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + diff --git a/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/conf/schema-folding-extra.xml b/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/conf/schema-folding-extra.xml deleted file mode 100644 index 5fa8e4e..0000000 --- a/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/conf/schema-folding-extra.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - id - - diff --git a/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/conf/schema-icucollate.xml b/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/conf/schema-icucollate.xml deleted file mode 100644 index 3ec19c6..0000000 --- a/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/conf/schema-icucollate.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text - id - - - - - - - - diff --git a/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/conf/schema-icucollatefilter.xml b/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/conf/schema-icucollatefilter.xml deleted file mode 100644 index dba7aeb..0000000 --- a/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/conf/schema-icucollatefilter.xml +++ /dev/null @@ -1,61 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text - id - - - - - diff --git a/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/conf/schema-icucollateoptions.xml b/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/conf/schema-icucollateoptions.xml deleted file mode 100644 index 2413fc2..0000000 --- a/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/conf/schema-icucollateoptions.xml +++ /dev/null @@ -1,69 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text - id - - - - - - - - diff --git a/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/conf/solrconfig-icucollate.xml b/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/conf/solrconfig-icucollate.xml deleted file mode 100644 index 2ae6c7f..0000000 --- a/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/conf/solrconfig-icucollate.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - diff --git a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUCollationKeyFilterFactory.java b/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUCollationKeyFilterFactory.java deleted file mode 100644 index 2386533..0000000 --- a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUCollationKeyFilterFactory.java +++ /dev/null @@ -1,278 +0,0 @@ -package org.apache.solr.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.io.StringReader; -import java.util.HashMap; -import java.util.Map; - -import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.core.KeywordTokenizer; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; - -import com.ibm.icu.text.Collator; -import com.ibm.icu.text.RuleBasedCollator; -import com.ibm.icu.util.ULocale; - -@Deprecated -public class TestICUCollationKeyFilterFactory extends BaseTokenStreamTestCase { - - /* - * Turkish has some funny casing. - * This test shows how you can solve this kind of thing easily with collation. - * Instead of using LowerCaseFilter, use a turkish collator with primary strength. - * Then things will sort and match correctly. - */ - public void testBasicUsage() throws IOException { - String turkishUpperCase = "I WÄ°LL USE TURKÄ°SH CASING"; - String turkishLowerCase = "ı will use turkish casıng"; - ICUCollationKeyFilterFactory factory = new ICUCollationKeyFilterFactory(); - Map args = new HashMap(); - args.put("locale", "tr"); - args.put("strength", "primary"); - factory.init(args); - factory.inform(new StringMockSolrResourceLoader("")); - TokenStream tsUpper = factory.create( - new KeywordTokenizer(new StringReader(turkishUpperCase))); - TokenStream tsLower = factory.create( - new KeywordTokenizer(new StringReader(turkishLowerCase))); - assertCollatesToSame(tsUpper, tsLower); - } - - /* - * Test usage of the decomposition option for unicode normalization. - */ - public void testNormalization() throws IOException { - String turkishUpperCase = "I W\u0049\u0307LL USE TURKÄ°SH CASING"; - String turkishLowerCase = "ı will use turkish casıng"; - ICUCollationKeyFilterFactory factory = new ICUCollationKeyFilterFactory(); - Map args = new HashMap(); - args.put("locale", "tr"); - args.put("strength", "primary"); - args.put("decomposition", "canonical"); - factory.init(args); - factory.inform(new StringMockSolrResourceLoader("")); - TokenStream tsUpper = factory.create( - new KeywordTokenizer(new StringReader(turkishUpperCase))); - TokenStream tsLower = factory.create( - new KeywordTokenizer(new StringReader(turkishLowerCase))); - assertCollatesToSame(tsUpper, tsLower); - } - - /* - * Test secondary strength, for english case is not significant. - */ - public void testSecondaryStrength() throws IOException { - String upperCase = "TESTING"; - String lowerCase = "testing"; - ICUCollationKeyFilterFactory factory = new ICUCollationKeyFilterFactory(); - Map args = new HashMap(); - args.put("locale", "en"); - args.put("strength", "secondary"); - args.put("decomposition", "no"); - factory.init(args); - factory.inform(new StringMockSolrResourceLoader("")); - TokenStream tsUpper = factory.create( - new KeywordTokenizer(new StringReader(upperCase))); - TokenStream tsLower = factory.create( - new KeywordTokenizer(new StringReader(lowerCase))); - assertCollatesToSame(tsUpper, tsLower); - } - - /* - * Setting alternate=shifted to shift whitespace, punctuation and symbols - * to quaternary level - */ - public void testIgnorePunctuation() throws IOException { - String withPunctuation = "foo-bar"; - String withoutPunctuation = "foo bar"; - ICUCollationKeyFilterFactory factory = new ICUCollationKeyFilterFactory(); - Map args = new HashMap(); - args.put("locale", "en"); - args.put("strength", "primary"); - args.put("alternate", "shifted"); - factory.init(args); - factory.inform(new StringMockSolrResourceLoader("")); - TokenStream tsPunctuation = factory.create( - new KeywordTokenizer(new StringReader(withPunctuation))); - TokenStream tsWithoutPunctuation = factory.create( - new KeywordTokenizer(new StringReader(withoutPunctuation))); - assertCollatesToSame(tsPunctuation, tsWithoutPunctuation); - } - - /* - * Setting alternate=shifted and variableTop to shift whitespace, but not - * punctuation or symbols, to quaternary level - */ - public void testIgnoreWhitespace() throws IOException { - String withSpace = "foo bar"; - String withoutSpace = "foobar"; - String withPunctuation = "foo-bar"; - ICUCollationKeyFilterFactory factory = new ICUCollationKeyFilterFactory(); - Map args = new HashMap(); - args.put("locale", "en"); - args.put("strength", "primary"); - args.put("alternate", "shifted"); - args.put("variableTop", " "); - factory.init(args); - factory.inform(new StringMockSolrResourceLoader("")); - TokenStream tsWithSpace = factory.create( - new KeywordTokenizer(new StringReader(withSpace))); - TokenStream tsWithoutSpace = factory.create( - new KeywordTokenizer(new StringReader(withoutSpace))); - assertCollatesToSame(tsWithSpace, tsWithoutSpace); - // now assert that punctuation still matters: foo-bar < foo bar - tsWithSpace = factory.create( - new KeywordTokenizer(new StringReader(withSpace))); - TokenStream tsWithPunctuation = factory.create( - new KeywordTokenizer(new StringReader(withPunctuation))); - assertCollation(tsWithPunctuation, tsWithSpace, -1); - } - - /* - * Setting numeric to encode digits with numeric value, so that - * foobar-9 sorts before foobar-10 - */ - public void testNumerics() throws IOException { - String nine = "foobar-9"; - String ten = "foobar-10"; - ICUCollationKeyFilterFactory factory = new ICUCollationKeyFilterFactory(); - Map args = new HashMap(); - args.put("locale", "en"); - args.put("numeric", "true"); - factory.init(args); - factory.inform(new StringMockSolrResourceLoader("")); - TokenStream tsNine = factory.create( - new KeywordTokenizer(new StringReader(nine))); - TokenStream tsTen = factory.create( - new KeywordTokenizer(new StringReader(ten))); - assertCollation(tsNine, tsTen, -1); - } - - /* - * Setting caseLevel=true to create an additional case level between - * secondary and tertiary - */ - public void testIgnoreAccentsButNotCase() throws IOException { - String withAccents = "résumé"; - String withoutAccents = "resume"; - String withAccentsUpperCase = "Résumé"; - String withoutAccentsUpperCase = "Resume"; - ICUCollationKeyFilterFactory factory = new ICUCollationKeyFilterFactory(); - Map args = new HashMap(); - args.put("locale", "en"); - args.put("strength", "primary"); - args.put("caseLevel", "true"); - factory.init(args); - factory.inform(new StringMockSolrResourceLoader("")); - TokenStream tsWithAccents = factory.create( - new KeywordTokenizer(new StringReader(withAccents))); - TokenStream tsWithoutAccents = factory.create( - new KeywordTokenizer(new StringReader(withoutAccents))); - assertCollatesToSame(tsWithAccents, tsWithoutAccents); - - TokenStream tsWithAccentsUpperCase = factory.create( - new KeywordTokenizer(new StringReader(withAccentsUpperCase))); - TokenStream tsWithoutAccentsUpperCase = factory.create( - new KeywordTokenizer(new StringReader(withoutAccentsUpperCase))); - assertCollatesToSame(tsWithAccentsUpperCase, tsWithoutAccentsUpperCase); - - // now assert that case still matters: resume < Resume - TokenStream tsLower = factory.create( - new KeywordTokenizer(new StringReader(withoutAccents))); - TokenStream tsUpper = factory.create( - new KeywordTokenizer(new StringReader(withoutAccentsUpperCase))); - assertCollation(tsLower, tsUpper, -1); - } - - /* - * Setting caseFirst=upper to cause uppercase strings to sort - * before lowercase ones. - */ - public void testUpperCaseFirst() throws IOException { - String lower = "resume"; - String upper = "Resume"; - ICUCollationKeyFilterFactory factory = new ICUCollationKeyFilterFactory(); - Map args = new HashMap(); - args.put("locale", "en"); - args.put("strength", "tertiary"); - args.put("caseFirst", "upper"); - factory.init(args); - factory.inform(new StringMockSolrResourceLoader("")); - TokenStream tsLower = factory.create( - new KeywordTokenizer(new StringReader(lower))); - TokenStream tsUpper = factory.create( - new KeywordTokenizer(new StringReader(upper))); - assertCollation(tsUpper, tsLower, -1); - } - - /* - * For german, you might want oe to sort and match with o umlaut. - * This is not the default, but you can make a customized ruleset to do this. - * - * The default is DIN 5007-1, this shows how to tailor a collator to get DIN 5007-2 behavior. - * http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4423383 - */ - public void testCustomRules() throws Exception { - RuleBasedCollator baseCollator = (RuleBasedCollator) Collator.getInstance(new ULocale("de_DE")); - - String DIN5007_2_tailorings = - "& ae , a\u0308 & AE , A\u0308"+ - "& oe , o\u0308 & OE , O\u0308"+ - "& ue , u\u0308 & UE , u\u0308"; - - RuleBasedCollator tailoredCollator = new RuleBasedCollator(baseCollator.getRules() + DIN5007_2_tailorings); - String tailoredRules = tailoredCollator.getRules(); - // - // at this point, you would save these tailoredRules to a file, - // and use the custom parameter. - // - String germanUmlaut = "Töne"; - String germanOE = "Toene"; - ICUCollationKeyFilterFactory factory = new ICUCollationKeyFilterFactory(); - Map args = new HashMap(); - args.put("custom", "rules.txt"); - args.put("strength", "primary"); - factory.init(args); - factory.inform(new StringMockSolrResourceLoader(tailoredRules)); - TokenStream tsUmlaut = factory.create( - new KeywordTokenizer(new StringReader(germanUmlaut))); - TokenStream tsOE = factory.create( - new KeywordTokenizer(new StringReader(germanOE))); - - assertCollatesToSame(tsUmlaut, tsOE); - } - - private void assertCollatesToSame(TokenStream stream1, TokenStream stream2) throws IOException { - assertCollation(stream1, stream2, 0); - } - - private void assertCollation(TokenStream stream1, TokenStream stream2, int comparison) throws IOException { - CharTermAttribute term1 = stream1 - .addAttribute(CharTermAttribute.class); - CharTermAttribute term2 = stream2 - .addAttribute(CharTermAttribute.class); - assertTrue(stream1.incrementToken()); - assertTrue(stream2.incrementToken()); - assertEquals(Integer.signum(comparison), Integer.signum(term1.toString().compareTo(term2.toString()))); - assertFalse(stream1.incrementToken()); - assertFalse(stream2.incrementToken()); - } -} diff --git a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUCollationKeyRangeQueries.java b/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUCollationKeyRangeQueries.java deleted file mode 100644 index 800dd31..0000000 --- a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUCollationKeyRangeQueries.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.analysis; - -import org.apache.solr.SolrTestCaseJ4; -import org.junit.BeforeClass; - -/** - * Tests {@link ICUCollationKeyFilterFactory} with RangeQueries - */ -public class TestICUCollationKeyRangeQueries extends SolrTestCaseJ4 { - - @BeforeClass - public static void beforeClass() throws Exception { - initCore("solrconfig-icucollate.xml","schema-icucollatefilter.xml", "analysis-extras/solr"); - // add some docs - assertU(adoc("id", "1", "text", "\u0633\u0627\u0628")); - assertU(adoc("id", "2", "text", "I WÄ°LL USE TURKÄ°SH CASING")); - assertU(adoc("id", "3", "text", "ı will use turkish casıng")); - assertU(adoc("id", "4", "text", "Töne")); - assertU(adoc("id", "5", "text", "I W\u0049\u0307LL USE TURKÄ°SH CASING")); - assertU(adoc("id", "6", "text", "Ｔｅｓｔｉｎｇ")); - assertU(adoc("id", "7", "text", "Tone")); - assertU(adoc("id", "8", "text", "Testing")); - assertU(adoc("id", "9", "text", "testing")); - assertU(adoc("id", "10", "text", "toene")); - assertU(adoc("id", "11", "text", "Tzne")); - assertU(adoc("id", "12", "text", "\u0698\u0698")); - assertU(commit()); - } - - /** - * Test termquery with german DIN 5007-1 primary strength. - * In this case, ö is equivalent to o (but not oe) - */ - public void testBasicTermQuery() { - assertQ("Collated TQ: ", - req("fl", "id", "q", "sort_de:tone", "sort", "id asc" ), - "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=4]", - "//result/doc[2]/int[@name='id'][.=7]" - ); - } - - /** - * Test rangequery again with the DIN 5007-1 collator. - * We do a range query of tone .. tp, in binary order this - * would retrieve nothing due to case and accent differences. - */ - public void testBasicRangeQuery() { - assertQ("Collated RangeQ: ", - req("fl", "id", "q", "sort_de:[tone TO tp]", "sort", "id asc" ), - "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=4]", - "//result/doc[2]/int[@name='id'][.=7]" - ); - } - - /** - * Test rangequery again with an Arabic collator. - * Binary order would normally order U+0633 in this range. - */ - public void testNegativeRangeQuery() { - assertQ("Collated RangeQ: ", - req("fl", "id", "q", "sort_ar:[\u062F TO \u0698]", "sort", "id asc" ), - "//*[@numFound='0']" - ); - } -} diff --git a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUFoldingFilterFactory.java b/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUFoldingFilterFactory.java index bce4228..dc30038 100644 --- a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUFoldingFilterFactory.java +++ b/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUFoldingFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUNormalizer2FilterFactory.java b/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUNormalizer2FilterFactory.java index a8702cd..0b1adf2 100644 --- a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUNormalizer2FilterFactory.java +++ b/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUNormalizer2FilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUTokenizerFactory.java b/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUTokenizerFactory.java index cc516f1..21e6afb 100644 --- a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUTokenizerFactory.java +++ b/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUTokenizerFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUTransformFilterFactory.java b/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUTransformFilterFactory.java index 07cb3e5..6092cdf 100644 --- a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUTransformFilterFactory.java +++ b/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUTransformFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestMorfologikFilterFactory.java b/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestMorfologikFilterFactory.java index 599fd42..30b3ba0 100644 --- a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestMorfologikFilterFactory.java +++ b/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestMorfologikFilterFactory.java @@ -7,9 +7,8 @@ import java.util.Map; import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.core.WhitespaceTokenizer; -import org.apache.solr.schema.IndexSchema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestSmartChineseFactories.java b/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestSmartChineseFactories.java index a4806ea..35fb947 100644 --- a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestSmartChineseFactories.java +++ b/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestSmartChineseFactories.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestStempelPolishStemFilterFactory.java b/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestStempelPolishStemFilterFactory.java index 4f69866..46e8308 100644 --- a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestStempelPolishStemFilterFactory.java +++ b/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestStempelPolishStemFilterFactory.java @@ -7,7 +7,7 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.apache.solr.core.SolrResourceLoader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java b/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java index 347e277..76124eb 100644 --- a/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java +++ b/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,7 +22,6 @@ import java.io.FileOutputStream; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; -import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.apache.solr.SolrTestCaseJ4; import org.junit.BeforeClass; @@ -33,7 +32,6 @@ import com.ibm.icu.util.ULocale; /** * Tests {@link ICUCollationField} with TermQueries, RangeQueries, and sort order. */ -@SuppressCodecs("Lucene3x") public class TestICUCollationField extends SolrTestCaseJ4 { @BeforeClass @@ -69,13 +67,13 @@ public class TestICUCollationField extends SolrTestCaseJ4 { tmpFile.mkdir(); // make data and conf dirs - new File(tmpFile, "data").mkdir(); - File confDir = new File(tmpFile, "conf"); - confDir.mkdir(); + new File(tmpFile + "/collection1", "data").mkdirs(); + File confDir = new File(tmpFile + "/collection1", "conf"); + confDir.mkdirs(); // copy over configuration files - FileUtils.copyFile(getFile("analysis-extras/solr/conf/solrconfig-icucollate.xml"), new File(confDir, "solrconfig.xml")); - FileUtils.copyFile(getFile("analysis-extras/solr/conf/schema-icucollate.xml"), new File(confDir, "schema.xml")); + FileUtils.copyFile(getFile("analysis-extras/solr/collection1/conf/solrconfig-icucollate.xml"), new File(confDir, "solrconfig.xml")); + FileUtils.copyFile(getFile("analysis-extras/solr/collection1/conf/schema-icucollate.xml"), new File(confDir, "schema.xml")); // generate custom collation rules (DIN 5007-2), saving to customrules.dat RuleBasedCollator baseCollator = (RuleBasedCollator) Collator.getInstance(new ULocale("de", "DE")); diff --git a/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldOptions.java b/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldOptions.java index 92ad734..95a2993 100644 --- a/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldOptions.java +++ b/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldOptions.java @@ -1,6 +1,6 @@ package org.apache.solr.schema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,14 +17,12 @@ package org.apache.solr.schema; * limitations under the License. */ -import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.apache.solr.SolrTestCaseJ4; import org.junit.BeforeClass; /** * Tests expert options of {@link ICUCollationField}. */ -@SuppressCodecs("Lucene3x") public class TestICUCollationFieldOptions extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { diff --git a/solr/contrib/clustering/CHANGES.txt b/solr/contrib/clustering/CHANGES.txt index ebda207..6e97c63 100644 --- a/solr/contrib/clustering/CHANGES.txt +++ b/solr/contrib/clustering/CHANGES.txt @@ -7,7 +7,11 @@ See http://wiki.apache.org/solr/ClusteringComponent CHANGES $Id$ -================== Release 4.0.0-dev ============== +================== Release 5.0.0 ============== + + (No changes) + +================== Release 4.0.0-ALPHA ============== * SOLR-3470: Bug fix: custom Carrot2 tokenizer and stemmer factories are respected now (Stanislaw Osinski, Dawid Weiss) diff --git a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java index c5b345a..697babe 100644 --- a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java +++ b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.clustering; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/ClusteringEngine.java b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/ClusteringEngine.java index 85bfffa..b8a7079 100644 --- a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/ClusteringEngine.java +++ b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/ClusteringEngine.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.clustering; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/ClusteringParams.java b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/ClusteringParams.java index d5ed5b6..19ba8ba 100644 --- a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/ClusteringParams.java +++ b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/ClusteringParams.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.clustering; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/DocumentClusteringEngine.java b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/DocumentClusteringEngine.java index 44f5de0..2926b8f 100644 --- a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/DocumentClusteringEngine.java +++ b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/DocumentClusteringEngine.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.clustering; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/SearchClusteringEngine.java b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/SearchClusteringEngine.java index 021f3f1..5be8e73 100644 --- a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/SearchClusteringEngine.java +++ b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/SearchClusteringEngine.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.clustering; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java index b086318..52a85eb 100644 --- a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java +++ b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.clustering.carrot2; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -147,7 +147,7 @@ public class CarrotClusteringEngine extends SearchClusteringEngine { final IResource foundResource = new IResource() { @Override - public InputStream open() throws IOException { + public InputStream open() { return new ByteArrayInputStream(asBytes); } diff --git a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotParams.java b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotParams.java index f9d7b75..1de3583 100644 --- a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotParams.java +++ b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotParams.java @@ -4,7 +4,7 @@ import java.util.Set; import com.google.common.collect.ImmutableSet; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2StemmerFactory.java b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2StemmerFactory.java index e890e86..9170e8b 100644 --- a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2StemmerFactory.java +++ b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2StemmerFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.clustering.carrot2; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -180,7 +180,7 @@ public class LuceneCarrot2StemmerFactory implements IStemmerFactory { private char[] buffer = new char[0]; - private LuceneStemmerAdapter() throws Exception { + private LuceneStemmerAdapter() { delegate = new org.apache.lucene.analysis.ar.ArabicStemmer(); normalizer = new org.apache.lucene.analysis.ar.ArabicNormalizer(); } diff --git a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2TokenizerFactory.java b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2TokenizerFactory.java index faaf85a..16b69bd 100644 --- a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2TokenizerFactory.java +++ b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2TokenizerFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.clustering.carrot2; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -141,7 +141,7 @@ public class LuceneCarrot2TokenizerFactory implements ITokenizerFactory { array.reset(term.buffer(), 0, term.length()); } - public void reset(Reader input) throws IOException { + public void reset(Reader input) { try { sentenceTokenizer.reset(input); wordTokenFilter = (TokenStream) tokenFilterClass.getConstructor( diff --git a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrStopwordsCarrot2LexicalDataFactory.java b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrStopwordsCarrot2LexicalDataFactory.java index c75ca10..880d232 100644 --- a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrStopwordsCarrot2LexicalDataFactory.java +++ b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrStopwordsCarrot2LexicalDataFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.clustering.carrot2; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/clustering/carrot2/stoplabels.mt b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/clustering/carrot2/stoplabels.mt new file mode 100644 index 0000000..71b2dde --- /dev/null +++ b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/clustering/carrot2/stoplabels.mt @@ -0,0 +1 @@ +customsolrstoplabel diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/clustering/carrot2/stopwords.mt b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/clustering/carrot2/stopwords.mt new file mode 100644 index 0000000..5dd05b1 --- /dev/null +++ b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/clustering/carrot2/stopwords.mt @@ -0,0 +1 @@ +customsolrstopword diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/clustering/custom/stoplabels.mt b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/clustering/custom/stoplabels.mt new file mode 100644 index 0000000..6ee28d8 --- /dev/null +++ b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/clustering/custom/stoplabels.mt @@ -0,0 +1 @@ +customsolrstoplabelcustomdir diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/clustering/custom/stopwords.mt b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/clustering/custom/stopwords.mt new file mode 100644 index 0000000..6e604da --- /dev/null +++ b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/clustering/custom/stopwords.mt @@ -0,0 +1 @@ +customsolrstopwordcustomdir diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/mapping-ISOLatin1Accent.txt b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/mapping-ISOLatin1Accent.txt new file mode 100644 index 0000000..ede7742 --- /dev/null +++ b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/mapping-ISOLatin1Accent.txt @@ -0,0 +1,246 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Syntax: +# "source" => "target" +# "source".length() > 0 (source cannot be empty.) +# "target".length() >= 0 (target can be empty.) + +# example: +# "À" => "A" +# "\u00C0" => "A" +# "\u00C0" => "\u0041" +# "ß" => "ss" +# "\t" => " " +# "\n" => "" + +# À => A +"\u00C0" => "A" + +# à => A +"\u00C1" => "A" + +#  => A +"\u00C2" => "A" + +# à => A +"\u00C3" => "A" + +# Ä => A +"\u00C4" => "A" + +# Ã… => A +"\u00C5" => "A" + +# Æ => AE +"\u00C6" => "AE" + +# Ç => C +"\u00C7" => "C" + +# È => E +"\u00C8" => "E" + +# É => E +"\u00C9" => "E" + +# Ê => E +"\u00CA" => "E" + +# Ë => E +"\u00CB" => "E" + +# ÃŒ => I +"\u00CC" => "I" + +# à => I +"\u00CD" => "I" + +# ÃŽ => I +"\u00CE" => "I" + +# à => I +"\u00CF" => "I" + +# IJ => IJ +"\u0132" => "IJ" + +# à => D +"\u00D0" => "D" + +# Ñ => N +"\u00D1" => "N" + +# Ã’ => O +"\u00D2" => "O" + +# Ó => O +"\u00D3" => "O" + +# Ô => O +"\u00D4" => "O" + +# Õ => O +"\u00D5" => "O" + +# Ö => O +"\u00D6" => "O" + +# Ø => O +"\u00D8" => "O" + +# Å’ => OE +"\u0152" => "OE" + +# Þ +"\u00DE" => "TH" + +# Ù => U +"\u00D9" => "U" + +# Ú => U +"\u00DA" => "U" + +# Û => U +"\u00DB" => "U" + +# Ãœ => U +"\u00DC" => "U" + +# à => Y +"\u00DD" => "Y" + +# Ÿ => Y +"\u0178" => "Y" + +# à => a +"\u00E0" => "a" + +# á => a +"\u00E1" => "a" + +# â => a +"\u00E2" => "a" + +# ã => a +"\u00E3" => "a" + +# ä => a +"\u00E4" => "a" + +# Ã¥ => a +"\u00E5" => "a" + +# æ => ae +"\u00E6" => "ae" + +# ç => c +"\u00E7" => "c" + +# è => e +"\u00E8" => "e" + +# é => e +"\u00E9" => "e" + +# ê => e +"\u00EA" => "e" + +# ë => e +"\u00EB" => "e" + +# ì => i +"\u00EC" => "i" + +# í => i +"\u00ED" => "i" + +# î => i +"\u00EE" => "i" + +# ï => i +"\u00EF" => "i" + +# ij => ij +"\u0133" => "ij" + +# ð => d +"\u00F0" => "d" + +# ñ => n +"\u00F1" => "n" + +# ò => o +"\u00F2" => "o" + +# ó => o +"\u00F3" => "o" + +# ô => o +"\u00F4" => "o" + +# õ => o +"\u00F5" => "o" + +# ö => o +"\u00F6" => "o" + +# ø => o +"\u00F8" => "o" + +# Å“ => oe +"\u0153" => "oe" + +# ß => ss +"\u00DF" => "ss" + +# þ => th +"\u00FE" => "th" + +# ù => u +"\u00F9" => "u" + +# ú => u +"\u00FA" => "u" + +# û => u +"\u00FB" => "u" + +# ü => u +"\u00FC" => "u" + +# ý => y +"\u00FD" => "y" + +# ÿ => y +"\u00FF" => "y" + +# ff => ff +"\uFB00" => "ff" + +# ï¬ => fi +"\uFB01" => "fi" + +# fl => fl +"\uFB02" => "fl" + +# ffi => ffi +"\uFB03" => "ffi" + +# ffl => ffl +"\uFB04" => "ffl" + +# ſt => ft +"\uFB05" => "ft" + +# st => st +"\uFB06" => "st" diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/protwords.txt b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/protwords.txt new file mode 100644 index 0000000..1dfc0ab --- /dev/null +++ b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/protwords.txt @@ -0,0 +1,21 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# Use a protected word file to protect against the stemmer reducing two +# unrelated words to the same base word. + +# Some non-words that normally won't be encountered, +# just to test that they won't be stemmed. +dontstems +zwhacky + diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/schema.xml b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/schema.xml new file mode 100644 index 0000000..64ca87a --- /dev/null +++ b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/schema.xml @@ -0,0 +1,352 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + text + + + + + + + + + + + + + + + + + diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml new file mode 100644 index 0000000..76cd7d4 --- /dev/null +++ b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml @@ -0,0 +1,476 @@ + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + ${solr.data.dir:} + + + + + single + + + + + + + + + + + + + + + + + + + + + + 1024 + + + + + + + + + + + + + true + + + + + + + + 50 + + + 200 + + + + + + + + + solr 0 10 + rocks 0 10 + static newSearcher warming query from solrconfig.xml + + + + + + + fast_warm 0 10 + static firstSearcher warming query from solrconfig.xml + + + + + false + + + 2 + + + + + + + + + + + + + + + + + + + + + + + explicit + + + + clustering + + + + + + + + explicit + + + + doc-clustering + + + + + + + + + + + default + org.carrot2.clustering.lingo.LingoClusteringAlgorithm + + + stc + org.carrot2.clustering.stc.STCClusteringAlgorithm + + + mock + org.apache.solr.handler.clustering.carrot2.MockClusteringAlgorithm + + + echo + org.apache.solr.handler.clustering.carrot2.EchoClusteringAlgorithm + + + lexical-resource-check + org.apache.solr.handler.clustering.carrot2.LexicalResourcesCheckClusteringAlgorithm + + + lexical-resource-check-custom-resource-dir + org.apache.solr.handler.clustering.carrot2.LexicalResourcesCheckClusteringAlgorithm + clustering/custom + + + custom-duplicating-tokenizer + org.apache.solr.handler.clustering.carrot2.EchoTokensClusteringAlgorithm + org.apache.solr.handler.clustering.carrot2.DuplicatingTokenizerFactory + + + custom-duplicating-stemmer + org.apache.solr.handler.clustering.carrot2.EchoStemsClusteringAlgorithm + org.apache.solr.handler.clustering.carrot2.DuplicatingStemmerFactory + + + + + + + + mock + org.apache.solr.handler.clustering.MockDocumentClusteringEngine + + + + + + + + + + + + + + + + + explicit + true + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + 5 + + + + + + + + + + solr + + + diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/spellings.txt b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/spellings.txt new file mode 100644 index 0000000..d7ede6f --- /dev/null +++ b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/spellings.txt @@ -0,0 +1,2 @@ +pizza +history \ No newline at end of file diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/stopwords.txt b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/stopwords.txt new file mode 100644 index 0000000..54f0b99 --- /dev/null +++ b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/stopwords.txt @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +#Standard english stop words taken from Lucene's StopAnalyzer +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +s +such +t +that +the +their +then +there +these +they +this +to +was +will +with +solrownstopword + diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/synonyms.txt b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/synonyms.txt new file mode 100644 index 0000000..b0e31cb --- /dev/null +++ b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/synonyms.txt @@ -0,0 +1,31 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +aaa => aaaa +bbb => bbbb1 bbbb2 +ccc => cccc1,cccc2 +a\=>a => b\=>b +a\,a => b\,b +fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +pixima => pixma + diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/conf/clustering/carrot2/stoplabels.mt b/solr/contrib/clustering/src/test-files/clustering/solr/conf/clustering/carrot2/stoplabels.mt deleted file mode 100644 index 71b2dde..0000000 --- a/solr/contrib/clustering/src/test-files/clustering/solr/conf/clustering/carrot2/stoplabels.mt +++ /dev/null @@ -1 +0,0 @@ -customsolrstoplabel diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/conf/clustering/carrot2/stopwords.mt b/solr/contrib/clustering/src/test-files/clustering/solr/conf/clustering/carrot2/stopwords.mt deleted file mode 100644 index 5dd05b1..0000000 --- a/solr/contrib/clustering/src/test-files/clustering/solr/conf/clustering/carrot2/stopwords.mt +++ /dev/null @@ -1 +0,0 @@ -customsolrstopword diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/conf/clustering/custom/stoplabels.mt b/solr/contrib/clustering/src/test-files/clustering/solr/conf/clustering/custom/stoplabels.mt deleted file mode 100644 index 6ee28d8..0000000 --- a/solr/contrib/clustering/src/test-files/clustering/solr/conf/clustering/custom/stoplabels.mt +++ /dev/null @@ -1 +0,0 @@ -customsolrstoplabelcustomdir diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/conf/clustering/custom/stopwords.mt b/solr/contrib/clustering/src/test-files/clustering/solr/conf/clustering/custom/stopwords.mt deleted file mode 100644 index 6e604da..0000000 --- a/solr/contrib/clustering/src/test-files/clustering/solr/conf/clustering/custom/stopwords.mt +++ /dev/null @@ -1 +0,0 @@ -customsolrstopwordcustomdir diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/conf/mapping-ISOLatin1Accent.txt b/solr/contrib/clustering/src/test-files/clustering/solr/conf/mapping-ISOLatin1Accent.txt deleted file mode 100644 index ede7742..0000000 --- a/solr/contrib/clustering/src/test-files/clustering/solr/conf/mapping-ISOLatin1Accent.txt +++ /dev/null @@ -1,246 +0,0 @@ -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Syntax: -# "source" => "target" -# "source".length() > 0 (source cannot be empty.) -# "target".length() >= 0 (target can be empty.) - -# example: -# "À" => "A" -# "\u00C0" => "A" -# "\u00C0" => "\u0041" -# "ß" => "ss" -# "\t" => " " -# "\n" => "" - -# À => A -"\u00C0" => "A" - -# à => A -"\u00C1" => "A" - -#  => A -"\u00C2" => "A" - -# à => A -"\u00C3" => "A" - -# Ä => A -"\u00C4" => "A" - -# Ã… => A -"\u00C5" => "A" - -# Æ => AE -"\u00C6" => "AE" - -# Ç => C -"\u00C7" => "C" - -# È => E -"\u00C8" => "E" - -# É => E -"\u00C9" => "E" - -# Ê => E -"\u00CA" => "E" - -# Ë => E -"\u00CB" => "E" - -# ÃŒ => I -"\u00CC" => "I" - -# à => I -"\u00CD" => "I" - -# ÃŽ => I -"\u00CE" => "I" - -# à => I -"\u00CF" => "I" - -# IJ => IJ -"\u0132" => "IJ" - -# à => D -"\u00D0" => "D" - -# Ñ => N -"\u00D1" => "N" - -# Ã’ => O -"\u00D2" => "O" - -# Ó => O -"\u00D3" => "O" - -# Ô => O -"\u00D4" => "O" - -# Õ => O -"\u00D5" => "O" - -# Ö => O -"\u00D6" => "O" - -# Ø => O -"\u00D8" => "O" - -# Å’ => OE -"\u0152" => "OE" - -# Þ -"\u00DE" => "TH" - -# Ù => U -"\u00D9" => "U" - -# Ú => U -"\u00DA" => "U" - -# Û => U -"\u00DB" => "U" - -# Ãœ => U -"\u00DC" => "U" - -# à => Y -"\u00DD" => "Y" - -# Ÿ => Y -"\u0178" => "Y" - -# à => a -"\u00E0" => "a" - -# á => a -"\u00E1" => "a" - -# â => a -"\u00E2" => "a" - -# ã => a -"\u00E3" => "a" - -# ä => a -"\u00E4" => "a" - -# Ã¥ => a -"\u00E5" => "a" - -# æ => ae -"\u00E6" => "ae" - -# ç => c -"\u00E7" => "c" - -# è => e -"\u00E8" => "e" - -# é => e -"\u00E9" => "e" - -# ê => e -"\u00EA" => "e" - -# ë => e -"\u00EB" => "e" - -# ì => i -"\u00EC" => "i" - -# í => i -"\u00ED" => "i" - -# î => i -"\u00EE" => "i" - -# ï => i -"\u00EF" => "i" - -# ij => ij -"\u0133" => "ij" - -# ð => d -"\u00F0" => "d" - -# ñ => n -"\u00F1" => "n" - -# ò => o -"\u00F2" => "o" - -# ó => o -"\u00F3" => "o" - -# ô => o -"\u00F4" => "o" - -# õ => o -"\u00F5" => "o" - -# ö => o -"\u00F6" => "o" - -# ø => o -"\u00F8" => "o" - -# Å“ => oe -"\u0153" => "oe" - -# ß => ss -"\u00DF" => "ss" - -# þ => th -"\u00FE" => "th" - -# ù => u -"\u00F9" => "u" - -# ú => u -"\u00FA" => "u" - -# û => u -"\u00FB" => "u" - -# ü => u -"\u00FC" => "u" - -# ý => y -"\u00FD" => "y" - -# ÿ => y -"\u00FF" => "y" - -# ff => ff -"\uFB00" => "ff" - -# ï¬ => fi -"\uFB01" => "fi" - -# fl => fl -"\uFB02" => "fl" - -# ffi => ffi -"\uFB03" => "ffi" - -# ffl => ffl -"\uFB04" => "ffl" - -# ſt => ft -"\uFB05" => "ft" - -# st => st -"\uFB06" => "st" diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/conf/protwords.txt b/solr/contrib/clustering/src/test-files/clustering/solr/conf/protwords.txt deleted file mode 100644 index 1dfc0ab..0000000 --- a/solr/contrib/clustering/src/test-files/clustering/solr/conf/protwords.txt +++ /dev/null @@ -1,21 +0,0 @@ -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#----------------------------------------------------------------------- -# Use a protected word file to protect against the stemmer reducing two -# unrelated words to the same base word. - -# Some non-words that normally won't be encountered, -# just to test that they won't be stemmed. -dontstems -zwhacky - diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/conf/schema.xml b/solr/contrib/clustering/src/test-files/clustering/solr/conf/schema.xml deleted file mode 100644 index 64ca87a..0000000 --- a/solr/contrib/clustering/src/test-files/clustering/solr/conf/schema.xml +++ /dev/null @@ -1,352 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - text - - - - - - - - - - - - - - - - - diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/conf/solrconfig.xml b/solr/contrib/clustering/src/test-files/clustering/solr/conf/solrconfig.xml deleted file mode 100644 index 430c061..0000000 --- a/solr/contrib/clustering/src/test-files/clustering/solr/conf/solrconfig.xml +++ /dev/null @@ -1,476 +0,0 @@ - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - ${solr.data.dir:} - - - - - single - - - - - - - - - - - - - - - - - - - - - - 1024 - - - - - - - - - - - - - true - - - - - - - - 50 - - - 200 - - - - - - - - - solr 0 10 - rocks 0 10 - static newSearcher warming query from solrconfig.xml - - - - - - - fast_warm 0 10 - static firstSearcher warming query from solrconfig.xml - - - - - false - - - 2 - - - - - - - - - - - - - - - - - - - - - - - explicit - - - - clustering - - - - - - - - explicit - - - - doc-clustering - - - - - - - - - - - default - org.carrot2.clustering.lingo.LingoClusteringAlgorithm - - - stc - org.carrot2.clustering.stc.STCClusteringAlgorithm - - - mock - org.apache.solr.handler.clustering.carrot2.MockClusteringAlgorithm - - - echo - org.apache.solr.handler.clustering.carrot2.EchoClusteringAlgorithm - - - lexical-resource-check - org.apache.solr.handler.clustering.carrot2.LexicalResourcesCheckClusteringAlgorithm - - - lexical-resource-check-custom-resource-dir - org.apache.solr.handler.clustering.carrot2.LexicalResourcesCheckClusteringAlgorithm - clustering/custom - - - custom-duplicating-tokenizer - org.apache.solr.handler.clustering.carrot2.EchoTokensClusteringAlgorithm - org.apache.solr.handler.clustering.carrot2.DuplicatingTokenizerFactory - - - custom-duplicating-stemmer - org.apache.solr.handler.clustering.carrot2.EchoStemsClusteringAlgorithm - org.apache.solr.handler.clustering.carrot2.DuplicatingStemmerFactory - - - - - - - - mock - org.apache.solr.handler.clustering.MockDocumentClusteringEngine - - - - - - - - - - - - - - - - - explicit - true - - - - - - - - - 100 - - - - - - - - 70 - - 0.5 - - [-\w ,/\n\"']{20,200} - - - - - - - ]]> - ]]> - - - - - - - - - - 5 - - - - - - - - - - solr - - - diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/conf/spellings.txt b/solr/contrib/clustering/src/test-files/clustering/solr/conf/spellings.txt deleted file mode 100644 index d7ede6f..0000000 --- a/solr/contrib/clustering/src/test-files/clustering/solr/conf/spellings.txt +++ /dev/null @@ -1,2 +0,0 @@ -pizza -history \ No newline at end of file diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/conf/stopwords.txt b/solr/contrib/clustering/src/test-files/clustering/solr/conf/stopwords.txt deleted file mode 100644 index 54f0b99..0000000 --- a/solr/contrib/clustering/src/test-files/clustering/solr/conf/stopwords.txt +++ /dev/null @@ -1,59 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#----------------------------------------------------------------------- -# a couple of test stopwords to test that the words are really being -# configured from this file: -stopworda -stopwordb - -#Standard english stop words taken from Lucene's StopAnalyzer -a -an -and -are -as -at -be -but -by -for -if -in -into -is -it -no -not -of -on -or -s -such -t -that -the -their -then -there -these -they -this -to -was -will -with -solrownstopword - diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/conf/synonyms.txt b/solr/contrib/clustering/src/test-files/clustering/solr/conf/synonyms.txt deleted file mode 100644 index b0e31cb..0000000 --- a/solr/contrib/clustering/src/test-files/clustering/solr/conf/synonyms.txt +++ /dev/null @@ -1,31 +0,0 @@ -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#----------------------------------------------------------------------- -#some test synonym mappings unlikely to appear in real input text -aaa => aaaa -bbb => bbbb1 bbbb2 -ccc => cccc1,cccc2 -a\=>a => b\=>b -a\,a => b\,b -fooaaa,baraaa,bazaaa - -# Some synonym groups specific to this example -GB,gib,gigabyte,gigabytes -MB,mib,megabyte,megabytes -Television, Televisions, TV, TVs -#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming -#after us won't split it into two words. - -# Synonym mappings can be used for spelling correction too -pixima => pixma - diff --git a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/AbstractClusteringTestCase.java b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/AbstractClusteringTestCase.java index d58d89d..50a1067 100644 --- a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/AbstractClusteringTestCase.java +++ b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/AbstractClusteringTestCase.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.clustering; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/ClusteringComponentTest.java b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/ClusteringComponentTest.java index b7524cd..1702901 100644 --- a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/ClusteringComponentTest.java +++ b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/ClusteringComponentTest.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.clustering; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/DistributedClusteringComponentTest.java b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/DistributedClusteringComponentTest.java index 41ccf23..69818ef 100644 --- a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/DistributedClusteringComponentTest.java +++ b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/DistributedClusteringComponentTest.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.clustering; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -25,7 +25,7 @@ public class DistributedClusteringComponentTest extends @Override public String getSolrHome() { - return "clustering/solr"; + return getFile("clustering/solr/collection1").getParent(); } @Override diff --git a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/MockDocumentClusteringEngine.java b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/MockDocumentClusteringEngine.java index 77b3fcf..135126b 100644 --- a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/MockDocumentClusteringEngine.java +++ b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/MockDocumentClusteringEngine.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.clustering; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java index 9450235..6691f86 100644 --- a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java +++ b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.clustering.carrot2; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/DuplicatingStemmerFactory.java b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/DuplicatingStemmerFactory.java index c090a15..baf208b 100644 --- a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/DuplicatingStemmerFactory.java +++ b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/DuplicatingStemmerFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.clustering.carrot2; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/DuplicatingTokenizerFactory.java b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/DuplicatingTokenizerFactory.java index 99e6b60..2d0e584 100644 --- a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/DuplicatingTokenizerFactory.java +++ b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/DuplicatingTokenizerFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.clustering.carrot2; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -39,7 +39,7 @@ public class DuplicatingTokenizerFactory implements ITokenizerFactory { } @Override - public void reset(Reader input) throws IOException { + public void reset(Reader input) { delegate.reset(input); } diff --git a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoClusteringAlgorithm.java b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoClusteringAlgorithm.java index d4b386c..9bf68ae 100644 --- a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoClusteringAlgorithm.java +++ b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoClusteringAlgorithm.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.clustering.carrot2; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoStemsClusteringAlgorithm.java b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoStemsClusteringAlgorithm.java index 52d8e05..9827eb7 100644 --- a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoStemsClusteringAlgorithm.java +++ b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoStemsClusteringAlgorithm.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.clustering.carrot2; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoTokensClusteringAlgorithm.java b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoTokensClusteringAlgorithm.java index 2ed2d95..f23004e 100644 --- a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoTokensClusteringAlgorithm.java +++ b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoTokensClusteringAlgorithm.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.clustering.carrot2; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/LexicalResourcesCheckClusteringAlgorithm.java b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/LexicalResourcesCheckClusteringAlgorithm.java index 4810f7a..ebac634 100644 --- a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/LexicalResourcesCheckClusteringAlgorithm.java +++ b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/LexicalResourcesCheckClusteringAlgorithm.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.clustering.carrot2; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/MockClusteringAlgorithm.java b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/MockClusteringAlgorithm.java index 4618596..3d4fd95 100644 --- a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/MockClusteringAlgorithm.java +++ b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/MockClusteringAlgorithm.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.clustering.carrot2; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java b/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java index e408a67..fafb00f 100644 --- a/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java +++ b/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -82,7 +82,7 @@ public class MailEntityProcessor extends EntityProcessorBase { } batchSize = getIntFromContext("batchSize", 20); customFilter = getStringFromContext("customFilter", ""); - String s = getStringFromContext("fetchMailsSince", ""); + String s = getStringFromContext("fetchMailsSince", null); if (s != null) try { fetchMailsSince = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse(s); diff --git a/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java b/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java index d07d179..adeac14 100644 --- a/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java +++ b/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/collection1/conf/dataimport-schema-no-unique-key.xml b/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/collection1/conf/dataimport-schema-no-unique-key.xml new file mode 100644 index 0000000..e502f74 --- /dev/null +++ b/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/collection1/conf/dataimport-schema-no-unique-key.xml @@ -0,0 +1,205 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text + + + + + diff --git a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/collection1/conf/dataimport-solrconfig.xml b/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/collection1/conf/dataimport-solrconfig.xml new file mode 100644 index 0000000..1b263dd --- /dev/null +++ b/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/collection1/conf/dataimport-solrconfig.xml @@ -0,0 +1,311 @@ + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + ${solr.data.dir:} + + + + + + + + + + + + 100000 + + + + + + + 1024 + + + + + + + + + + + + + true + + + + + + + + 50 + + + 200 + + + + + + + + + solr 0 10 + rocks 0 10 + static newSearcher warming query from solrconfig.xml + + + + + + + + + + + false + + + 4 + + + + + + + + + + + + + + + + + + + + + + + explicit + + + + + + + + + + + + explicit + + + + + + + + + + + + *:* + + + + diff --git a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/dataimport-schema-no-unique-key.xml b/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/dataimport-schema-no-unique-key.xml deleted file mode 100644 index e502f74..0000000 --- a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/dataimport-schema-no-unique-key.xml +++ /dev/null @@ -1,205 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text - - - - - diff --git a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/dataimport-solrconfig.xml b/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/dataimport-solrconfig.xml deleted file mode 100644 index ca6186c..0000000 --- a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/dataimport-solrconfig.xml +++ /dev/null @@ -1,311 +0,0 @@ - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - ${solr.data.dir:} - - - - - - - - - - - - 100000 - - - - - - - 1024 - - - - - - - - - - - - - true - - - - - - - - 50 - - - 200 - - - - - - - - - solr 0 10 - rocks 0 10 - static newSearcher warming query from solrconfig.xml - - - - - - - - - - - false - - - 4 - - - - - - - - - - - - - - - - - - - - - - - explicit - - - - - - - - - - - - explicit - - - - - - - - - - - - *:* - - - - diff --git a/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java b/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java index 70e1c2a..3f356fb 100644 --- a/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java +++ b/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java b/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java index 7cc8b30..9b5ac0e 100644 --- a/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java +++ b/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/CHANGES.txt b/solr/contrib/dataimporthandler/CHANGES.txt index 1fac866..466ff49 100644 --- a/solr/contrib/dataimporthandler/CHANGES.txt +++ b/solr/contrib/dataimporthandler/CHANGES.txt @@ -8,11 +8,18 @@ HTTP data sources quick and easy. $Id$ -================== 4.0.0-dev ============== +================== 5.0.0 ============== + + (No changes) + +================== 4.0.0-ALPHA ============== Bug Fixes ---------------------- * SOLR-3430: Added a new test against a real SQL database. Fixed problems revealed by this new test related to the expanded cache support added to 3.6/SOLR-2382 (James Dyer) + +* SOLR-1958: When using the MailEntityProcessor, import would fail if fetchMailsSince was not specified. + (Max Lynch via James Dyer) Other Changes ---------------------- diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinContentStreamDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinContentStreamDataSource.java index 06b169c..1cbac64 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinContentStreamDataSource.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinContentStreamDataSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinFileDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinFileDataSource.java index 9559c06..3ebf35a 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinFileDataSource.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinFileDataSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinURLDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinURLDataSource.java index aa564f4..b7d7b73 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinURLDataSource.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinURLDataSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/CachePropertyUtil.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/CachePropertyUtil.java index 18fc519..ee9b1f1 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/CachePropertyUtil.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/CachePropertyUtil.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.dataimport; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/CachedSqlEntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/CachedSqlEntityProcessor.java index b67a68c..56c464c 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/CachedSqlEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/CachedSqlEntityProcessor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ClobTransformer.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ClobTransformer.java index 45aff14..690e9db 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ClobTransformer.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ClobTransformer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ConfigParseUtil.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ConfigParseUtil.java index 4e56a8b..70a4df9 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ConfigParseUtil.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ConfigParseUtil.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContentStreamDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContentStreamDataSource.java index 5bb3cdd..015b7f4 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContentStreamDataSource.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContentStreamDataSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Context.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Context.java index 0dcd7f3..83d2670 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Context.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Context.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContextImpl.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContextImpl.java index a318647..c77a75b 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContextImpl.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContextImpl.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.dataimport; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCacheSupport.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCacheSupport.java index 05da675..82285ab 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCacheSupport.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCacheSupport.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.dataimport; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHLogLevels.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHLogLevels.java index 00fca8a..833d750 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHLogLevels.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHLogLevels.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.dataimport; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHPropertiesWriter.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHPropertiesWriter.java index 7971d74..5bdfc2e 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHPropertiesWriter.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHPropertiesWriter.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.dataimport; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriter.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriter.java index 9140730..774b350 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriter.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriter.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.dataimport; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriterBase.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriterBase.java index 399cc6a..12cd563 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriterBase.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriterBase.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.dataimport; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java index e0760a5..b082a01 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java index 49d47cd..81b49c2 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java index 6561862..b21c40e 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataSource.java index 1f805d9..37ad464 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataSource.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java index f093f97..a2d3e91 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DebugInfo.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DebugInfo.java index 8af586d..b35fe91 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DebugInfo.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DebugInfo.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DebugLogger.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DebugLogger.java index 06ee91a..d3bad1f 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DebugLogger.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DebugLogger.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.dataimport; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java index 3331f91..7d904e3 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessor.java index f285be3..81f68c4 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java index 591647a..330c58f 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java index a78b286..f514138 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Evaluator.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Evaluator.java index 3393ad4..edf7534 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Evaluator.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Evaluator.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EvaluatorBag.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EvaluatorBag.java index b0ff86b..f477a37 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EvaluatorBag.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EvaluatorBag.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.dataimport; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EventListener.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EventListener.java index b3cbee2..0f22513 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EventListener.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EventListener.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java index 0fa0ddb..6f8ebb7 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldStreamDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldStreamDataSource.java index adbcc4d..bb5d4e8 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldStreamDataSource.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldStreamDataSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileDataSource.java index ca37c73..a153469 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileDataSource.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileDataSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java index 520943f..fd9a207 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java index 526976c..0160f43 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/HttpDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/HttpDataSource.java index a658310..04633f3 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/HttpDataSource.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/HttpDataSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java index df4b33f..83f43db 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/LineEntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/LineEntityProcessor.java index e526e83..00d8136 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/LineEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/LineEntityProcessor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/LogTransformer.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/LogTransformer.java index 704a03c..f0a1b97 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/LogTransformer.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/LogTransformer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/MockDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/MockDataSource.java index 32048ea..469f582 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/MockDataSource.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/MockDataSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java index 754e39b..91e8996 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.dataimport; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/PlainTextEntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/PlainTextEntityProcessor.java index b88c85f..857865c 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/PlainTextEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/PlainTextEntityProcessor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RegexTransformer.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RegexTransformer.java index d680c9d..babd3e3 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RegexTransformer.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RegexTransformer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RequestInfo.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RequestInfo.java index 550995a..864e545 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RequestInfo.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RequestInfo.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ScriptTransformer.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ScriptTransformer.java index 46c6407..5dc058a 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ScriptTransformer.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ScriptTransformer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java index b9ab396..decbabc 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.dataimport; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java index 26e6399..0f7824a 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java @@ -18,11 +18,10 @@ package org.apache.solr.handler.dataimport; */ import org.apache.http.client.HttpClient; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.client.solrj.impl.XMLResponseParser; import org.apache.solr.client.solrj.response.QueryResponse; @@ -70,7 +69,7 @@ public class SolrEntityProcessor extends EntityProcessorBase { private int rows = ROWS_DEFAULT; private String[] filterQueries; private String[] fields; - private String queryType; + private String requestHandler;// 'qt' param private int timeout = TIMEOUT_SECS; private boolean initDone = false; @@ -83,7 +82,7 @@ public class SolrEntityProcessor extends EntityProcessorBase { * @return a {@link HttpClient} instance used for interfacing with a source Solr service */ protected HttpClient getHttpClient() { - return new DefaultHttpClient(new ThreadSafeClientConnManager()); + return HttpClientUtil.createClient(null); } @Override @@ -170,7 +169,7 @@ public class SolrEntityProcessor extends EntityProcessorBase { if (fieldsAsString != null) { this.fields = fieldsAsString.split(","); } - this.queryType = context.getResolvedEntityAttribute(CommonParams.QT); + this.requestHandler = context.getResolvedEntityAttribute(CommonParams.QT); String timeoutAsString = context.getResolvedEntityAttribute(TIMEOUT); if (timeoutAsString != null) { this.timeout = Integer.parseInt(timeoutAsString); @@ -184,7 +183,7 @@ public class SolrEntityProcessor extends EntityProcessorBase { solrQuery.addField(field); } } - solrQuery.setQueryType(queryType); + solrQuery.setRequestHandler(requestHandler); solrQuery.setFilterQueries(filterQueries); solrQuery.setTimeAllowed(timeout * 1000); diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java index 627009d..7df6e26 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SortedMapBackedCache.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SortedMapBackedCache.java index dc5e02a..46d6288 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SortedMapBackedCache.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SortedMapBackedCache.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.dataimport; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SqlEntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SqlEntityProcessor.java index 5f77063..864a137 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SqlEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SqlEntityProcessor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/TemplateString.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/TemplateString.java index 7724de8..77128dc 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/TemplateString.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/TemplateString.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/TemplateTransformer.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/TemplateTransformer.java index 97a4b11..b883436 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/TemplateTransformer.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/TemplateTransformer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Transformer.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Transformer.java index 758b243..4bbde75 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Transformer.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Transformer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/URLDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/URLDataSource.java index 626040b..f462466 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/URLDataSource.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/URLDataSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java index 8a91f0f..b3ded90 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolverImpl.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolverImpl.java index c4dffa6..cdd0ba7 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolverImpl.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolverImpl.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java index 70bfae7..44bfe5a 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathRecordReader.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathRecordReader.java index ff5eb5f..9b2aa97 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathRecordReader.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathRecordReader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java index 5ab2477..ddec036 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigNameConstants.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigNameConstants.java index dcf1a08..c92feef 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigNameConstants.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigNameConstants.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigParseUtil.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigParseUtil.java index fb1c305..71c45b1 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigParseUtil.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigParseUtil.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java index 885558d..d7f3d9d 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java @@ -9,7 +9,7 @@ import java.util.Properties; import org.apache.solr.handler.dataimport.DataImporter; import org.w3c.dom.Element; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Document.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Document.java index 8be2dba..05b0d0a 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Document.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Document.java @@ -9,7 +9,7 @@ import java.util.Properties; import org.apache.solr.handler.dataimport.DataImporter; import org.w3c.dom.Element; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Entity.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Entity.java index 7ed0b0f..b002abe 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Entity.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Entity.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/EntityField.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/EntityField.java index 878c74b..806150c 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/EntityField.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/EntityField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Field.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Field.java index 3296474..da0851d 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Field.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Field.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Script.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Script.java index 26e109e..5ec59a2 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Script.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Script.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/contentstream-solrconfig.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/contentstream-solrconfig.xml new file mode 100644 index 0000000..3a8b15d --- /dev/null +++ b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/contentstream-solrconfig.xml @@ -0,0 +1,321 @@ + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + ${solr.data.dir:} + + + + + + + + + + + + 100000 + + + + + + + 1024 + + + + + + + + + + + + + true + + + + + + + + 50 + + + 200 + + + + + + + + + solr 0 10 + rocks 0 10 + static newSearcher warming query from solrconfig.xml + + + + + + + + + + + false + + + 4 + + + + + + + + + + + + + + + + + + + + + + + explicit + + + + + + + data-config.xml + + + + + + + + + explicit + + + + + + + + + + + + *:* + + + + + + + + + + diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/data-config-end-to-end.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/data-config-end-to-end.xml new file mode 100644 index 0000000..6dc7e33 --- /dev/null +++ b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/data-config-end-to-end.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/data-config-with-datasource.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/data-config-with-datasource.xml new file mode 100644 index 0000000..9566a54 --- /dev/null +++ b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/data-config-with-datasource.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/data-config-with-transformer.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/data-config-with-transformer.xml new file mode 100644 index 0000000..c58b21d --- /dev/null +++ b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/data-config-with-transformer.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataconfig-contentstream.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataconfig-contentstream.xml new file mode 100644 index 0000000..7520e74 --- /dev/null +++ b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataconfig-contentstream.xml @@ -0,0 +1,10 @@ + + + + + + + + + diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-nodatasource-solrconfig.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-nodatasource-solrconfig.xml new file mode 100644 index 0000000..796f831 --- /dev/null +++ b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-nodatasource-solrconfig.xml @@ -0,0 +1,313 @@ + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + ${solr.data.dir:} + + + + + single + + + + + + + + + 100000 + + + + + + + 1024 + + + + + + + + + + + + + true + + + + + + + + 50 + + + 200 + + + + + + + + + solr 0 10 + rocks 0 10 + static newSearcher warming query from solrconfig.xml + + + + + + + + + + + false + + + 4 + + + + + + + + + + + + + + + + + + + + + + + explicit + + + + + + + + + + + + explicit + + + + + + + + + + + + *:* + + + + diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-schema.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-schema.xml new file mode 100644 index 0000000..2bf706b --- /dev/null +++ b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-schema.xml @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + desc + + \ No newline at end of file diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solr_id-schema.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solr_id-schema.xml new file mode 100644 index 0000000..33e4bfc --- /dev/null +++ b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solr_id-schema.xml @@ -0,0 +1,307 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + solr_id + + + desc + + + + + diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig-end-to-end.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig-end-to-end.xml new file mode 100644 index 0000000..27e54be --- /dev/null +++ b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig-end-to-end.xml @@ -0,0 +1,45 @@ + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + 100000 + + + + + + + + explicit + + + + + data-config-end-to-end.xml + + + + + explicit + + + + diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig.xml new file mode 100644 index 0000000..16d4575 --- /dev/null +++ b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig.xml @@ -0,0 +1,317 @@ + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + ${solr.data.dir:} + + + + + + + + + + + + 100000 + + + + + + + 1024 + + + + + + + + + + + + + true + + + + + + + + 50 + + + 200 + + + + + + + + + solr 0 10 + rocks 0 10 + static newSearcher warming query from solrconfig.xml + + + + + + + + + + + false + + + 4 + + + + + + + + + + + + + + + + + + + + + + + explicit + + + + + + + + + + + + explicit + + + + + + + + + + + + *:* + + + + + + + + + + diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/protwords.txt b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/protwords.txt new file mode 100644 index 0000000..7878147 --- /dev/null +++ b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/protwords.txt @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#use a protected word file to avoid stemming two +#unrelated words to the same base word. +#to test, we will use words that would normally obviously be stemmed. +cats +ridding diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/single-entity-data-config.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/single-entity-data-config.xml new file mode 100644 index 0000000..f9d3523 --- /dev/null +++ b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/single-entity-data-config.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/stopwords.txt b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/stopwords.txt new file mode 100644 index 0000000..688e307 --- /dev/null +++ b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/stopwords.txt @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +stopworda +stopwordb diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/synonyms.txt b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/synonyms.txt new file mode 100644 index 0000000..a7624f0 --- /dev/null +++ b/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/synonyms.txt @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +a => aa +b => b1 b2 +c => c1,c2 +a\=>a => b\=>b +a\,a => b\,b +foo,bar,baz + +Television,TV,Televisions diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/contentstream-solrconfig.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/contentstream-solrconfig.xml deleted file mode 100644 index d5df80a..0000000 --- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/contentstream-solrconfig.xml +++ /dev/null @@ -1,321 +0,0 @@ - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - ${solr.data.dir:} - - - - - - - - - - - - 100000 - - - - - - - 1024 - - - - - - - - - - - - - true - - - - - - - - 50 - - - 200 - - - - - - - - - solr 0 10 - rocks 0 10 - static newSearcher warming query from solrconfig.xml - - - - - - - - - - - false - - - 4 - - - - - - - - - - - - - - - - - - - - - - - explicit - - - - - - - data-config.xml - - - - - - - - - explicit - - - - - - - - - - - - *:* - - - - - - - - - - diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/data-config-end-to-end.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/data-config-end-to-end.xml deleted file mode 100644 index 6dc7e33..0000000 --- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/data-config-end-to-end.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/data-config-with-datasource.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/data-config-with-datasource.xml deleted file mode 100644 index 9566a54..0000000 --- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/data-config-with-datasource.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/data-config-with-transformer.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/data-config-with-transformer.xml deleted file mode 100644 index c58b21d..0000000 --- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/data-config-with-transformer.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataconfig-contentstream.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataconfig-contentstream.xml deleted file mode 100644 index 7520e74..0000000 --- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataconfig-contentstream.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-nodatasource-solrconfig.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-nodatasource-solrconfig.xml deleted file mode 100644 index 925283c..0000000 --- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-nodatasource-solrconfig.xml +++ /dev/null @@ -1,313 +0,0 @@ - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - ${solr.data.dir:} - - - - - single - - - - - - - - - 100000 - - - - - - - 1024 - - - - - - - - - - - - - true - - - - - - - - 50 - - - 200 - - - - - - - - - solr 0 10 - rocks 0 10 - static newSearcher warming query from solrconfig.xml - - - - - - - - - - - false - - - 4 - - - - - - - - - - - - - - - - - - - - - - - explicit - - - - - - - - - - - - explicit - - - - - - - - - - - - *:* - - - - diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-schema.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-schema.xml deleted file mode 100644 index 2bf706b..0000000 --- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-schema.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - desc - - \ No newline at end of file diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-solr_id-schema.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-solr_id-schema.xml deleted file mode 100644 index 33e4bfc..0000000 --- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-solr_id-schema.xml +++ /dev/null @@ -1,307 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - solr_id - - - desc - - - - - diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-solrconfig-end-to-end.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-solrconfig-end-to-end.xml deleted file mode 100644 index 27e54be..0000000 --- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-solrconfig-end-to-end.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - 100000 - - - - - - - - explicit - - - - - data-config-end-to-end.xml - - - - - explicit - - - - diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-solrconfig.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-solrconfig.xml deleted file mode 100644 index 5d54c51..0000000 --- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-solrconfig.xml +++ /dev/null @@ -1,317 +0,0 @@ - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - ${solr.data.dir:} - - - - - - - - - - - - 100000 - - - - - - - 1024 - - - - - - - - - - - - - true - - - - - - - - 50 - - - 200 - - - - - - - - - solr 0 10 - rocks 0 10 - static newSearcher warming query from solrconfig.xml - - - - - - - - - - - false - - - 4 - - - - - - - - - - - - - - - - - - - - - - - explicit - - - - - - - - - - - - explicit - - - - - - - - - - - - *:* - - - - - - - - - - diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/protwords.txt b/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/protwords.txt deleted file mode 100644 index 7878147..0000000 --- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/protwords.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#use a protected word file to avoid stemming two -#unrelated words to the same base word. -#to test, we will use words that would normally obviously be stemmed. -cats -ridding diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/single-entity-data-config.xml b/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/single-entity-data-config.xml deleted file mode 100644 index f9d3523..0000000 --- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/single-entity-data-config.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/stopwords.txt b/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/stopwords.txt deleted file mode 100644 index 688e307..0000000 --- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/stopwords.txt +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -stopworda -stopwordb diff --git a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/synonyms.txt b/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/synonyms.txt deleted file mode 100644 index a7624f0..0000000 --- a/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/synonyms.txt +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -a => aa -b => b1 b2 -c => c1,c2 -a\=>a => b\=>b -a\,a => b\,b -foo,bar,baz - -Television,TV,Televisions diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java index 83941e5..23682d8 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.dataimport; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java index 5793d79..e33c630 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.dataimport; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java index 50ade67..bcbf6aa 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -68,7 +68,7 @@ public abstract class AbstractDataImportHandlerTestCase extends @After public void tearDown() throws Exception { // remove dataimport.properties - File f = new File("solr/conf/dataimport.properties"); + File f = new File("solr/collection1/conf/dataimport.properties"); log.info("Looking for dataimport.properties at: " + f.getAbsolutePath()); if (f.exists()) { log.info("Deleting dataimport.properties"); diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/DestroyCountCache.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/DestroyCountCache.java index 223a1cc..8b83f5f 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/DestroyCountCache.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/DestroyCountCache.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.dataimport; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockInitialContextFactory.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockInitialContextFactory.java index e4ff3fc..f33b9e9 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockInitialContextFactory.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockInitialContextFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestCachedSqlEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestCachedSqlEntityProcessor.java index d9a314d..9119e58 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestCachedSqlEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestCachedSqlEntityProcessor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestClobTransformer.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestClobTransformer.java index b1eb394..7ec575c 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestClobTransformer.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestClobTransformer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java index c41a435..a9308d4 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -40,7 +40,7 @@ import java.util.List; * @since solr 1.4 */ public class TestContentStreamDataSource extends AbstractDataImportHandlerTestCase { - private static final String CONF_DIR = "dih/solr/conf/"; + private static final String CONF_DIR = "dih/solr/collection1/conf/"; SolrInstance instance = null; JettySolrRunner jetty; @@ -151,8 +151,8 @@ public class TestContentStreamDataSource extends AbstractDataImportHandlerTestCa homeDir = new File(home, "inst"); - dataDir = new File(homeDir, "data"); - confDir = new File(homeDir, "conf"); + dataDir = new File(homeDir + "/collection1", "data"); + confDir = new File(homeDir + "/collection1", "conf"); homeDir.mkdirs(); dataDir.mkdirs(); diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContextImpl.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContextImpl.java index e47b512..2870d93 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContextImpl.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContextImpl.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDIHEndToEnd.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDIHEndToEnd.java index 5952b5d..8a1da70 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDIHEndToEnd.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDIHEndToEnd.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.dataimport; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDataConfig.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDataConfig.java index da0fe28..05ee93b 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDataConfig.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDataConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java index 606d573..29bf2f5 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java index 64eee7b..a3a751b 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java index d6c7b8b..5509dd3 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java index ba51935..bf16bbc 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEphemeralCache.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEphemeralCache.java index ce71937..b107926 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEphemeralCache.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEphemeralCache.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.dataimport; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestErrorHandling.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestErrorHandling.java index bbd506e..5edc9d8 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestErrorHandling.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestErrorHandling.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.dataimport; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -46,7 +46,7 @@ public class TestErrorHandling extends AbstractDataImportHandlerTestCase { } @AfterClass - public static void afterClass() throws Exception { + public static void afterClass() { if (savedFactory == null) { System.clearProperty("solr.directoryFactory"); } else { diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEvaluatorBag.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEvaluatorBag.java index 6c4fd77..ec05696 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEvaluatorBag.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEvaluatorBag.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java index 3e0fc43..d328ab7 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java index cb2bc8a..b5c4e18 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.dataimport; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestJdbcDataSource.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestJdbcDataSource.java index 239e148..10399d4 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestJdbcDataSource.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestJdbcDataSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestLineEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestLineEntityProcessor.java index 5548924..920ca4a 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestLineEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestLineEntityProcessor.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.dataimport; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNumberFormatTransformer.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNumberFormatTransformer.java index b799717..d3dbc4d 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNumberFormatTransformer.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNumberFormatTransformer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java index f27137c..2af1b05 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestRegexTransformer.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestRegexTransformer.java index 50835d7..4eb2ef4 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestRegexTransformer.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestRegexTransformer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestScriptTransformer.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestScriptTransformer.java index b92ac81..79aa3c2 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestScriptTransformer.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestScriptTransformer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java index ed515e7..36cb3f6 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java @@ -27,10 +27,10 @@ import java.util.Map; import java.util.Map.Entry; import org.apache.commons.io.FileUtils; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; +import org.apache.http.client.HttpClient; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.common.SolrInputDocument; import org.junit.After; @@ -51,7 +51,7 @@ public class TestSolrEntityProcessorEndToEnd extends AbstractDataImportHandlerTe private static final String SOLR_CONFIG = "dataimport-solrconfig.xml"; private static final String SOLR_SCHEMA = "dataimport-schema.xml"; private static final String SOLR_HOME = "dih/solr"; - private static final String CONF_DIR = "dih" + File.separator + "solr" + File.separator + "conf" + File.separator; + private static final String CONF_DIR = "dih" + File.separator + "solr" + File.separator + "collection1" + File.separator + "conf" + File.separator; private static final List> DB_DOCS = new ArrayList>(); private static final List> SOLR_DOCS = new ArrayList>(); @@ -101,13 +101,13 @@ public class TestSolrEntityProcessorEndToEnd extends AbstractDataImportHandlerTe //TODO: fix this test to close its directories static String savedFactory; @BeforeClass - public static void beforeClass() throws Exception { + public static void beforeClass() { savedFactory = System.getProperty("solr.DirectoryFactory"); System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory"); } @AfterClass - public static void afterClass() throws Exception { + public static void afterClass() { if (savedFactory == null) { System.clearProperty("solr.directoryFactory"); } else { @@ -274,7 +274,7 @@ public class TestSolrEntityProcessorEndToEnd extends AbstractDataImportHandlerTe sidl.add(sd); } - DefaultHttpClient client = new DefaultHttpClient(new ThreadSafeClientConnManager()); + HttpClient client = HttpClientUtil.createClient(null); URL url = new URL(getSourceUrl(jetty.getLocalPort())); HttpSolrServer solrServer = new HttpSolrServer(url.toExternalForm(), client); solrServer.add(sidl); @@ -308,8 +308,8 @@ public class TestSolrEntityProcessorEndToEnd extends AbstractDataImportHandlerTe + System.currentTimeMillis()); homeDir = new File(home + "inst"); - dataDir = new File(homeDir, "data"); - confDir = new File(homeDir, "conf"); + dataDir = new File(homeDir + "/collection1", "data"); + confDir = new File(homeDir + "/collection1", "conf"); homeDir.mkdirs(); dataDir.mkdirs(); diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java index 0a1ee19..afd9450 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSortedMapBackedCache.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSortedMapBackedCache.java index 6014fa7..c02ebbf 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSortedMapBackedCache.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSortedMapBackedCache.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.dataimport; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java index 931d633..66cfd11 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor2.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor2.java index b46702c..cc21639 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor2.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor2.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java index b7caea7..c232ea6 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,7 +22,6 @@ import org.junit.BeforeClass; import org.junit.Test; import java.io.File; -import java.io.FileOutputStream; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -70,7 +69,7 @@ public class TestSqlEntityProcessorDelta extends AbstractDataImportHandlerTestCa } @AfterClass - public static void afterClass() throws Exception { + public static void afterClass() { if (savedFactory == null) { System.clearProperty("solr.directoryFactory"); } else { diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta2.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta2.java index c3ce382..6fd7ffb 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta2.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta2.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta3.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta3.java index 3e9694e..2d6f638 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta3.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta3.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDeltaPrefixedPk.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDeltaPrefixedPk.java index 2cc40a4..b5088a7 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDeltaPrefixedPk.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDeltaPrefixedPk.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestTemplateString.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestTemplateString.java index 5334ec1..5c47ca7 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestTemplateString.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestTemplateString.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestTemplateTransformer.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestTemplateTransformer.java index b1fb53c..48fdeb0 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestTemplateTransformer.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestTemplateTransformer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestURLDataSource.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestURLDataSource.java index 6f720e7..ca14f94 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestURLDataSource.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestURLDataSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolver.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolver.java index d3fcae2..926f559 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolver.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java index 61644df..2f7bea1 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathRecordReader.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathRecordReader.java index 619e45b..1f7290c 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathRecordReader.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathRecordReader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/extraction/CHANGES.txt b/solr/contrib/extraction/CHANGES.txt index 7434d9d..fd20533 100644 --- a/solr/contrib/extraction/CHANGES.txt +++ b/solr/contrib/extraction/CHANGES.txt @@ -24,7 +24,11 @@ Current Version: Tika 1.1 (released 2012-03-23) $Id$ -================== Release 4.0.0-dev ============== +================== Release 5.0.0 ============== + + (No changes) + +================== Release 4.0.0-ALPHA ============== * SOLR-3254: Upgrade Solr to Tika 1.1 (janhoy) diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java index 1fef2e5..526d5b9 100644 --- a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java +++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -44,6 +44,7 @@ import org.apache.tika.parser.AutoDetectParser; import org.apache.tika.parser.DefaultParser; import org.apache.tika.parser.ParseContext; import org.apache.tika.parser.Parser; +import org.apache.tika.parser.PasswordProvider; import org.apache.tika.sax.XHTMLContentHandler; import org.apache.tika.sax.xpath.Matcher; import org.apache.tika.sax.xpath.MatchingContentHandler; @@ -90,7 +91,6 @@ public class ExtractingDocumentLoader extends ContentStreamLoader { protected TikaConfig config; protected SolrContentHandlerFactory factory; - //protected Collection dateFormats = DateUtil.DEFAULT_DATE_FORMATS; public ExtractingDocumentLoader(SolrQueryRequest req, UpdateRequestProcessor processor, TikaConfig config, SolrContentHandlerFactory factory) { @@ -114,8 +114,6 @@ public class ExtractingDocumentLoader extends ContentStreamLoader { /** * this must be MT safe... may be called concurrently from multiple threads. * - * @param - * @param */ void doAdd(SolrContentHandler handler, AddUpdateCommand template) throws IOException { @@ -206,6 +204,23 @@ public class ExtractingDocumentLoader extends ContentStreamLoader { try{ //potentially use a wrapper handler for parsing, but we still need the SolrContentHandler for getting the document. ParseContext context = new ParseContext();//TODO: should we design a way to pass in parse context? + + // Password handling + RegexRulesPasswordProvider epp = new RegexRulesPasswordProvider(); + String pwMapFile = params.get(ExtractingParams.PASSWORD_MAP_FILE); + if(pwMapFile != null && pwMapFile.length() > 0) { + InputStream is = req.getCore().getResourceLoader().openResource(pwMapFile); + if(is != null) { + log.debug("Password file supplied: "+pwMapFile); + epp.parse(is); + } + } + context.set(PasswordProvider.class, epp); + String resourcePassword = params.get(ExtractingParams.RESOURCE_PASSWORD); + if(resourcePassword != null) { + epp.setExplicitPassword(resourcePassword); + log.debug("Literal password supplied for file "+resourceName); + } parser.parse(inputStream, parsingHandler, metadata, context); } catch (TikaException e) { if(ignoreTikaException) diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingMetadataConstants.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingMetadataConstants.java index 474e16c..71aced1 100644 --- a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingMetadataConstants.java +++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingMetadataConstants.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java index 565becf..a229b52 100644 --- a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java +++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -96,6 +96,10 @@ public interface ExtractingParams { */ public static final String CAPTURE_ATTRIBUTES = "captureAttr"; + /** + * Literal field values will by default override other values such as metadata and content. Set this to false to revert to pre-4.0 behaviour + */ + public static final String LITERALS_OVERRIDE = "literalsOverride"; /** * Capture the specified fields (and everything included below it that isn't capture by some other capture field) separately from the default. This is different @@ -132,6 +136,10 @@ public interface ExtractingParams { */ public static final String RESOURCE_NAME = "resource.name"; + /** + * Optional. The password for this resource. Will be used instead of the rule based password lookup mechanisms + */ + public static final String RESOURCE_PASSWORD = "resource.password"; /** * Optional. If specified, the prefix will be prepended to all Metadata, such that it would be possible @@ -144,4 +152,14 @@ public interface ExtractingParams { * will be used instead. */ public static final String DEFAULT_FIELD = "defaultField"; + + /** + * Optional. If specified, loads the file as a source for password lookups for Tika encrypted documents. + *

      + * File format is Java properties format with one key=value per line. + * The key is evaluated as a regex against the file name, and the value is the password + * The rules are evaluated top-bottom, i.e. the first match will be used + * If you want a fallback password to be always used, supply a .*= at the end + */ + public static final String PASSWORD_MAP_FILE = "passwordsFile"; } diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java index c6c5c2f..cfee4d9 100644 --- a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java +++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/RegexRulesPasswordProvider.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/RegexRulesPasswordProvider.java new file mode 100644 index 0000000..1ab323b --- /dev/null +++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/RegexRulesPasswordProvider.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.extraction; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.LinkedHashMap; +import java.util.Map.Entry; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaMetadataKeys; +import org.apache.tika.parser.PasswordProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Password provider for Extracting request handler which finds correct + * password based on file name matching against a list of regular expressions. + * The list of passwords is supplied in an optional Map. + * If an explicit password is set, it will be used. + */ +public class RegexRulesPasswordProvider implements PasswordProvider { + private static final Logger log = LoggerFactory.getLogger(RegexRulesPasswordProvider.class); + + private LinkedHashMap passwordMap = new LinkedHashMap(); + private String explicitPassword; + + @Override + public String getPassword(Metadata meta) { + if(getExplicitPassword() != null) { + return getExplicitPassword(); + } + + if(passwordMap.size() > 0) + return lookupPasswordFromMap(meta.get(TikaMetadataKeys.RESOURCE_NAME_KEY)); + + return null; + } + + private String lookupPasswordFromMap(String fileName) { + if(fileName != null && fileName.length() > 0) { + for(Entry e : passwordMap.entrySet()) { + if(e.getKey().matcher(fileName).matches()) { + return e.getValue(); + } + } + } + return null; + } + + /** + * Parses rule file from stream and returns a Map of all rules found + * @param is input stream for the file + */ + public static LinkedHashMap parseRulesFile(InputStream is) { + LinkedHashMap rules = new LinkedHashMap(); + BufferedReader br = new BufferedReader(new InputStreamReader(is)); + String line; + try { + int linenum = 0; + while ((line = br.readLine()) != null) { + linenum++; + // Remove comments + String[] arr = line.split("#"); + if(arr.length > 0) + line = arr[0].trim(); + if(line.length() == 0) + continue; + int sep = line.indexOf("="); + if(sep <= 0) { + log.warn("Wrong format of password line "+linenum); + continue; + } + String pass = line.substring(sep+1).trim(); + String regex = line.substring(0, sep).trim(); + try { + Pattern pattern = Pattern.compile(regex); + rules.put(pattern, pass); + } catch(PatternSyntaxException pse) { + log.warn("Key of line "+linenum+" was not a valid regex pattern", pse); + continue; + } + } + is.close(); + } catch (IOException e) { + throw new RuntimeException(); + } + return rules; + } + + /** + * Initialize rules through file input stream. This is a convenience for first calling + * setPasswordMap(parseRulesFile(is)). + * @param is the input stream with rules file, one line per rule on format regex=password + */ + public void parse(InputStream is) { + setPasswordMap(parseRulesFile(is)); + } + + public LinkedHashMap getPasswordMap() { + return passwordMap; + } + + public void setPasswordMap(LinkedHashMap linkedHashMap) { + this.passwordMap = linkedHashMap; + } + + /** + * Gets the explicit password, if set + * @return the password, or null if not set + */ + public String getExplicitPassword() { + return explicitPassword; + } + + /** + * Sets an explicit password which will be used instead of password map + * @param explicitPassword the password to use + */ + public void setExplicitPassword(String explicitPassword) { + this.explicitPassword = explicitPassword; + } + + /** + * Resets explicit password, so that map will be used for lookups + */ + public void resetExplicitPassword() { + this.explicitPassword = null; + } + +} diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java index 3998f77..ad84ad5 100644 --- a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java +++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -66,6 +66,9 @@ public class SolrContentHandler extends DefaultHandler implements ExtractingPara protected String unknownFieldPrefix = ""; protected String defaultField = ""; + private boolean literalsOverride; + private Set literalFieldNames; + public SolrContentHandler(Metadata metadata, SolrParams params, IndexSchema schema) { this(metadata, params, schema, DateUtil.DEFAULT_DATE_FORMATS); } @@ -81,6 +84,7 @@ public class SolrContentHandler extends DefaultHandler implements ExtractingPara this.lowerNames = params.getBool(LOWERNAMES, false); this.captureAttribs = params.getBool(CAPTURE_ATTRIBUTES, false); + this.literalsOverride = params.getBool(LITERALS_OVERRIDE, true); this.unknownFieldPrefix = params.get(UNKNOWN_FIELD_PREFIX, ""); this.defaultField = params.get(DEFAULT_FIELD, ""); String[] captureFields = params.getParams(CAPTURE_ELEMENTS); @@ -107,13 +111,11 @@ public class SolrContentHandler extends DefaultHandler implements ExtractingPara * @see #addLiterals() */ public SolrInputDocument newDocument() { - float boost = 1.0f; - //handle the metadata extracted from the document - addMetadata(); - - //handle the literals from the params + //handle the literals from the params. NOTE: This MUST be called before the others in order for literals to override other values addLiterals(); + //handle the metadata extracted from the document + addMetadata(); //add in the content addContent(); @@ -134,8 +136,10 @@ public class SolrContentHandler extends DefaultHandler implements ExtractingPara protected void addCapturedContent() { for (Map.Entry entry : fieldBuilders.entrySet()) { if (entry.getValue().length() > 0) { - addField(entry.getKey(), entry.getValue().toString(), null); - } + String fieldName = entry.getKey(); + if (literalsOverride && literalFieldNames.contains(fieldName)) + continue; + addField(fieldName, entry.getValue().toString(), null); } } } @@ -144,6 +148,8 @@ public class SolrContentHandler extends DefaultHandler implements ExtractingPara * and the {@link #catchAllBuilder} */ protected void addContent() { + if (literalsOverride && literalFieldNames.contains(contentFieldName)) + return; addField(contentFieldName, catchAllBuilder.toString(), null); } @@ -152,12 +158,14 @@ public class SolrContentHandler extends DefaultHandler implements ExtractingPara */ protected void addLiterals() { Iterator paramNames = params.getParameterNamesIterator(); + literalFieldNames = new HashSet(); while (paramNames.hasNext()) { String pname = paramNames.next(); if (!pname.startsWith(LITERALS_PREFIX)) continue; String name = pname.substring(LITERALS_PREFIX.length()); addField(name, null, params.getParams(pname)); + literalFieldNames.add(name); } } @@ -166,6 +174,8 @@ public class SolrContentHandler extends DefaultHandler implements ExtractingPara */ protected void addMetadata() { for (String name : metadata.names()) { + if (literalsOverride && literalFieldNames.contains(name)) + continue; String[] vals = metadata.getValues(name); addField(name, null, vals); } diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandlerFactory.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandlerFactory.java index acf94a2..c91dd47 100644 --- a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandlerFactory.java +++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandlerFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/extraction/src/test-files/extraction/enctypted-password-is-solrRules.pdf b/solr/contrib/extraction/src/test-files/extraction/enctypted-password-is-solrRules.pdf new file mode 100644 index 0000000..300a476 Binary files /dev/null and b/solr/contrib/extraction/src/test-files/extraction/enctypted-password-is-solrRules.pdf differ diff --git a/solr/contrib/extraction/src/test-files/extraction/password-is-Word2010.docx b/solr/contrib/extraction/src/test-files/extraction/password-is-Word2010.docx new file mode 100755 index 0000000..24010d3 Binary files /dev/null and b/solr/contrib/extraction/src/test-files/extraction/password-is-Word2010.docx differ diff --git a/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/passwordRegex.properties b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/passwordRegex.properties new file mode 100644 index 0000000..8a4eff1 --- /dev/null +++ b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/passwordRegex.properties @@ -0,0 +1,7 @@ +# Filename regex -> password map +# Example any file ending in .doc should use password foobar: +# .*\.doc = fooBar +# +# Note: Apache Tika 1.1 supports password for .pdf and .docx only, not .doc or other formats +.*\.pdf$ = solrRules +.*\.docx$ = Word2010 \ No newline at end of file diff --git a/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/protwords.txt b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/protwords.txt new file mode 100644 index 0000000..7878147 --- /dev/null +++ b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/protwords.txt @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#use a protected word file to avoid stemming two +#unrelated words to the same base word. +#to test, we will use words that would normally obviously be stemmed. +cats +ridding diff --git a/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml new file mode 100644 index 0000000..82a3d0f --- /dev/null +++ b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml @@ -0,0 +1,475 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text + id + + + + + + + + + + + + + + + diff --git a/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml new file mode 100644 index 0000000..a63987d --- /dev/null +++ b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml @@ -0,0 +1,243 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + + 1024 + + + + + + + + + + + true + + + + + true + + 10 + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + dismax + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + + + + + + + 100 + + + + + + 70 + + + + + + + ]]> + ]]> + + + + + + + + + + max-age=30, public + + + + + solr + solrconfig.xml scheam.xml admin-extra.html + + + + prefix-${solr.test.sys.prop2}-suffix + + diff --git a/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/stopwords.txt b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/stopwords.txt new file mode 100644 index 0000000..688e307 --- /dev/null +++ b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/stopwords.txt @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +stopworda +stopwordb diff --git a/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/synonyms.txt b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/synonyms.txt new file mode 100644 index 0000000..a7624f0 --- /dev/null +++ b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/synonyms.txt @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +a => aa +b => b1 b2 +c => c1,c2 +a\=>a => b\=>b +a\,a => b\,b +foo,bar,baz + +Television,TV,Televisions diff --git a/solr/contrib/extraction/src/test-files/extraction/solr/conf/protwords.txt b/solr/contrib/extraction/src/test-files/extraction/solr/conf/protwords.txt deleted file mode 100644 index 7878147..0000000 --- a/solr/contrib/extraction/src/test-files/extraction/solr/conf/protwords.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#use a protected word file to avoid stemming two -#unrelated words to the same base word. -#to test, we will use words that would normally obviously be stemmed. -cats -ridding diff --git a/solr/contrib/extraction/src/test-files/extraction/solr/conf/schema.xml b/solr/contrib/extraction/src/test-files/extraction/solr/conf/schema.xml deleted file mode 100644 index 82a3d0f..0000000 --- a/solr/contrib/extraction/src/test-files/extraction/solr/conf/schema.xml +++ /dev/null @@ -1,475 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text - id - - - - - - - - - - - - - - - diff --git a/solr/contrib/extraction/src/test-files/extraction/solr/conf/solrconfig.xml b/solr/contrib/extraction/src/test-files/extraction/solr/conf/solrconfig.xml deleted file mode 100644 index 674b9db..0000000 --- a/solr/contrib/extraction/src/test-files/extraction/solr/conf/solrconfig.xml +++ /dev/null @@ -1,243 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - ${solr.data.dir:} - - - - - - - - - - - - - - - - - - - 1024 - - - - - - - - - - - true - - - - - true - - 10 - - - - - - - - - - - - - - - - - - - - - - - - - true - - - - dismax - - - - - - - 4 - true - text,name,subject,title,whitetok - - - - - - - 4 - true - text,name,subject,title,whitetok - - - - - - - - - - - - - 100 - - - - - - 70 - - - - - - - ]]> - ]]> - - - - - - - - - - max-age=30, public - - - - - solr - solrconfig.xml scheam.xml admin-extra.html - - - - prefix-${solr.test.sys.prop2}-suffix - - diff --git a/solr/contrib/extraction/src/test-files/extraction/solr/conf/stopwords.txt b/solr/contrib/extraction/src/test-files/extraction/solr/conf/stopwords.txt deleted file mode 100644 index 688e307..0000000 --- a/solr/contrib/extraction/src/test-files/extraction/solr/conf/stopwords.txt +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -stopworda -stopwordb diff --git a/solr/contrib/extraction/src/test-files/extraction/solr/conf/synonyms.txt b/solr/contrib/extraction/src/test-files/extraction/solr/conf/synonyms.txt deleted file mode 100644 index a7624f0..0000000 --- a/solr/contrib/extraction/src/test-files/extraction/solr/conf/synonyms.txt +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -a => aa -b => b1 b2 -c => c1,c2 -a\=>a => b\=>b -a\,a => b\,b -foo,bar,baz - -Television,TV,Televisions diff --git a/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java b/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java index b5045ff..b1995a7 100644 --- a/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java +++ b/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.extraction; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -394,7 +394,7 @@ public class ExtractingRequestHandlerTest extends SolrTestCaseJ4 { try{ loadLocal("extraction/password-is-solrcell.docx", "literal.id", "one"); - fail("TikaException is expected because of trying to extract text from password protected word file."); + fail("TikaException is expected because of trying to extract text from password protected word file without supplying a password."); } catch(Exception expected){} assertU(commit()); @@ -444,6 +444,138 @@ public class ExtractingRequestHandlerTest extends SolrTestCaseJ4 { } catch(Exception expected){} } + + public void testLiteralsOverride() throws Exception { + ExtractingRequestHandler handler = (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract"); + assertTrue("handler is null and it shouldn't be", handler != null); + + assertQ(req("*:*"), "//*[@numFound='0']"); + + // Here Tika should parse out a title for this document: + loadLocal("extraction/solr-word.pdf", + "fmap.created", "extractedDate", + "fmap.producer", "extractedProducer", + "fmap.creator", "extractedCreator", + "fmap.Keywords", "extractedKeywords", + "fmap.Author", "extractedAuthor", + "literal.id", "three", + "fmap.content", "extractedContent", + "fmap.language", "extractedLanguage", + "fmap.Creation-Date", "extractedDate", + "fmap.AAPL:Keywords", "ignored_a", + "fmap.xmpTPg:NPages", "ignored_a", + "fmap.Last-Modified", "extractedDate"); + + // Here the literal value should override the Tika-parsed title: + loadLocal("extraction/solr-word.pdf", + "literal.title", "wolf-man", + "fmap.created", "extractedDate", + "fmap.producer", "extractedProducer", + "fmap.creator", "extractedCreator", + "fmap.Keywords", "extractedKeywords", + "fmap.Author", "extractedAuthor", + "literal.id", "four", + "fmap.content", "extractedContent", + "fmap.language", "extractedLanguage", + "fmap.Creation-Date", "extractedDate", + "fmap.AAPL:Keywords", "ignored_a", + "fmap.xmpTPg:NPages", "ignored_a", + "fmap.Last-Modified", "extractedDate"); + + // Here we mimic the old behaviour where literals are added, not overridden + loadLocal("extraction/solr-word.pdf", + "literalsOverride", "false", + // Trick - we first map the metadata-title to an ignored field before we replace with literal title + "fmap.title", "ignored_a", + "literal.title", "old-behaviour", + "literal.extractedKeywords", "literalkeyword", + "fmap.created", "extractedDate", + "fmap.producer", "extractedProducer", + "fmap.creator", "extractedCreator", + "fmap.Keywords", "extractedKeywords", + "fmap.Author", "extractedAuthor", + "literal.id", "five", + "fmap.content", "extractedContent", + "fmap.language", "extractedLanguage", + "fmap.Creation-Date", "extractedDate", + "fmap.AAPL:Keywords", "ignored_a", + "fmap.xmpTPg:NPages", "ignored_a", + "fmap.Last-Modified", "extractedDate"); + + assertU(commit()); + + assertQ(req("title:solr-word"), "//*[@numFound='1']"); + assertQ(req("title:wolf-man"), "//*[@numFound='1']"); + assertQ(req("extractedKeywords:(solr AND word AND pdf AND literalkeyword)"), "//*[@numFound='1']"); + } + + @Test + public void testPasswordProtected() throws Exception { + // PDF, Passwords from resource.password + loadLocal("extraction/enctypted-password-is-solrRules.pdf", + "fmap.created", "extractedDate", + "fmap.producer", "extractedProducer", + "fmap.creator", "extractedCreator", + "fmap.Keywords", "extractedKeywords", + "fmap.Creation-Date", "extractedDate", + "uprefix", "ignored_", + "fmap.Author", "extractedAuthor", + "fmap.content", "wdf_nocase", + "literal.id", "pdfpwliteral", + "resource.name", "enctypted-password-is-solrRules.pdf", + "resource.password", "solrRules", + "fmap.Last-Modified", "extractedDate"); + + // PDF, Passwords from passwords property file + loadLocal("extraction/enctypted-password-is-solrRules.pdf", + "fmap.created", "extractedDate", + "fmap.producer", "extractedProducer", + "fmap.creator", "extractedCreator", + "fmap.Keywords", "extractedKeywords", + "fmap.Creation-Date", "extractedDate", + "uprefix", "ignored_", + "fmap.Author", "extractedAuthor", + "fmap.content", "wdf_nocase", + "literal.id", "pdfpwfile", + "resource.name", "enctypted-password-is-solrRules.pdf", + "passwordsFile", "passwordRegex.properties", // Passwords-file + "fmap.Last-Modified", "extractedDate"); + + // DOCX, Explicit password + loadLocal("extraction/password-is-Word2010.docx", + "fmap.created", "extractedDate", + "fmap.producer", "extractedProducer", + "fmap.creator", "extractedCreator", + "fmap.Keywords", "extractedKeywords", + "fmap.Creation-Date", "extractedDate", + "fmap.Author", "extractedAuthor", + "fmap.content", "wdf_nocase", + "uprefix", "ignored_", + "literal.id", "docxpwliteral", + "resource.name", "password-is-Word2010.docx", + "resource.password", "Word2010", // Explicit password + "fmap.Last-Modified", "extractedDate"); + + // DOCX, Passwords from file + loadLocal("extraction/password-is-Word2010.docx", + "fmap.created", "extractedDate", + "fmap.producer", "extractedProducer", + "fmap.creator", "extractedCreator", + "fmap.Keywords", "extractedKeywords", + "fmap.Creation-Date", "extractedDate", + "uprefix", "ignored_", + "fmap.Author", "extractedAuthor", + "fmap.content", "wdf_nocase", + "literal.id", "docxpwfile", + "resource.name", "password-is-Word2010.docx", + "passwordsFile", "passwordRegex.properties", // Passwords-file + "fmap.Last-Modified", "extractedDate"); + + assertU(commit()); + Thread.sleep(100); + assertQ(req("wdf_nocase:\"This is a test of PDF\""), "//*[@numFound='2']"); + assertQ(req("wdf_nocase:\"Test password protected word doc\""), "//*[@numFound='2']"); + } SolrQueryResponse loadLocal(String filename, String... args) throws Exception { LocalSolrQueryRequest req = (LocalSolrQueryRequest) req(args); diff --git a/solr/contrib/langid/CHANGES.txt b/solr/contrib/langid/CHANGES.txt index 9a730e4..7c1d871 100644 --- a/solr/contrib/langid/CHANGES.txt +++ b/solr/contrib/langid/CHANGES.txt @@ -7,7 +7,11 @@ See http://wiki.apache.org/solr/LanguageDetection for details $Id$ -================== Release 4.0.0 ================== +================== Release 5.0.0 ================== + +(No changes) + +================== Release 4.0.0-ALPHA ================== (No changes) diff --git a/solr/contrib/langid/src/java/org/apache/solr/update/processor/DetectedLanguage.java b/solr/contrib/langid/src/java/org/apache/solr/update/processor/DetectedLanguage.java index 0c58a96..b8371f9 100644 --- a/solr/contrib/langid/src/java/org/apache/solr/update/processor/DetectedLanguage.java +++ b/solr/contrib/langid/src/java/org/apache/solr/update/processor/DetectedLanguage.java @@ -1,6 +1,6 @@ package org.apache.solr.update.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessor.java b/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessor.java index f0f3156..83e949c 100644 --- a/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessor.java +++ b/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessor.java @@ -1,6 +1,6 @@ package org.apache.solr.update.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessorFactory.java b/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessorFactory.java index fd94868..413b97e 100644 --- a/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessorFactory.java +++ b/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessorFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.update.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangIdParams.java b/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangIdParams.java index e43b504..8c35bfd 100644 --- a/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangIdParams.java +++ b/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangIdParams.java @@ -1,6 +1,6 @@ package org.apache.solr.update.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java b/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java index 0ef8e4f..4bab7d0 100644 --- a/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java +++ b/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java @@ -1,6 +1,6 @@ package org.apache.solr.update.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessor.java b/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessor.java index 7747e63..4ba326d 100644 --- a/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessor.java +++ b/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessor.java @@ -1,6 +1,6 @@ package org.apache.solr.update.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessorFactory.java b/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessorFactory.java index 0c940b2..fb0d503 100644 --- a/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessorFactory.java +++ b/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessorFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.update.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/langid/src/test-files/langid/solr/collection1/conf/schema.xml b/solr/contrib/langid/src/test-files/langid/solr/collection1/conf/schema.xml new file mode 100644 index 0000000..4c3d2e8 --- /dev/null +++ b/solr/contrib/langid/src/test-files/langid/solr/collection1/conf/schema.xml @@ -0,0 +1,81 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + name + id + + + diff --git a/solr/contrib/langid/src/test-files/langid/solr/collection1/conf/solrconfig-languageidentifier.xml b/solr/contrib/langid/src/test-files/langid/solr/collection1/conf/solrconfig-languageidentifier.xml new file mode 100644 index 0000000..6b2fdcd --- /dev/null +++ b/solr/contrib/langid/src/test-files/langid/solr/collection1/conf/solrconfig-languageidentifier.xml @@ -0,0 +1,98 @@ + + + + + + + + + + ${solr.data.dir:./solr/data} + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + explicit + 10 + + + + + true + + + + + + + + + + max-age=30, public + + + + + + lang_id + + + + + + + + true + name,subject + true + language_s + language_sm + th:thai + 0.5 + fallback + + + + + + + + + + true + name,subject + true + language_s + language_sm + th:thai + 0.5 + fallback + + + + + + diff --git a/solr/contrib/langid/src/test-files/langid/solr/conf/schema.xml b/solr/contrib/langid/src/test-files/langid/solr/conf/schema.xml deleted file mode 100644 index 4c3d2e8..0000000 --- a/solr/contrib/langid/src/test-files/langid/solr/conf/schema.xml +++ /dev/null @@ -1,81 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - name - id - - - diff --git a/solr/contrib/langid/src/test-files/langid/solr/conf/solrconfig-languageidentifier.xml b/solr/contrib/langid/src/test-files/langid/solr/conf/solrconfig-languageidentifier.xml deleted file mode 100644 index 6b2fdcd..0000000 --- a/solr/contrib/langid/src/test-files/langid/solr/conf/solrconfig-languageidentifier.xml +++ /dev/null @@ -1,98 +0,0 @@ - - - - - - - - - - ${solr.data.dir:./solr/data} - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - explicit - 10 - - - - - true - - - - - - - - - - max-age=30, public - - - - - - lang_id - - - - - - - - true - name,subject - true - language_s - language_sm - th:thai - 0.5 - fallback - - - - - - - - - - true - name,subject - true - language_s - language_sm - th:thai - 0.5 - fallback - - - - - - diff --git a/solr/contrib/langid/src/test/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessorFactoryTest.java b/solr/contrib/langid/src/test/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessorFactoryTest.java index 10df90b..7b7382f 100644 --- a/solr/contrib/langid/src/test/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessorFactoryTest.java +++ b/solr/contrib/langid/src/test/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessorFactoryTest.java @@ -1,6 +1,6 @@ package org.apache.solr.update.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java b/solr/contrib/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java index fdcaf18..cde227e 100644 --- a/solr/contrib/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java +++ b/solr/contrib/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/langid/src/test/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessorFactoryTest.java b/solr/contrib/langid/src/test/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessorFactoryTest.java index d65c66d..bf3a40b 100644 --- a/solr/contrib/langid/src/test/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessorFactoryTest.java +++ b/solr/contrib/langid/src/test/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessorFactoryTest.java @@ -1,6 +1,6 @@ package org.apache.solr.update.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/uima/CHANGES.txt b/solr/contrib/uima/CHANGES.txt index c43b24c..9126fbe 100644 --- a/solr/contrib/uima/CHANGES.txt +++ b/solr/contrib/uima/CHANGES.txt @@ -22,7 +22,11 @@ WhitespaceTokenizer v2.3.1 $Id$ -================== 4.0.0-dev ============== +================== 5.0.0 ============== + +(No Changes) + +================== 4.0.0-ALPHA ============== (No Changes) diff --git a/solr/contrib/uima/src/java/org/apache/solr/uima/processor/SolrUIMAConfiguration.java b/solr/contrib/uima/src/java/org/apache/solr/uima/processor/SolrUIMAConfiguration.java index 8cbab1b..96f3c46 100644 --- a/solr/contrib/uima/src/java/org/apache/solr/uima/processor/SolrUIMAConfiguration.java +++ b/solr/contrib/uima/src/java/org/apache/solr/uima/processor/SolrUIMAConfiguration.java @@ -1,6 +1,6 @@ package org.apache.solr.uima.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/uima/src/java/org/apache/solr/uima/processor/SolrUIMAConfigurationReader.java b/solr/contrib/uima/src/java/org/apache/solr/uima/processor/SolrUIMAConfigurationReader.java index 98e6c88..fdf82ff 100644 --- a/solr/contrib/uima/src/java/org/apache/solr/uima/processor/SolrUIMAConfigurationReader.java +++ b/solr/contrib/uima/src/java/org/apache/solr/uima/processor/SolrUIMAConfigurationReader.java @@ -1,6 +1,6 @@ package org.apache.solr.uima.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAToSolrMapper.java b/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAToSolrMapper.java index 80a96ef..59bf196 100644 --- a/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAToSolrMapper.java +++ b/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAToSolrMapper.java @@ -1,6 +1,6 @@ package org.apache.solr.uima.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java b/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java index a8a6235..586355b 100644 --- a/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java +++ b/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java @@ -1,6 +1,6 @@ package org.apache.solr.uima.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -40,7 +40,6 @@ import java.util.Map; /** * Update document(s) to be indexed with UIMA extracted information * - * */ public class UIMAUpdateRequestProcessor extends UpdateRequestProcessor { @@ -102,16 +101,24 @@ public class UIMAUpdateRequestProcessor extends UpdateRequestProcessor { new StringBuilder(". ").append(logField).append("=") .append((String)cmd.getSolrInputDocument().getField(logField).getValue()) .append(", ").toString(); - int len = Math.min(text.length(), 100); + int len; + String debugString; + if (text != null && text.length() > 0) { + len = Math.min(text.length(), 100); + debugString = new StringBuilder(" text=\"").append(text.substring(0, len)).append("...\"").toString(); + } + else { + debugString = " null text"; + } if (solrUIMAConfiguration.isIgnoreErrors()) { log.warn(new StringBuilder("skip the text processing due to ") .append(e.getLocalizedMessage()).append(optionalFieldInfo) - .append(" text=\"").append(text.substring(0, len)).append("...\"").toString()); + .append(debugString).toString()); } else { throw new SolrException(ErrorCode.SERVER_ERROR, - new StringBuilder("processing error: ") + new StringBuilder("processing error ") .append(e.getLocalizedMessage()).append(optionalFieldInfo) - .append(" text=\"").append(text.substring(0, len)).append("...\"").toString(), e); + .append(debugString).toString(), e); } } super.processAdd(cmd); @@ -143,7 +150,7 @@ public class UIMAUpdateRequestProcessor extends UpdateRequestProcessor { /* process a field value executing UIMA the CAS containing it as document text */ private JCas processText(String textFieldValue) throws ResourceInitializationException, AnalysisEngineProcessException { - log.info(new StringBuffer("Analazying text").toString()); + log.info(new StringBuffer("Analyzing text").toString()); /* get the UIMA analysis engine */ AnalysisEngine ae = aeProvider.getAE(); diff --git a/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorFactory.java b/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorFactory.java index 8fab664..0855f4c 100644 --- a/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorFactory.java +++ b/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.uima.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/uima/src/resources/solr/collection1/conf/aggregate-uima-config.xml b/solr/contrib/uima/src/resources/solr/collection1/conf/aggregate-uima-config.xml new file mode 100644 index 0000000..842b8e5 --- /dev/null +++ b/solr/contrib/uima/src/resources/solr/collection1/conf/aggregate-uima-config.xml @@ -0,0 +1,48 @@ + + + + + + + + VALID_ALCHEMYAPI_KEY + VALID_ALCHEMYAPI_KEY + VALID_ALCHEMYAPI_KEY + VALID_ALCHEMYAPI_KEY + VALID_ALCHEMYAPI_KEY + VALID_OPENCALAIS_KEY + + /org/apache/uima/desc/OverridingParamsExtServicesAE.xml + + false + + text + title + + + + + org.apache.uima.jcas.tcas.Annotation + convertText + tag + + + + + + + diff --git a/solr/contrib/uima/src/resources/solr/collection1/conf/uima-fields.xml b/solr/contrib/uima/src/resources/solr/collection1/conf/uima-fields.xml new file mode 100644 index 0000000..270aaa6 --- /dev/null +++ b/solr/contrib/uima/src/resources/solr/collection1/conf/uima-fields.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/solr/contrib/uima/src/resources/solr/conf/aggregate-uima-config.xml b/solr/contrib/uima/src/resources/solr/conf/aggregate-uima-config.xml deleted file mode 100644 index 842b8e5..0000000 --- a/solr/contrib/uima/src/resources/solr/conf/aggregate-uima-config.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - - - VALID_ALCHEMYAPI_KEY - VALID_ALCHEMYAPI_KEY - VALID_ALCHEMYAPI_KEY - VALID_ALCHEMYAPI_KEY - VALID_ALCHEMYAPI_KEY - VALID_OPENCALAIS_KEY - - /org/apache/uima/desc/OverridingParamsExtServicesAE.xml - - false - - text - title - - - - - org.apache.uima.jcas.tcas.Annotation - convertText - tag - - - - - - - diff --git a/solr/contrib/uima/src/resources/solr/conf/uima-fields.xml b/solr/contrib/uima/src/resources/solr/conf/uima-fields.xml deleted file mode 100644 index 270aaa6..0000000 --- a/solr/contrib/uima/src/resources/solr/conf/uima-fields.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/protwords.txt b/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/protwords.txt new file mode 100644 index 0000000..1dfc0ab --- /dev/null +++ b/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/protwords.txt @@ -0,0 +1,21 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# Use a protected word file to protect against the stemmer reducing two +# unrelated words to the same base word. + +# Some non-words that normally won't be encountered, +# just to test that they won't be stemmed. +dontstems +zwhacky + diff --git a/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/schema.xml b/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/schema.xml new file mode 100644 index 0000000..a322655 --- /dev/null +++ b/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/schema.xml @@ -0,0 +1,678 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + text + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/solrconfig.xml b/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/solrconfig.xml new file mode 100644 index 0000000..0444bf8 --- /dev/null +++ b/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/solrconfig.xml @@ -0,0 +1,971 @@ + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + + + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + + + + + + + + + + 1024 + + + + + + + + + + + + + + + + true + + + + + + + + 20 + + + 200 + + + + + + + + + + + + + + solr rocks + 0 + 10 + + + static firstSearcher warming query from + solrconfig.xml + + + + + + false + + + 2 + + + + + + + + + + + + + + + + + + + + + + + explicit + + + + + + + + + + + + + dismax + explicit + 0.01 + + text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 + manu^1.1 cat^1.4 + + + text^0.2 features^1.1 name^1.5 manu^1.4 + manu_exact^1.9 + + + popularity^0.5 recip(price,1,1000,1000)^0.3 + + + id,name,price,score + + + 2<-1 5<-2 6<90% + 100 + *:* + + text features name + + 0 + + name + regex + + + + + + + dismax + explicit + text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 + 2<-1 5<-2 6<90% + + incubationdate_dt:[* TO NOW/DAY-1MONTH]^2.2 + + + + inStock:true + + + + cat + manu_exact + price:[* TO 500] + price:[500 TO *] + + + + + + + + + + textSpell + + + default + name + ./spellchecker + + + + + + + + + + + + false + + false + + 1 + + + spellcheck + + + + + + + + true + + + tvComponent + + + + + + + + + default + + org.carrot2.clustering.lingo.LingoClusteringAlgorithm + + 20 + + + stc + org.carrot2.clustering.stc.STCClusteringAlgorithm + + + + + true + default + true + + name + id + + features + + true + + + + false + + + clusteringComponent + + + + + + + + text + true + ignored_ + + + true + links + ignored_ + + + + + + + + + + true + + + termsComponent + + + + + + + + + + uima + + + + + + + + + + + + + + + + + + + + + + + + explicit + true + + + + + + + + + + 3 + + /uima/TestAE.xml + + false + + text + + + + + org.apache.uima.SentenceAnnotation + + coveredText + sentence + + + + org.apache.solr.uima.ts.SentimentAnnotation + + mood + sentiment + + + + org.apache.solr.uima.ts.EntityAnnotation + + entity + name + *_sm + + + + + + + + + + + + + 3 + + /uima/TestAE.xml + + false + + text + + + + + a-type-which-can-have-multiple-features + + A + 1 + + + B + 2 + + + + + + + + + + + + 3 + + /uima/TestExceptionAE.xml + false + + false + + text + + + + + + + + + + + + + 3 + + /uima/TestExceptionAE.xml + true + + id + + false + + text + + + + + + + + + + + + + 5 + + + + + + + + + + * + + + diff --git a/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/spellings.txt b/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/spellings.txt new file mode 100644 index 0000000..162a044 --- /dev/null +++ b/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/spellings.txt @@ -0,0 +1,2 @@ +pizza +history diff --git a/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/stopwords.txt b/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/stopwords.txt new file mode 100644 index 0000000..b5824da --- /dev/null +++ b/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/stopwords.txt @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +#Standard english stop words taken from Lucene's StopAnalyzer +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +s +such +t +that +the +their +then +there +these +they +this +to +was +will +with + diff --git a/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/synonyms.txt b/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/synonyms.txt new file mode 100644 index 0000000..b0e31cb --- /dev/null +++ b/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/synonyms.txt @@ -0,0 +1,31 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +aaa => aaaa +bbb => bbbb1 bbbb2 +ccc => cccc1,cccc2 +a\=>a => b\=>b +a\,a => b\,b +fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +pixima => pixma + diff --git a/solr/contrib/uima/src/test-files/uima/solr/conf/protwords.txt b/solr/contrib/uima/src/test-files/uima/solr/conf/protwords.txt deleted file mode 100644 index 1dfc0ab..0000000 --- a/solr/contrib/uima/src/test-files/uima/solr/conf/protwords.txt +++ /dev/null @@ -1,21 +0,0 @@ -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#----------------------------------------------------------------------- -# Use a protected word file to protect against the stemmer reducing two -# unrelated words to the same base word. - -# Some non-words that normally won't be encountered, -# just to test that they won't be stemmed. -dontstems -zwhacky - diff --git a/solr/contrib/uima/src/test-files/uima/solr/conf/schema.xml b/solr/contrib/uima/src/test-files/uima/solr/conf/schema.xml deleted file mode 100644 index a322655..0000000 --- a/solr/contrib/uima/src/test-files/uima/solr/conf/schema.xml +++ /dev/null @@ -1,678 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - text - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/solr/contrib/uima/src/test-files/uima/solr/conf/solrconfig.xml b/solr/contrib/uima/src/test-files/uima/solr/conf/solrconfig.xml deleted file mode 100644 index 5decc62..0000000 --- a/solr/contrib/uima/src/test-files/uima/solr/conf/solrconfig.xml +++ /dev/null @@ -1,971 +0,0 @@ - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - - - - - - - - - - - - - ${solr.data.dir:} - - - - - - - - - - - - - - - - - - - - - - - - - - - 1024 - - - - - - - - - - - - - - - - true - - - - - - - - 20 - - - 200 - - - - - - - - - - - - - - solr rocks - 0 - 10 - - - static firstSearcher warming query from - solrconfig.xml - - - - - - false - - - 2 - - - - - - - - - - - - - - - - - - - - - - - explicit - - - - - - - - - - - - - dismax - explicit - 0.01 - - text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 - manu^1.1 cat^1.4 - - - text^0.2 features^1.1 name^1.5 manu^1.4 - manu_exact^1.9 - - - popularity^0.5 recip(price,1,1000,1000)^0.3 - - - id,name,price,score - - - 2<-1 5<-2 6<90% - 100 - *:* - - text features name - - 0 - - name - regex - - - - - - - dismax - explicit - text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 - 2<-1 5<-2 6<90% - - incubationdate_dt:[* TO NOW/DAY-1MONTH]^2.2 - - - - inStock:true - - - - cat - manu_exact - price:[* TO 500] - price:[500 TO *] - - - - - - - - - - textSpell - - - default - name - ./spellchecker - - - - - - - - - - - - false - - false - - 1 - - - spellcheck - - - - - - - - true - - - tvComponent - - - - - - - - - default - - org.carrot2.clustering.lingo.LingoClusteringAlgorithm - - 20 - - - stc - org.carrot2.clustering.stc.STCClusteringAlgorithm - - - - - true - default - true - - name - id - - features - - true - - - - false - - - clusteringComponent - - - - - - - - text - true - ignored_ - - - true - links - ignored_ - - - - - - - - - - true - - - termsComponent - - - - - - - - - - uima - - - - - - - - - - - - - - - - - - - - - - - - explicit - true - - - - - - - - - - 3 - - /uima/TestAE.xml - - false - - text - - - - - org.apache.uima.SentenceAnnotation - - coveredText - sentence - - - - org.apache.solr.uima.ts.SentimentAnnotation - - mood - sentiment - - - - org.apache.solr.uima.ts.EntityAnnotation - - entity - name - *_sm - - - - - - - - - - - - - 3 - - /uima/TestAE.xml - - false - - text - - - - - a-type-which-can-have-multiple-features - - A - 1 - - - B - 2 - - - - - - - - - - - - 3 - - /uima/TestExceptionAE.xml - false - - false - - text - - - - - - - - - - - - - 3 - - /uima/TestExceptionAE.xml - true - - id - - false - - text - - - - - - - - - - - - - 5 - - - - - - - - - - * - - - diff --git a/solr/contrib/uima/src/test-files/uima/solr/conf/spellings.txt b/solr/contrib/uima/src/test-files/uima/solr/conf/spellings.txt deleted file mode 100644 index 162a044..0000000 --- a/solr/contrib/uima/src/test-files/uima/solr/conf/spellings.txt +++ /dev/null @@ -1,2 +0,0 @@ -pizza -history diff --git a/solr/contrib/uima/src/test-files/uima/solr/conf/stopwords.txt b/solr/contrib/uima/src/test-files/uima/solr/conf/stopwords.txt deleted file mode 100644 index b5824da..0000000 --- a/solr/contrib/uima/src/test-files/uima/solr/conf/stopwords.txt +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#----------------------------------------------------------------------- -# a couple of test stopwords to test that the words are really being -# configured from this file: -stopworda -stopwordb - -#Standard english stop words taken from Lucene's StopAnalyzer -a -an -and -are -as -at -be -but -by -for -if -in -into -is -it -no -not -of -on -or -s -such -t -that -the -their -then -there -these -they -this -to -was -will -with - diff --git a/solr/contrib/uima/src/test-files/uima/solr/conf/synonyms.txt b/solr/contrib/uima/src/test-files/uima/solr/conf/synonyms.txt deleted file mode 100644 index b0e31cb..0000000 --- a/solr/contrib/uima/src/test-files/uima/solr/conf/synonyms.txt +++ /dev/null @@ -1,31 +0,0 @@ -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#----------------------------------------------------------------------- -#some test synonym mappings unlikely to appear in real input text -aaa => aaaa -bbb => bbbb1 bbbb2 -ccc => cccc1,cccc2 -a\=>a => b\=>b -a\,a => b\,b -fooaaa,baraaa,bazaaa - -# Some synonym groups specific to this example -GB,gib,gigabyte,gigabytes -MB,mib,megabyte,megabytes -Television, Televisions, TV, TVs -#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming -#after us won't split it into two words. - -# Synonym mappings can be used for spelling correction too -pixima => pixma - diff --git a/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml b/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml index 54d8590..40cacc3 100644 --- a/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml +++ b/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml @@ -363,7 +363,7 @@ diff --git a/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java b/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java index 00f01b1..ee92123 100644 --- a/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java +++ b/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java @@ -1,6 +1,6 @@ package org.apache.solr.uima.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Map; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.MultiMapSolrParams; @@ -46,6 +47,7 @@ import org.junit.Test; * * */ +@Slow public class UIMAUpdateRequestProcessorTest extends SolrTestCaseJ4 { @BeforeClass diff --git a/solr/contrib/uima/src/test/org/apache/solr/uima/processor/an/DummyEntityAnnotator.java b/solr/contrib/uima/src/test/org/apache/solr/uima/processor/an/DummyEntityAnnotator.java index e59da12..6444b89 100644 --- a/solr/contrib/uima/src/test/org/apache/solr/uima/processor/an/DummyEntityAnnotator.java +++ b/solr/contrib/uima/src/test/org/apache/solr/uima/processor/an/DummyEntityAnnotator.java @@ -7,7 +7,7 @@ import org.apache.uima.analysis_engine.AnalysisEngineProcessException; import org.apache.uima.jcas.JCas; import org.apache.uima.jcas.tcas.Annotation; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/uima/src/test/org/apache/solr/uima/processor/an/DummyExceptionAnnotator.java b/solr/contrib/uima/src/test/org/apache/solr/uima/processor/an/DummyExceptionAnnotator.java index d0de60f..5838ade 100644 --- a/solr/contrib/uima/src/test/org/apache/solr/uima/processor/an/DummyExceptionAnnotator.java +++ b/solr/contrib/uima/src/test/org/apache/solr/uima/processor/an/DummyExceptionAnnotator.java @@ -4,7 +4,7 @@ import org.apache.uima.analysis_component.JCasAnnotator_ImplBase; import org.apache.uima.analysis_engine.AnalysisEngineProcessException; import org.apache.uima.jcas.JCas; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/uima/src/test/org/apache/solr/uima/processor/an/DummySentimentAnnotator.java b/solr/contrib/uima/src/test/org/apache/solr/uima/processor/an/DummySentimentAnnotator.java index b91d986..04f1e59 100644 --- a/solr/contrib/uima/src/test/org/apache/solr/uima/processor/an/DummySentimentAnnotator.java +++ b/solr/contrib/uima/src/test/org/apache/solr/uima/processor/an/DummySentimentAnnotator.java @@ -9,7 +9,7 @@ import org.apache.uima.analysis_engine.AnalysisEngineProcessException; import org.apache.uima.jcas.JCas; import org.apache.uima.jcas.tcas.Annotation; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/velocity/src/java/org/apache/solr/response/PageTool.java b/solr/contrib/velocity/src/java/org/apache/solr/response/PageTool.java index af00c33..48eff97 100644 --- a/solr/contrib/velocity/src/java/org/apache/solr/response/PageTool.java +++ b/solr/contrib/velocity/src/java/org/apache/solr/response/PageTool.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/velocity/src/java/org/apache/solr/response/SolrParamResourceLoader.java b/solr/contrib/velocity/src/java/org/apache/solr/response/SolrParamResourceLoader.java index 6319a60..ab486bb 100644 --- a/solr/contrib/velocity/src/java/org/apache/solr/response/SolrParamResourceLoader.java +++ b/solr/contrib/velocity/src/java/org/apache/solr/response/SolrParamResourceLoader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/velocity/src/java/org/apache/solr/response/SolrVelocityResourceLoader.java b/solr/contrib/velocity/src/java/org/apache/solr/response/SolrVelocityResourceLoader.java index 67feca0..9596bbb 100644 --- a/solr/contrib/velocity/src/java/org/apache/solr/response/SolrVelocityResourceLoader.java +++ b/solr/contrib/velocity/src/java/org/apache/solr/response/SolrVelocityResourceLoader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java b/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java index cfb0bc6..34ab76c 100644 --- a/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java +++ b/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/schema.xml b/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/schema.xml new file mode 100644 index 0000000..c3499be --- /dev/null +++ b/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/schema.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + id + + id + + + + diff --git a/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/solrconfig.xml b/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/solrconfig.xml new file mode 100644 index 0000000..c770a3f --- /dev/null +++ b/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/solrconfig.xml @@ -0,0 +1,106 @@ + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + ${solr.data.dir:} + + + + + + + + + + + + 1024 + + + + true + + 20 + 200 + + + + + + + + static firstSearcher warming in solrconfig.xml + + + + false + 2 + + + + + + + + + + explicit + 10 + + + + + + + + + + *:* + + + diff --git a/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/velocity/VM_global_library.vm b/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/velocity/VM_global_library.vm new file mode 100644 index 0000000..398b8a2 --- /dev/null +++ b/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/velocity/VM_global_library.vm @@ -0,0 +1,50 @@ + + + +#macro(nl2ul $named_list) +

        + #foreach($kv in $named_list) +
      • $kv.key ($kv.value) + #nl2ul($kv.value) +
        • + #end +
      +#end + +#macro(url_for_filters $filter_queries) + /solr/itas?fq=$filter_queries +#end + +#macro(url_for_home)/solr/itas#end + +#macro(url_for_start $start)/solr/itas?start=$start&q=$!{esc.url($params.get('q'))}#end + +## TODO: need to add escaping +#macro(url_for_nested_facet $field $value)/solr/itas?fq=$field:%22$value%22#end + +#macro(link_to_previous_page $text) + #if($page.current_page_number > 1) + #set($prev_start = $page.start - $page.results_per_page) + $text + #end +#end + +#macro(link_to_next_page $text) + #if($page.current_page_number < $page.page_count) + #set($next_start = $page.start + $page.results_per_page) + $text + #end +#end + +#macro(link_to_page $page_number $text) + #if($page_number == $page.current_page_number) + $text + #else + #if($page_number <= $page.page_count) + #set($page_start = $page_number * $page.results_per_page - $page.results_per_page) + $text + #end + #end +#end + +#macro(param $key)$request.params.get($key)#end diff --git a/solr/contrib/velocity/src/test-files/velocity/solr/conf/schema.xml b/solr/contrib/velocity/src/test-files/velocity/solr/conf/schema.xml deleted file mode 100644 index c3499be..0000000 --- a/solr/contrib/velocity/src/test-files/velocity/solr/conf/schema.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - - - - - - - - - - id - - id - - - - diff --git a/solr/contrib/velocity/src/test-files/velocity/solr/conf/solrconfig.xml b/solr/contrib/velocity/src/test-files/velocity/solr/conf/solrconfig.xml deleted file mode 100644 index c770a3f..0000000 --- a/solr/contrib/velocity/src/test-files/velocity/solr/conf/solrconfig.xml +++ /dev/null @@ -1,106 +0,0 @@ - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - ${solr.data.dir:} - - - - - - - - - - - - 1024 - - - - true - - 20 - 200 - - - - - - - - static firstSearcher warming in solrconfig.xml - - - - false - 2 - - - - - - - - - - explicit - 10 - - - - - - - - - - *:* - - - diff --git a/solr/contrib/velocity/src/test-files/velocity/solr/conf/velocity/VM_global_library.vm b/solr/contrib/velocity/src/test-files/velocity/solr/conf/velocity/VM_global_library.vm deleted file mode 100644 index 398b8a2..0000000 --- a/solr/contrib/velocity/src/test-files/velocity/solr/conf/velocity/VM_global_library.vm +++ /dev/null @@ -1,50 +0,0 @@ - - - -#macro(nl2ul $named_list) -
        - #foreach($kv in $named_list) -
      • $kv.key ($kv.value) - #nl2ul($kv.value) -
        • - #end -
      -#end - -#macro(url_for_filters $filter_queries) - /solr/itas?fq=$filter_queries -#end - -#macro(url_for_home)/solr/itas#end - -#macro(url_for_start $start)/solr/itas?start=$start&q=$!{esc.url($params.get('q'))}#end - -## TODO: need to add escaping -#macro(url_for_nested_facet $field $value)/solr/itas?fq=$field:%22$value%22#end - -#macro(link_to_previous_page $text) - #if($page.current_page_number > 1) - #set($prev_start = $page.start - $page.results_per_page) - $text - #end -#end - -#macro(link_to_next_page $text) - #if($page.current_page_number < $page.page_count) - #set($next_start = $page.start + $page.results_per_page) - $text - #end -#end - -#macro(link_to_page $page_number $text) - #if($page_number == $page.current_page_number) - $text - #else - #if($page_number <= $page.page_count) - #set($page_start = $page_number * $page.results_per_page - $page.results_per_page) - $text - #end - #end -#end - -#macro(param $key)$request.params.get($key)#end diff --git a/solr/contrib/velocity/src/test/org/apache/solr/velocity/VelocityResponseWriterTest.java b/solr/contrib/velocity/src/test/org/apache/solr/velocity/VelocityResponseWriterTest.java index 79aeb42..23046ea 100644 --- a/solr/contrib/velocity/src/test/org/apache/solr/velocity/VelocityResponseWriterTest.java +++ b/solr/contrib/velocity/src/test/org/apache/solr/velocity/VelocityResponseWriterTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/doc-files/tutorial.html b/solr/core/src/java/doc-files/tutorial.html index 66b9a5a..685c973 100755 --- a/solr/core/src/java/doc-files/tutorial.html +++ b/solr/core/src/java/doc-files/tutorial.html @@ -56,12 +56,15 @@ To follow along with this tutorial, you will need...
    • Java 1.6 or greater. Some places you can get it are from Oracle, - Open JDK, - IBM, or -
      - Running java -version at the command line should indicate a version - number starting with 1.6. Gnu's GCJ is not supported and does not work with Solr. -
      • + Open JDK, or + IBM. +
        +
      • Running java -version at the command + line should indicate a version number starting with 1.6. +
        • +
      • Gnu's GCJ is not supported and does not work with Solr.
        • +
      +
    • A Solr release.
      • @@ -97,16 +100,18 @@ To launch Jetty with the Solr WAR, and the example configs, just run the 
       user:~/solr/example$ java -jar start.jar
-2012-03-27 17:11:29.529:INFO::Logging to STDERR via org.mortbay.log.StdErrLog
-2012-03-27 17:11:29.696:INFO::jetty-6.1-SNAPSHOT
+2012-06-06 15:25:59.815:INFO:oejs.Server:jetty-8.1.2.v20120308
+2012-06-06 15:25:59.834:INFO:oejdp.ScanningAppProvider:Deployment monitor .../solr/example/webapps at interval 0
+2012-06-06 15:25:59.839:INFO:oejd.DeploymentManager:Deployable added: .../solr/example/webapps/solr.war
 ...
-2012-03-27 17:11:32.343:INFO::Started SocketConnector@0.0.0.0:8983
+Jun 6, 2012 3:26:03 PM org.apache.solr.core.SolrCore registerSearcher
+INFO: [collection1] Registered new searcher Searcher@7527e2ee main{StandardDirectoryReader(segments_1:1)}

      This will start up the Jetty application server on port 8983, and use your terminal to display the logging information from Solr.

      -You can see that the Solr is running by loading http://localhost:8983/solr/admin/ in your web browser. This is the main starting point for Administering Solr. +You can see that the Solr is running by loading http://localhost:8983/solr/ in your web browser. This is the main starting point for Administering Solr.

      @@ -118,19 +123,21 @@ You can see that the Solr is running by loading

      Your Solr server is up and running, but it doesn't contain any data. You can -modify a Solr index by POSTing XML Documents containing instructions to add (or -update) documents, delete documents, commit pending adds and deletes, and -optimize your index. +modify a Solr index by POSTing commands to Solr to add (or +update) documents, delete documents, and commit pending adds and deletes. +These commands can be in a +variety of formats.

      -The exampledocs directory contains samples of the types of -instructions Solr expects, as well as a java utility for posting them from the -command line (a post.sh shell script is also available, but for -this tutorial we'll use the cross-platform Java client).

      -

      To try this, -open a new terminal window, enter the exampledocs directory, and run -"java -jar post.jar" on some of the XML files in that directory, -indicating the URL of the Solr server: +The exampledocs directory contains sample files +showing of the types of commands Solr accepts, as well as a java utility +for posting them from the command line (a post.sh +shell script is also available, but for this tutorial we'll use the +cross-platform Java client). +

      +

      To try this, open a new terminal window, enter the exampledocs directory, +and run "java -jar post.jar" on some of the XML +files in that directory. 

       user:~/solr/example/exampledocs$ java -jar post.jar solr.xml monitor.xml
@@ -142,16 +149,15 @@ SimplePostTool: COMMITting Solr index changes..

      You have now indexed two documents in Solr, and committed these changes. -You can now search for "solr" using the "Make a Query" interface on the Admin screen, and you should get one result. -Clicking the "Search" button should take you to the following URL... +You can now search for "solr" by loading the "Query" tab in the Admin interface, and entering "solr" in the "q" text box. Clicking the "Execute Query" button should display the following URL containing one result...

      - -http://localhost:8983/solr/select/?q=solr&start=0&rows=10&indent=on +http://localhost:8983/solr/collection1/select?q=solr&wt=xml

      -You can index all of the sample data, using the following command (assuming your command line shell supports the *.xml notation): +You can index all of the sample data, using the following command +(assuming your command line shell supports the *.xml notation): 

       user:~/solr/example/exampledocs$ java -jar post.jar *.xml
@@ -161,12 +167,7 @@ SimplePostTool: POSTing file gb18030-example.xml
 SimplePostTool: POSTing file hd.xml
 SimplePostTool: POSTing file ipod_other.xml
 SimplePostTool: POSTing file ipod_video.xml
-SimplePostTool: POSTing file mem.xml
-SimplePostTool: POSTing file money.xml
-SimplePostTool: POSTing file monitor2.xml
-SimplePostTool: POSTing file monitor.xml
-SimplePostTool: POSTing file mp500.xml
-SimplePostTool: POSTing file sd500.xml
+...
 SimplePostTool: POSTing file solr.xml
 SimplePostTool: POSTing file utf8-example.xml
 SimplePostTool: POSTing file vidcard.xml
@@ -178,15 +179,15 @@ SimplePostTool: COMMITting Solr index changes..
 
        
   
 
      • 
-video
+video
 
        • 
   
 
      • 
-name:video
+name:video
 
        • 
   
 
      • 
-+video +price:[* TO 400]
++video +price:[* TO 400]
 
        • 
 
 
@@ -234,14 +235,14 @@ SimplePostTool: COMMITting Solr index changes..
 You may have noticed that even though the file solr.xml has now
 been POSTed to the server twice, you still only get 1 result when searching for
 "solr".  This is because the example schema.xml specifies a "uniqueKey" field
-called "id".  Whenever you POST instructions to Solr to add a
+called "id".  Whenever you POST commands to Solr to add a
 document with the same value for the uniqueKey as an existing document, it
 automatically replaces it for you.  You can see that that has happened by
 looking at the values for numDocs and maxDoc in the
 "CORE"/searcher section of the statistics page...  
        
 
        
 
-http://localhost:8983/solr/admin/stats.jsp
+http://localhost:8983/solr/#/collection1/plugins/core?entry=searcher
 
 
        
 
        
@@ -261,28 +262,61 @@ in subsequent searches.
 
        
 
 
        Deleting Data
        
-
        You can delete data by POSTing a delete command to the update URL and specifying the value
-      of the document's unique key field, or a query that matches multiple documents (be careful with that one!).  Since these commands
-      are smaller, we will specify them right on the command line rather than reference an XML file.
-    
        
-
        Execute the following command to delete a document
        
-
        java -Ddata=args -Dcommit=no -jar post.jar "<delete><id>SP2514N</id></delete>"
        
-
        Now if you go to the statistics page and scroll down
-       to the UPDATE_HANDLERS section and verify that "deletesById : 1"
        
-
        If you search for id:SP2514N it will still be found,
-       because index changes are not visible until changes are committed and a new searcher is opened.  To cause
-       this to happen, send a commit command to Solr (post.jar does this for you by default):
        
+
+
        
+You can delete data by POSTing a delete command to the update URL and 
+specifying the value of the document's unique key field, or a query that 
+matches multiple documents (be careful with that one!).  Since these commands
+are smaller, we will specify them right on the command line rather than 
+reference an XML file.
+
        
+
+
        Execute the following command to delete a specific document
        
+
        java -Ddata=args -Dcommit=false -jar post.jar "<delete><id>SP2514N</id></delete>"
        
+
+
        
+Because we have specified "commit=false", a search for id:SP2514N we still find the document we have deleted.  Since the example configuration uses Solr's "autoCommit" feature Solr will still automatically persist this change to the index, but it will not affect search results until an "openSearcher" commit is explicitly executed.
+
        
+
+
        
+Using the statistics page 
+for the updateHandler you can observe this delete
+propogate to disk by watching the deletesById 
+value drop to 0 as the cumulative_deletesById 
+and autocommit values increase.
+
        
+
+
        
+Here is an example of using delete-by-query to delete anything with
+DDR in the name:
+
        
+
        java -Dcommit=false -Ddata=args -jar post.jar "<delete><query>name:DDR</query></delete>"
        
+
+
        
+You can force a new searcher to be opened to reflect these changes by sending a commit command to Solr (which post.jar does for you by default):
+
        
 
        java -jar post.jar
        
-
        Now re-execute the previous search and verify that no matching documents are found.  Also revisit the
-    statistics page and observe the changes in both the UPDATE_HANDLERS section and the CORE section.
        
-
        Here is an example of using delete-by-query to delete anything with
-      DDR in the name:
        
-
        java -Ddata=args -jar post.jar "<delete><query>name:DDR</query></delete>"
        
-
        Commit can be an expensive operation so it's best to make many changes to an index in a batch and
-      then send the commit command at the end.  There is also an optimize command that does the same thing as commit,
-      in addition to merging all index segments into a single segment, making it faster to search and causing any
-      deleted documents to be removed.  All of the update commands are documented here.
-    
        
+
+
        
+Now re-execute the previous search 
+and verify that no matching documents are found.  You can also revisit the
+statistics page and observe the changes to both the number of commits in the updateHandler and the numDocs in the searcher.
+
        
+
+
        
+Commits that open a new searcher can be expensive operations so it's best to 
+make many changes to an index in a batch and then send the 
+commit command at the end.  
+There is also an optimize command that does the 
+same things as commit, but also forces all index 
+segments to be merged into a single segment -- this can be very resource 
+intsenive, but may be worthwhile for improving search speed if your index 
+changes very infrequently.
+
        
+
        
+All of the update commands can be specified using either XML or JSON.
+
        
+
 
        To continue with the tutorial, re-add any documents you may have deleted by going to the exampledocs directory and executing
        
 
        java -jar post.jar *.xml
        
 
@@ -293,32 +327,34 @@ in subsequent searches.
 
        
 
        
     Searches are done via HTTP GET on the select URL with the query string in the q parameter.
-    You can pass a number of optional request parameters
+    You can pass a number of optional request parameters
     to the request handler to control what information is returned.  For example, you can use the "fl" parameter
     to control what stored fields are returned, and if the relevancy score is returned:
   
        
 
 
        
-    Solr provides a query form within the web admin interface
-    that allows setting the various request parameters and is useful when testing or debugging queries.
-  
        
+The query form 
+provided in the web admin interface allows setting various request parameters 
+and is useful when testing or debugging queries.
+
        
+
 
 
        Sorting
        
 
        
@@ -328,15 +364,15 @@ in subsequent searches.
 
        
@@ -346,11 +382,11 @@ in subsequent searches.
 
@@ -360,7 +396,7 @@ in subsequent searches.
 
@@ -387,7 +423,7 @@ in subsequent searches.
   
        
 
        
     
-...&q=video card&fl=name,id&hl=true&hl.fl=name,features
+...&q=video card&fl=name,id&hl=true&hl.fl=name,features
   
 
        
 
        
@@ -412,7 +448,7 @@ in subsequent searches.
   
        
 
        
     
-...&q=*:*&facet=true&facet.field=cat
+...&q=*:*&facet=true&facet.field=cat
   
 
        
 
        
@@ -425,7 +461,7 @@ in subsequent searches.
   
        
 
        
     
-...&q=*:*&facet=true&facet.field=cat&facet.field=inStock
+...&q=*:*&facet=true&facet.field=cat&facet.field=inStock
   
 
        
 
        
@@ -435,15 +471,15 @@ in subsequent searches.
   
        
 
        
     
-...&q=ipod&facet=true&facet.query=price:[0 TO 100]&facet.query=price:[100 TO *]
+...&q=ipod&facet=true&facet.query=price:[0 TO 100]&facet.query=price:[100 TO *]
   
 
        
 
        
-    One can even facet by date ranges.  This example requests counts for the manufacture date (manufacturedate_dt field) for each year between 2004 and 2010.
+    Solr can even facet by numeric ranges (including dates).  This example requests counts for the manufacture date (manufacturedate_dt field) for each year between 2004 and 2010.
   
        
 
        
     
-...&q=*:*&facet=true&facet.date=manufacturedate_dt&facet.date.start=2004-01-01T00:00:00Z&facet.date.end=2010-01-01T00:00:00Z&facet.date.gap=+1YEAR
+...&q=*:*&facet=true&facet.rage=manufacturedate_dt&facet.rage.start=2004-01-01T00:00:00Z&facet.rage.end=2010-01-01T00:00:00Z&facet.range.gap=+1YEAR
   
 
        
 
        
@@ -461,13 +497,13 @@ in subsequent searches.
 
        Search UI
        
 
        
 
        
-    Solr includes an example search interface built with velocity templating
-    that demonstrates many features, including searching, faceting, highlighting,
-    autocomplete, and geospatial searching. 
-  
        
+Solr includes an example search interface built with velocity templating
+that demonstrates many features, including searching, faceting, highlighting,
+autocomplete, and geospatial searching. 
+
        
 
        
-    Try it out at
-    http://localhost:8983/solr/browse
+Try it out at
+http://localhost:8983/solr/collection1/browse
   
 
        
 
         
@@ -485,19 +521,24 @@ in subsequent searches.
   
        
 
        
     The schema defines
-    the fields in the index and what type of analysis is applied to them.  The current schema your server is using
-    may be accessed via the [SCHEMA] link on the admin page.
-  
        
+    the fields in the index and what type of analysis is applied to them.  The current schema your collection is using
+    may be viewed directly via the Schema tab in the Admin UI, or explored dynamicly using the Schema Browser tab.
+
        
 
        
-    The best analysis components (tokenization and filtering) for your textual content depends heavily on language.
-    As you can see in the above [SCHEMA] link, the fields in the example schema are using a fieldType
-    named text_general, which has defaults appropriate for all languages.
-  
        
+The best analysis components (tokenization and filtering) for your textual 
+content depends heavily on language.
+As you can see in the Schema Browser, 
+many of the fields in the example schema are using a 
+fieldType named 
+text_general, which has defaults appropriate for 
+most languages.
+
        
+
 
        
   If you know your textual content is English, as is the case for the example 
   documents in this tutorial, and you'd like to apply English-specific stemming
   and stop word removal, as well as split compound words, you can use the 
-  text_en_splitting fieldType instead.
+  text_en_splitting fieldType instead.
   Go ahead and edit the schema.xml in the 
   solr/example/solr/conf directory,
   to use the text_en_splitting fieldType for 
@@ -519,25 +560,25 @@ in subsequent searches.
 
          
     
 
        • A search for
-  power-shot
+  power-shot
   can match PowerShot, and
-  adata
+  adata
   can match A-DATA by using the 
   WordDelimiterFilter and LowerCaseFilter.
 
          • 
 
     
 
        • A search for
-  features:recharging
+  features:recharging
   can match Rechargeable using the stemming 
   features of PorterStemFilter.
 
          • 
 
     
 
        • A search for
-  "1 gigabyte"
+  "1 gigabyte"
   can match 1GB, and the commonly misspelled
-  pixima can matches Pixma using the 
+  pixima can matches Pixma using the 
   SynonymFilter.
 
          • 
 
@@ -547,53 +588,52 @@ in subsequent searches.
     available for use is here.
   
          
 
+
+
 
          Analysis Debugging
          
 
          
-  There is a handy analysis
-  debugging page where you can see how a text value is broken down into words,
-  and shows the resulting tokens after they pass through each filter in the chain.
+There is a handy Analysis tab
+where you can see how a text value is broken down into words by both Index time nad Query time analysis chains for a field or field type.  This page shows the resulting tokens after they pass through each filter in the chains.
 
          
 
          
-  This
-  url shows how "Canon Power-Shot SD500" would 
-  shows the tokens that would be instead be created using the 
-  text_en_splitting type.  Each row of
+  This url
+  shows the tokens created from 
+  "Canon Power-Shot SD500" 
+  using the 
+  text_en_splitting type.  Each section of
   the table shows the resulting tokens after having passed through the next 
-  TokenFilter in the analyzer.
+  TokenFilter in the (Index) analyzer.
   Notice how both powershot and 
   power, shot 
-  are indexed.  Tokens generated at the same position
-  are shown in the same column, in this case 
-  shot and 
-  powershot.  (Compare the previous output with
-  The tokens produced using the text_general field type.)
+  are indexed, using tokens that have the same "position".
+  (Compare the previous output with
+  The tokens produced using the text_general field type.)
 
          
+
 
          
-  Selecting verbose output
-  will show more details, such as the name of each analyzer component in the 
-  chain, token positions, and the start and end positions of the token in 
-  the original text.
+Mousing over the section label to the left of the section will display the full name of the analyzer component at that stage of the chain.  Toggling the "Verbose Output" checkbox will show/hide the detailed token attributes.
 
          
 
          
-  Selecting highlight matches
-  when both index and query values are provided will take the resulting 
-  terms from the query value and highlight
-  all matches in the index value analysis.
+When both Index and Query
+values are provided, two tables will be displayed side by side showing the 
+results of each chain.  Terms in the Index chain results that are equivilent 
+to the final terms produced by the Query chain will be highlighted.
 
          
 
          
   Other interesting examples:
 
          
 
diff --git a/solr/core/src/java/org/apache/solr/SolrLogFormatter.java b/solr/core/src/java/org/apache/solr/SolrLogFormatter.java
index 7f07304..399a28e 100644
--- a/solr/core/src/java/org/apache/solr/SolrLogFormatter.java
+++ b/solr/core/src/java/org/apache/solr/SolrLogFormatter.java
@@ -1,6 +1,6 @@
 package org.apache.solr;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -19,13 +19,14 @@ package org.apache.solr;
 
 import org.apache.solr.cloud.ZkController;
 import org.apache.solr.common.SolrException;
-import org.apache.solr.common.cloud.CloudState;
-import org.apache.solr.common.cloud.CoreState;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.request.SolrRequestInfo;
 import org.slf4j.LoggerFactory;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.WeakHashMap;
@@ -94,15 +95,11 @@ public class SolrLogFormatter extends Formatter {
     methodAlias.put(new Method("org.apache.solr.update.processor.LogUpdateProcessor","finish"), "");
   }
 
-
-
-
   public static class CoreInfo {
-    public static int maxCoreNum;
-    public String shortId;
-    public String url;
-    CoreState coreState;  // should be fine to keep a hard reference to this
-    // CloudState cloudState;  // should be fine to keep this hard reference since cloudstate is immutable and doesn't have pointers to anything heavyweight (like SolrCore, CoreContainer, etc)
+    static int maxCoreNum;
+    String shortId;
+    String url;
+    Map coreProps;
   }
 
   Map coreInfoMap = new WeakHashMap();    // TODO: use something that survives across a core reload?
@@ -199,11 +196,15 @@ sb.append("(group_name=").append(tg.getName()).append(")");
           sb.append(" url="+info.url + " node="+zkController.getNodeName());
         }
 
-        // look to see if local core state changed
-        CoreState coreState = zkController.getCoreState(core.getName());
-        if (coreState != info.coreState) {
-          sb.append(" " + info.shortId + "_STATE=" + coreState);
-          info.coreState = coreState;
+        if(info.coreProps == null) {
+          info.coreProps = getCoreProps(zkController, core);
+        }
+
+        Map coreProps = getCoreProps(zkController, core);
+        if(!coreProps.equals(info.coreProps)) {
+          info.coreProps = coreProps;
+          final String corePropsString = "coll:" + core.getCoreDescriptor().getCloudDescriptor().getCollectionName() + " core:" + core.getName() + " props:" + coreProps;
+          sb.append(" " + info.shortId + "_STATE=" + corePropsString);
         }
       }
     }
@@ -260,6 +261,16 @@ sb.append("(group_name=").append(tg.getName()).append(")");
     return sb.toString();
   }
 
+  private Map getCoreProps(ZkController zkController, SolrCore core) {
+    final String collection = core.getCoreDescriptor().getCloudDescriptor().getCollectionName();
+    ZkNodeProps props = zkController.getCloudState().getShardProps(collection,  ZkStateReader.getCoreNodeName(zkController.getNodeName(), core.getName()));
+    if(props!=null) {
+      return props.getProperties(); 
+    }
+    return Collections.EMPTY_MAP;
+  }
+
+
   private Method classAndMethod = new Method(null,null); // don't need to be thread safe
   private String getShortClassName(String name, String method) {
     classAndMethod.className = name;
diff --git a/solr/core/src/java/org/apache/solr/analysis/ASCIIFoldingFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/ASCIIFoldingFilterFactory.java
index 0061b25..da507d9 100644
--- a/solr/core/src/java/org/apache/solr/analysis/ASCIIFoldingFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/ASCIIFoldingFilterFactory.java
@@ -1,5 +1,5 @@
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/ArabicLetterTokenizerFactory.java b/solr/core/src/java/org/apache/solr/analysis/ArabicLetterTokenizerFactory.java
deleted file mode 100644
index 60a852c..0000000
--- a/solr/core/src/java/org/apache/solr/analysis/ArabicLetterTokenizerFactory.java
+++ /dev/null
@@ -1,47 +0,0 @@
-package org.apache.solr.analysis;
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.lucene.analysis.ar.ArabicLetterTokenizer;
-import org.apache.lucene.analysis.util.TokenizerFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Reader;
-import java.util.Map;
-
-
-/**
- * Factory for {@link ArabicLetterTokenizer}
- * @deprecated (3.1) Use StandardTokenizerFactory instead.
- **/
-@Deprecated
-public class ArabicLetterTokenizerFactory extends TokenizerFactory {
-
-  private static final Logger log = LoggerFactory.getLogger(ArabicLetterTokenizerFactory.class);
-
-  @Override
-  public void init(Map args) {
-    super.init(args);
-    assureMatchVersion();
-    log.warn(getClass().getSimpleName() + " is deprecated. Use StandardTokenizeFactory instead.");
-  }
-
-  public ArabicLetterTokenizer create(Reader input) {
-    return new ArabicLetterTokenizer(luceneMatchVersion, input);
-  }
-}
diff --git a/solr/core/src/java/org/apache/solr/analysis/ArabicNormalizationFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/ArabicNormalizationFilterFactory.java
index 3792899..97fc662 100644
--- a/solr/core/src/java/org/apache/solr/analysis/ArabicNormalizationFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/ArabicNormalizationFilterFactory.java
@@ -1,5 +1,5 @@
 package org.apache.solr.analysis;
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/ArabicStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/ArabicStemFilterFactory.java
index 08e0400..0baa6b5 100644
--- a/solr/core/src/java/org/apache/solr/analysis/ArabicStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/ArabicStemFilterFactory.java
@@ -1,5 +1,5 @@
 package org.apache.solr.analysis;
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/BeiderMorseFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/BeiderMorseFilterFactory.java
index e8ed237..82caa4a 100644
--- a/solr/core/src/java/org/apache/solr/analysis/BeiderMorseFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/BeiderMorseFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/BrazilianStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/BrazilianStemFilterFactory.java
index d1716b6..8928d64 100644
--- a/solr/core/src/java/org/apache/solr/analysis/BrazilianStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/BrazilianStemFilterFactory.java
@@ -1,5 +1,5 @@
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/BulgarianStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/BulgarianStemFilterFactory.java
index 1676a93..5c91995 100644
--- a/solr/core/src/java/org/apache/solr/analysis/BulgarianStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/BulgarianStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/CJKBigramFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/CJKBigramFilterFactory.java
index 54025c3..bea0bff 100644
--- a/solr/core/src/java/org/apache/solr/analysis/CJKBigramFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/CJKBigramFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/CJKTokenizerFactory.java b/solr/core/src/java/org/apache/solr/analysis/CJKTokenizerFactory.java
deleted file mode 100644
index 8e2e7c7..0000000
--- a/solr/core/src/java/org/apache/solr/analysis/CJKTokenizerFactory.java
+++ /dev/null
@@ -1,43 +0,0 @@
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.solr.analysis;
-
-import org.apache.lucene.analysis.cjk.CJKTokenizer;
-import org.apache.lucene.analysis.util.TokenizerFactory;
-
-import java.io.Reader;
-
-/** 
- * Factory for {@link CJKTokenizer}. 
- * 
          - * <fieldType name="text_cjk" class="solr.TextField" positionIncrementGap="100">
- *   <analyzer>
- *     <tokenizer class="solr.CJKTokenizerFactory"/>
- *   </analyzer>
- * </fieldType>
          
- * @deprecated
- */
-@Deprecated
-public class CJKTokenizerFactory extends TokenizerFactory {
-  public CJKTokenizer create(Reader in) {
-    return new CJKTokenizer(in);
-  }
-}
-
diff --git a/solr/core/src/java/org/apache/solr/analysis/CJKWidthFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/CJKWidthFilterFactory.java
index 84f1137..3213b3c 100644
--- a/solr/core/src/java/org/apache/solr/analysis/CJKWidthFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/CJKWidthFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/CapitalizationFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/CapitalizationFilterFactory.java
index 1dd96fd..e26bac2 100644
--- a/solr/core/src/java/org/apache/solr/analysis/CapitalizationFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/CapitalizationFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/ChineseFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/ChineseFilterFactory.java
deleted file mode 100644
index 002ee3e..0000000
--- a/solr/core/src/java/org/apache/solr/analysis/ChineseFilterFactory.java
+++ /dev/null
@@ -1,48 +0,0 @@
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.solr.analysis;
-import java.util.Map;
-
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.cn.ChineseFilter;
-import org.apache.lucene.analysis.util.TokenFilterFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Factory for {@link ChineseFilter}
- * @deprecated Use {@link StopFilterFactory} instead.
- */
-@Deprecated
-public class ChineseFilterFactory extends TokenFilterFactory {
-
-  private static final Logger log = LoggerFactory.getLogger(ChineseFilterFactory.class);
-
-  @Override
-  public void init(Map args) {
-    super.init(args);
-    log.warn(getClass().getSimpleName() + " is deprecated. Use StopFilterFactory instead.");
-  }
-  
-  public ChineseFilter create(TokenStream in) {
-    return new ChineseFilter(in);
-  }
-}
-
diff --git a/solr/core/src/java/org/apache/solr/analysis/ChineseTokenizerFactory.java b/solr/core/src/java/org/apache/solr/analysis/ChineseTokenizerFactory.java
deleted file mode 100644
index 27e12cf..0000000
--- a/solr/core/src/java/org/apache/solr/analysis/ChineseTokenizerFactory.java
+++ /dev/null
@@ -1,49 +0,0 @@
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.solr.analysis;
-
-import java.io.Reader;
-import java.util.Map;
-
-import org.apache.lucene.analysis.cn.ChineseTokenizer;
-import org.apache.lucene.analysis.util.TokenizerFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/** 
- * Factory for {@link ChineseTokenizer}
- * @deprecated Use {@link StandardTokenizerFactory} instead.
- */
-@Deprecated
-public class ChineseTokenizerFactory extends TokenizerFactory {
-
-  private static final Logger log = LoggerFactory.getLogger(ChineseTokenizerFactory.class);
-
-  @Override
-  public void init(Map args) {
-    super.init(args);
-    log.warn(getClass().getSimpleName() + " is deprecated. Use StandardTokenizeFactory instead.");
-  }
-  
-  public ChineseTokenizer create(Reader in) {
-    return new ChineseTokenizer(in);
-  }
-}
-
diff --git a/solr/core/src/java/org/apache/solr/analysis/ClassicFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/ClassicFilterFactory.java
index 180c97f..12ae66f 100644
--- a/solr/core/src/java/org/apache/solr/analysis/ClassicFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/ClassicFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/ClassicTokenizerFactory.java b/solr/core/src/java/org/apache/solr/analysis/ClassicTokenizerFactory.java
index f422ae3..4efe6e7 100644
--- a/solr/core/src/java/org/apache/solr/analysis/ClassicTokenizerFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/ClassicTokenizerFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/CollationKeyFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/CollationKeyFilterFactory.java
deleted file mode 100644
index 8d412a7..0000000
--- a/solr/core/src/java/org/apache/solr/analysis/CollationKeyFilterFactory.java
+++ /dev/null
@@ -1,175 +0,0 @@
-package org.apache.solr.analysis;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.text.Collator;
-import java.text.ParseException;
-import java.text.RuleBasedCollator;
-import java.util.Locale;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.*;
-import org.apache.lucene.collation.CollationKeyFilter;
-
-/**
- * Factory for {@link CollationKeyFilter}.
- * 
          
- * This factory can be created in two ways: 
- * 
            
- *  
          • Based upon a system collator associated with a Locale.
- *  
          • Based upon a tailored ruleset.
- * 
          
- * 
          
- * Using a System collator:
- * 
            
- *  
          • language: ISO-639 language code (mandatory)
- *  
          • country: ISO-3166 country code (optional)
- *  
          • variant: vendor or browser-specific code (optional)
- *  
          • strength: 'primary','secondary','tertiary', or 'identical' (optional)
- *  
          • decomposition: 'no','canonical', or 'full' (optional)
- * 
          
- * 
          
- * Using a Tailored ruleset:
- * 
            
- *  
          • custom: UTF-8 text file containing rules supported by RuleBasedCollator (mandatory)
- *  
          • strength: 'primary','secondary','tertiary', or 'identical' (optional)
- *  
          • decomposition: 'no','canonical', or 'full' (optional)
- * 
          
- * 
- * 
          - * <fieldType name="text_clltnky" class="solr.TextField" positionIncrementGap="100">
- *   <analyzer>
- *     <tokenizer class="solr.KeywordTokenizerFactory"/>
- *     <filter class="solr.CollationKeyFilterFactory" language="ja" country="JP"/>
- *   </analyzer>
- * </fieldType>
          
- * 
- * @see Collator
- * @see Locale
- * @see RuleBasedCollator
- * @since solr 3.1
- * @deprecated use {@link org.apache.solr.schema.CollationField} instead.
- */
-@Deprecated
-public class CollationKeyFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent, ResourceLoaderAware {
-  private Collator collator;
-
-  public void inform(ResourceLoader loader) {
-    String custom = args.get("custom");
-    String language = args.get("language");
-    String country = args.get("country");
-    String variant = args.get("variant");
-    String strength = args.get("strength");
-    String decomposition = args.get("decomposition");
-    
-    if (custom == null && language == null)
-      throw new InitializationException("Either custom or language is required.");
-    
-    if (custom != null && 
-        (language != null || country != null || variant != null))
-      throw new InitializationException("Cannot specify both language and custom. "
-          + "To tailor rules for a built-in language, see the javadocs for RuleBasedCollator. "
-          + "Then save the entire customized ruleset to a file, and use with the custom parameter");
-    
-    if (language != null) { 
-      // create from a system collator, based on Locale.
-      collator = createFromLocale(language, country, variant);
-    } else { 
-      // create from a custom ruleset
-      collator = createFromRules(custom, loader);
-    }
-    
-    // set the strength flag, otherwise it will be the default.
-    if (strength != null) {
-      if (strength.equalsIgnoreCase("primary"))
-        collator.setStrength(Collator.PRIMARY);
-      else if (strength.equalsIgnoreCase("secondary"))
-        collator.setStrength(Collator.SECONDARY);
-      else if (strength.equalsIgnoreCase("tertiary"))
-        collator.setStrength(Collator.TERTIARY);
-      else if (strength.equalsIgnoreCase("identical"))
-        collator.setStrength(Collator.IDENTICAL);
-      else
-        throw new InitializationException("Invalid strength: " + strength);
-    }
-    
-    // set the decomposition flag, otherwise it will be the default.
-    if (decomposition != null) {
-      if (decomposition.equalsIgnoreCase("no"))
-        collator.setDecomposition(Collator.NO_DECOMPOSITION);
-      else if (decomposition.equalsIgnoreCase("canonical"))
-        collator.setDecomposition(Collator.CANONICAL_DECOMPOSITION);
-      else if (decomposition.equalsIgnoreCase("full"))
-        collator.setDecomposition(Collator.FULL_DECOMPOSITION);
-      else
-        throw new InitializationException("Invalid decomposition: " + decomposition);
-    }
-  }
-  
-  public TokenStream create(TokenStream input) {
-    return new CollationKeyFilter(input, collator);
-  }
-  
-  /*
-   * Create a locale from language, with optional country and variant.
-   * Then return the appropriate collator for the locale.
-   */
-  private Collator createFromLocale(String language, String country, String variant) {
-    Locale locale;
-    
-    if (language != null && country == null && variant != null)
-      throw new InitializationException("To specify variant, country is required");
-    else if (language != null && country != null && variant != null)
-      locale = new Locale(language, country, variant);
-    else if (language != null && country != null)
-      locale = new Locale(language, country);
-    else 
-      locale = new Locale(language);
-    
-    return Collator.getInstance(locale);
-  }
-  
-  /*
-   * Read custom rules from a file, and create a RuleBasedCollator
-   * The file cannot support comments, as # might be in the rules!
-   */
-  private Collator createFromRules(String fileName, ResourceLoader loader) {
-    InputStream input = null;
-    try {
-     input = loader.openResource(fileName);
-     String rules = IOUtils.toString(input, "UTF-8");
-     return new RuleBasedCollator(rules);
-    } catch (IOException e) {
-      // io error
-      throw new InitializationException("IOException thrown while loading rules", e);
-    } catch (ParseException e) {
-      // invalid rules
-      throw new InitializationException("ParseException thrown while parsing rules", e);
-    } finally {
-      IOUtils.closeQuietly(input);
-    }
-  }
-  
-  @Override
-  public AbstractAnalysisFactory getMultiTermComponent() {
-    return this;
-  }
-}
diff --git a/solr/core/src/java/org/apache/solr/analysis/CommonGramsFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/CommonGramsFilterFactory.java
index 78e6fdc..f32df81 100644
--- a/solr/core/src/java/org/apache/solr/analysis/CommonGramsFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/CommonGramsFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/CommonGramsQueryFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/CommonGramsQueryFilterFactory.java
index 5ba568c..38e407f 100644
--- a/solr/core/src/java/org/apache/solr/analysis/CommonGramsQueryFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/CommonGramsQueryFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/CzechStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/CzechStemFilterFactory.java
index 259fe72..fd35534 100644
--- a/solr/core/src/java/org/apache/solr/analysis/CzechStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/CzechStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/DelimitedPayloadTokenFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/DelimitedPayloadTokenFilterFactory.java
index 93599e6..007b83d 100644
--- a/solr/core/src/java/org/apache/solr/analysis/DelimitedPayloadTokenFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/DelimitedPayloadTokenFilterFactory.java
@@ -1,5 +1,5 @@
 package org.apache.solr.analysis;
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/DictionaryCompoundWordTokenFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/DictionaryCompoundWordTokenFilterFactory.java
index e20821e..4664dc3 100644
--- a/solr/core/src/java/org/apache/solr/analysis/DictionaryCompoundWordTokenFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/DictionaryCompoundWordTokenFilterFactory.java
@@ -1,5 +1,5 @@
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/DoubleMetaphoneFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/DoubleMetaphoneFilterFactory.java
index 42a7e60..87042b9 100644
--- a/solr/core/src/java/org/apache/solr/analysis/DoubleMetaphoneFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/DoubleMetaphoneFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/EdgeNGramFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/EdgeNGramFilterFactory.java
index e784e8e..5ac1886 100644
--- a/solr/core/src/java/org/apache/solr/analysis/EdgeNGramFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/EdgeNGramFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/EdgeNGramTokenizerFactory.java b/solr/core/src/java/org/apache/solr/analysis/EdgeNGramTokenizerFactory.java
index 02d3be7..83f39de 100755
--- a/solr/core/src/java/org/apache/solr/analysis/EdgeNGramTokenizerFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/EdgeNGramTokenizerFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/ElisionFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/ElisionFilterFactory.java
index b2fc8c3..c522ed4 100644
--- a/solr/core/src/java/org/apache/solr/analysis/ElisionFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/ElisionFilterFactory.java
@@ -1,5 +1,5 @@
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/EnglishMinimalStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/EnglishMinimalStemFilterFactory.java
index 4bf2144..ed31a86 100644
--- a/solr/core/src/java/org/apache/solr/analysis/EnglishMinimalStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/EnglishMinimalStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/EnglishPossessiveFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/EnglishPossessiveFilterFactory.java
index 8f28e4b..36d153d 100644
--- a/solr/core/src/java/org/apache/solr/analysis/EnglishPossessiveFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/EnglishPossessiveFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/FSTSynonymFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/FSTSynonymFilterFactory.java
deleted file mode 100644
index 495c448..0000000
--- a/solr/core/src/java/org/apache/solr/analysis/FSTSynonymFilterFactory.java
+++ /dev/null
@@ -1,168 +0,0 @@
-package org.apache.solr.analysis;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.nio.charset.Charset;
-import java.nio.charset.CharsetDecoder;
-import java.nio.charset.CodingErrorAction;
-import java.text.ParseException;
-import java.util.List;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.LowerCaseFilter;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.lucene.analysis.synonym.SynonymFilter;
-import org.apache.lucene.analysis.synonym.SynonymMap;
-import org.apache.lucene.analysis.synonym.SolrSynonymParser;
-import org.apache.lucene.analysis.synonym.WordnetSynonymParser;
-import org.apache.lucene.analysis.util.*;
-import org.apache.lucene.util.Version;
-import org.apache.solr.common.util.StrUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @deprecated (3.4) use {@link SynonymFilterFactory} instead. this is only a backwards compatibility
- *                   mechanism that will be removed in Lucene 5.0
- */
-// NOTE: rename this to "SynonymFilterFactory" and nuke that delegator in Lucene 5.0!
-@Deprecated
-final class FSTSynonymFilterFactory extends TokenFilterFactory implements ResourceLoaderAware {
-
-  public static final Logger log = LoggerFactory.getLogger(FSTSynonymFilterFactory.class);
-
-  private SynonymMap map;
-  private boolean ignoreCase;
-  
-  @Override
-  public TokenStream create(TokenStream input) {
-    // if the fst is null, it means there's actually no synonyms... just return the original stream
-    // as there is nothing to do here.
-    return map.fst == null ? input : new SynonymFilter(input, map, ignoreCase);
-  }
-
-  @Override
-  public void inform(ResourceLoader loader) {
-    final boolean ignoreCase = getBoolean("ignoreCase", false); 
-    this.ignoreCase = ignoreCase;
-
-    String tf = args.get("tokenizerFactory");
-
-    final TokenizerFactory factory = tf == null ? null : loadTokenizerFactory(loader, tf);
-    
-    Analyzer analyzer = new Analyzer() {
-      @Override
-      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
-        Tokenizer tokenizer = factory == null ? new WhitespaceTokenizer(Version.LUCENE_31, reader) : factory.create(reader);
-        TokenStream stream = ignoreCase ? new LowerCaseFilter(Version.LUCENE_31, tokenizer) : tokenizer;
-        return new TokenStreamComponents(tokenizer, stream);
-      }
-    };
-
-    String format = args.get("format");
-    try {
-      if (format == null || format.equals("solr")) {
-        // TODO: expose dedup as a parameter?
-        map = loadSolrSynonyms(loader, true, analyzer);
-      } else if (format.equals("wordnet")) {
-        map = loadWordnetSynonyms(loader, true, analyzer);
-      } else {
-        // TODO: somehow make this more pluggable
-        throw new InitializationException("Unrecognized synonyms format: " + format);
-      }
-    } catch (Exception e) {
-      throw new InitializationException("Exception thrown while loading synonyms", e);
-    }
-    
-    if (map.fst == null) {
-      log.warn("Synonyms loaded with " + args + " has empty rule set!");
-    }
-  }
-  
-  /**
-   * Load synonyms from the solr format, "format=solr".
-   */
-  private SynonymMap loadSolrSynonyms(ResourceLoader loader, boolean dedup, Analyzer analyzer) throws IOException, ParseException {
-    final boolean expand = getBoolean("expand", true);
-    String synonyms = args.get("synonyms");
-    if (synonyms == null)
-      throw new InitializationException("Missing required argument 'synonyms'.");
-    
-    CharsetDecoder decoder = Charset.forName("UTF-8").newDecoder()
-      .onMalformedInput(CodingErrorAction.REPORT)
-      .onUnmappableCharacter(CodingErrorAction.REPORT);
-    
-    SolrSynonymParser parser = new SolrSynonymParser(dedup, expand, analyzer);
-    File synonymFile = new File(synonyms);
-    if (synonymFile.exists()) {
-      decoder.reset();
-      parser.add(new InputStreamReader(loader.openResource(synonyms), decoder));
-    } else {
-      List files = StrUtils.splitFileNames(synonyms);
-      for (String file : files) {
-        decoder.reset();
-        parser.add(new InputStreamReader(loader.openResource(file), decoder));
-      }
-    }
-    return parser.build();
-  }
-  
-  /**
-   * Load synonyms from the wordnet format, "format=wordnet".
-   */
-  private SynonymMap loadWordnetSynonyms(ResourceLoader loader, boolean dedup, Analyzer analyzer) throws IOException, ParseException {
-    final boolean expand = getBoolean("expand", true);
-    String synonyms = args.get("synonyms");
-    if (synonyms == null)
-      throw new InitializationException("Missing required argument 'synonyms'.");
-    
-    CharsetDecoder decoder = Charset.forName("UTF-8").newDecoder()
-      .onMalformedInput(CodingErrorAction.REPORT)
-      .onUnmappableCharacter(CodingErrorAction.REPORT);
-    
-    WordnetSynonymParser parser = new WordnetSynonymParser(dedup, expand, analyzer);
-    File synonymFile = new File(synonyms);
-    if (synonymFile.exists()) {
-      decoder.reset();
-      parser.add(new InputStreamReader(loader.openResource(synonyms), decoder));
-    } else {
-      List files = StrUtils.splitFileNames(synonyms);
-      for (String file : files) {
-        decoder.reset();
-        parser.add(new InputStreamReader(loader.openResource(file), decoder));
-      }
-    }
-    return parser.build();
-  }
-  
-  private TokenizerFactory loadTokenizerFactory(ResourceLoader loader, String cname){
-    TokenizerFactory tokFactory = loader.newInstance(cname, TokenizerFactory.class);
-    tokFactory.setLuceneMatchVersion(luceneMatchVersion);
-    tokFactory.init(args);
-    if (tokFactory instanceof ResourceLoaderAware) {
-      ((ResourceLoaderAware) tokFactory).inform(loader);
-    }
-    return tokFactory;
-  }
-}
diff --git a/solr/core/src/java/org/apache/solr/analysis/FinnishLightStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/FinnishLightStemFilterFactory.java
index 3c4ce08..0e70606 100644
--- a/solr/core/src/java/org/apache/solr/analysis/FinnishLightStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/FinnishLightStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/FrenchLightStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/FrenchLightStemFilterFactory.java
index 53f361d..8b50110 100644
--- a/solr/core/src/java/org/apache/solr/analysis/FrenchLightStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/FrenchLightStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/FrenchMinimalStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/FrenchMinimalStemFilterFactory.java
index 4c1715f..d3d3bf9 100644
--- a/solr/core/src/java/org/apache/solr/analysis/FrenchMinimalStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/FrenchMinimalStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/GalicianMinimalStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/GalicianMinimalStemFilterFactory.java
index 7744477..4f838ab 100644
--- a/solr/core/src/java/org/apache/solr/analysis/GalicianMinimalStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/GalicianMinimalStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/GalicianStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/GalicianStemFilterFactory.java
index d529f5e..824a521 100644
--- a/solr/core/src/java/org/apache/solr/analysis/GalicianStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/GalicianStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/GermanLightStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/GermanLightStemFilterFactory.java
index 38bb1c4..60824ce 100644
--- a/solr/core/src/java/org/apache/solr/analysis/GermanLightStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/GermanLightStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/GermanMinimalStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/GermanMinimalStemFilterFactory.java
index 8905231..08a89da 100644
--- a/solr/core/src/java/org/apache/solr/analysis/GermanMinimalStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/GermanMinimalStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/GermanNormalizationFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/GermanNormalizationFilterFactory.java
index e344b3b..27ac20b 100644
--- a/solr/core/src/java/org/apache/solr/analysis/GermanNormalizationFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/GermanNormalizationFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/GermanStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/GermanStemFilterFactory.java
index 30951d2..1e7edbd 100644
--- a/solr/core/src/java/org/apache/solr/analysis/GermanStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/GermanStemFilterFactory.java
@@ -1,5 +1,5 @@
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/GreekLowerCaseFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/GreekLowerCaseFilterFactory.java
index f524eae..8889920 100644
--- a/solr/core/src/java/org/apache/solr/analysis/GreekLowerCaseFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/GreekLowerCaseFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/GreekStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/GreekStemFilterFactory.java
index 93862ef..d8dae6d 100644
--- a/solr/core/src/java/org/apache/solr/analysis/GreekStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/GreekStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/HTMLStripCharFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/HTMLStripCharFilterFactory.java
index 7d7be9e..bdc5e05 100644
--- a/solr/core/src/java/org/apache/solr/analysis/HTMLStripCharFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/HTMLStripCharFilterFactory.java
@@ -1,7 +1,7 @@
 package org.apache.solr.analysis;
 
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/HindiNormalizationFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/HindiNormalizationFilterFactory.java
index 8d1e1c5..6a11b19 100644
--- a/solr/core/src/java/org/apache/solr/analysis/HindiNormalizationFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/HindiNormalizationFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/HindiStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/HindiStemFilterFactory.java
index a172a8a..2e495e4 100644
--- a/solr/core/src/java/org/apache/solr/analysis/HindiStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/HindiStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/HungarianLightStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/HungarianLightStemFilterFactory.java
index 8412643..10aea57 100644
--- a/solr/core/src/java/org/apache/solr/analysis/HungarianLightStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/HungarianLightStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/HunspellStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/HunspellStemFilterFactory.java
index 004c3c1..ffb529e 100644
--- a/solr/core/src/java/org/apache/solr/analysis/HunspellStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/HunspellStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -40,7 +40,10 @@ import org.apache.lucene.analysis.util.TokenFilterFactory;
  * Both parameters dictionary and affix are mandatory.
  * 
          
  * The parameter ignoreCase (true/false) controls whether matching is case sensitive or not. Default false.
- * 
           
+ * 
          
+ * The parameter strictAffixParsing (true/false) controls whether the affix parsing is strict or not. Default true.
+ * If strict an error while reading an affix rule causes a ParseException, otherwise is ignored.
+ * 
          
  * Dictionaries for many languages are available through the OpenOffice project.
  * 
  * See http://wiki.apache.org/solr/Hunspell
@@ -50,6 +53,7 @@ public class HunspellStemFilterFactory extends TokenFilterFactory implements Res
   private static final String PARAM_DICTIONARY = "dictionary";
   private static final String PARAM_AFFIX = "affix";
   private static final String PARAM_IGNORE_CASE = "ignoreCase";
+  private static final String PARAM_STRICT_AFFIX_PARSING = "strictAffixParsing";
   private static final String TRUE = "true";
   private static final String FALSE = "false";
   
@@ -72,12 +76,21 @@ public class HunspellStemFilterFactory extends TokenFilterFactory implements Res
       else throw new InitializationException("Unknown value for " + PARAM_IGNORE_CASE + ": " + pic + ". Must be true or false");
     }
 
+
+    String strictAffixParsingParam = args.get(PARAM_STRICT_AFFIX_PARSING);
+    boolean strictAffixParsing = true;
+    if(strictAffixParsingParam != null) {
+      if(strictAffixParsingParam.equalsIgnoreCase(FALSE)) strictAffixParsing = false;
+      else if(strictAffixParsingParam.equalsIgnoreCase(TRUE)) strictAffixParsing = true;
+      else throw new InitializationException("Unknown value for " + PARAM_STRICT_AFFIX_PARSING + ": " + strictAffixParsingParam + ". Must be true or false");
+    }
+
     try {
       List dictionaries = new ArrayList();
       for (String file : dictionaryFiles) {
         dictionaries.add(loader.openResource(file));
       }
-      this.dictionary = new HunspellDictionary(loader.openResource(affixFile), dictionaries, luceneMatchVersion, ignoreCase);
+      this.dictionary = new HunspellDictionary(loader.openResource(affixFile), dictionaries, luceneMatchVersion, ignoreCase, strictAffixParsing);
     } catch (Exception e) {
       throw new InitializationException("Unable to load hunspell data! [dictionary=" + args.get("dictionary") + ",affix=" + affixFile + "]", e);
     }
diff --git a/solr/core/src/java/org/apache/solr/analysis/HyphenatedWordsFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/HyphenatedWordsFilterFactory.java
index 80dcadf..91faf90 100755
--- a/solr/core/src/java/org/apache/solr/analysis/HyphenatedWordsFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/HyphenatedWordsFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/HyphenationCompoundWordTokenFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/HyphenationCompoundWordTokenFilterFactory.java
index 709a548..d4a0da3 100644
--- a/solr/core/src/java/org/apache/solr/analysis/HyphenationCompoundWordTokenFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/HyphenationCompoundWordTokenFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/IndicNormalizationFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/IndicNormalizationFilterFactory.java
index e9cfb15..2709e75 100644
--- a/solr/core/src/java/org/apache/solr/analysis/IndicNormalizationFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/IndicNormalizationFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/IndonesianStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/IndonesianStemFilterFactory.java
index 1bae0dc..92b1d46 100644
--- a/solr/core/src/java/org/apache/solr/analysis/IndonesianStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/IndonesianStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/IrishLowerCaseFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/IrishLowerCaseFilterFactory.java
index 1028869..e5f1d85 100644
--- a/solr/core/src/java/org/apache/solr/analysis/IrishLowerCaseFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/IrishLowerCaseFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/ItalianLightStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/ItalianLightStemFilterFactory.java
index c895d42..7e79479 100644
--- a/solr/core/src/java/org/apache/solr/analysis/ItalianLightStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/ItalianLightStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/JapaneseBaseFormFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/JapaneseBaseFormFilterFactory.java
index 0ece745..600991f 100644
--- a/solr/core/src/java/org/apache/solr/analysis/JapaneseBaseFormFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/JapaneseBaseFormFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/JapaneseKatakanaStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/JapaneseKatakanaStemFilterFactory.java
index 09ffed7..602a5f1 100644
--- a/solr/core/src/java/org/apache/solr/analysis/JapaneseKatakanaStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/JapaneseKatakanaStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/JapanesePartOfSpeechStopFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/JapanesePartOfSpeechStopFilterFactory.java
index e2761b5..ce91b63 100644
--- a/solr/core/src/java/org/apache/solr/analysis/JapanesePartOfSpeechStopFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/JapanesePartOfSpeechStopFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/JapaneseReadingFormFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/JapaneseReadingFormFilterFactory.java
index 18bcf7b..ba0c541 100644
--- a/solr/core/src/java/org/apache/solr/analysis/JapaneseReadingFormFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/JapaneseReadingFormFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/JapaneseTokenizerFactory.java b/solr/core/src/java/org/apache/solr/analysis/JapaneseTokenizerFactory.java
index dd5aa90..cf30e97 100644
--- a/solr/core/src/java/org/apache/solr/analysis/JapaneseTokenizerFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/JapaneseTokenizerFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/KStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/KStemFilterFactory.java
index 42d2f0d..f8f4057 100644
--- a/solr/core/src/java/org/apache/solr/analysis/KStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/KStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/KeepWordFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/KeepWordFilterFactory.java
index 8545079..86a5ed1 100644
--- a/solr/core/src/java/org/apache/solr/analysis/KeepWordFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/KeepWordFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/KeywordMarkerFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/KeywordMarkerFilterFactory.java
index f2b55ad..3274a43 100644
--- a/solr/core/src/java/org/apache/solr/analysis/KeywordMarkerFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/KeywordMarkerFilterFactory.java
@@ -6,7 +6,7 @@ import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
 import org.apache.lucene.analysis.util.*;
 import org.apache.lucene.analysis.TokenStream;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/KeywordTokenizerFactory.java b/solr/core/src/java/org/apache/solr/analysis/KeywordTokenizerFactory.java
index 065f413..678dda5 100644
--- a/solr/core/src/java/org/apache/solr/analysis/KeywordTokenizerFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/KeywordTokenizerFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/LatvianStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/LatvianStemFilterFactory.java
index f008846..3fb4b26 100644
--- a/solr/core/src/java/org/apache/solr/analysis/LatvianStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/LatvianStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilter.java b/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilter.java
index 3deb84f..b81c679 100644
--- a/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilter.java
+++ b/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilter.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java
index 60c506b..09cc1f9 100644
--- a/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java
@@ -1,7 +1,7 @@
 package org.apache.solr.analysis;
 
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/LengthFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/LengthFilterFactory.java
index ad458e1..756b202 100644
--- a/solr/core/src/java/org/apache/solr/analysis/LengthFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/LengthFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/LetterTokenizerFactory.java b/solr/core/src/java/org/apache/solr/analysis/LetterTokenizerFactory.java
index 63c6ba4..fcd5691 100644
--- a/solr/core/src/java/org/apache/solr/analysis/LetterTokenizerFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/LetterTokenizerFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/LimitTokenCountFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/LimitTokenCountFilterFactory.java
index 0b4684c..0dfe9f9 100644
--- a/solr/core/src/java/org/apache/solr/analysis/LimitTokenCountFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/LimitTokenCountFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/LowerCaseFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/LowerCaseFilterFactory.java
index 016149b..f49e557 100644
--- a/solr/core/src/java/org/apache/solr/analysis/LowerCaseFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/LowerCaseFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/LowerCaseTokenizerFactory.java b/solr/core/src/java/org/apache/solr/analysis/LowerCaseTokenizerFactory.java
index 6997a1c..253d1ee 100644
--- a/solr/core/src/java/org/apache/solr/analysis/LowerCaseTokenizerFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/LowerCaseTokenizerFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/MappingCharFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/MappingCharFilterFactory.java
index 346132e..f68b800 100644
--- a/solr/core/src/java/org/apache/solr/analysis/MappingCharFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/MappingCharFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/NGramFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/NGramFilterFactory.java
index 2c9ae79..d21708b 100644
--- a/solr/core/src/java/org/apache/solr/analysis/NGramFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/NGramFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/NGramTokenizerFactory.java b/solr/core/src/java/org/apache/solr/analysis/NGramTokenizerFactory.java
index 9f6fc62..c2ed40d 100755
--- a/solr/core/src/java/org/apache/solr/analysis/NGramTokenizerFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/NGramTokenizerFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/NorwegianLightStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/NorwegianLightStemFilterFactory.java
index 674d971..840e726 100644
--- a/solr/core/src/java/org/apache/solr/analysis/NorwegianLightStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/NorwegianLightStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/NorwegianMinimalStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/NorwegianMinimalStemFilterFactory.java
index 3c8ab45..b8ec071 100644
--- a/solr/core/src/java/org/apache/solr/analysis/NorwegianMinimalStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/NorwegianMinimalStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/NumericPayloadTokenFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/NumericPayloadTokenFilterFactory.java
index 574c482..605df3e 100644
--- a/solr/core/src/java/org/apache/solr/analysis/NumericPayloadTokenFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/NumericPayloadTokenFilterFactory.java
@@ -1,5 +1,5 @@
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/PathHierarchyTokenizerFactory.java b/solr/core/src/java/org/apache/solr/analysis/PathHierarchyTokenizerFactory.java
index 2adfc83..6fd7f9e 100644
--- a/solr/core/src/java/org/apache/solr/analysis/PathHierarchyTokenizerFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/PathHierarchyTokenizerFactory.java
@@ -1,5 +1,5 @@
 package org.apache.solr.analysis;
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/PatternReplaceCharFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/PatternReplaceCharFilterFactory.java
index 0801893..2e9f2ef 100644
--- a/solr/core/src/java/org/apache/solr/analysis/PatternReplaceCharFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/PatternReplaceCharFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -19,20 +19,18 @@ package org.apache.solr.analysis;
 
 import java.util.Map;
 import java.util.regex.Pattern;
-import java.util.regex.PatternSyntaxException;
 
 import org.apache.lucene.analysis.CharStream;
 import org.apache.lucene.analysis.pattern.PatternReplaceCharFilter;
 import org.apache.lucene.analysis.util.CharFilterFactory;
-import org.apache.lucene.analysis.util.InitializationException;
 
 /**
  * Factory for {@link PatternReplaceCharFilter}. 
  * 
            * <fieldType name="text_ptnreplace" class="solr.TextField" positionIncrementGap="100">
  *   <analyzer>
- *     <charFilter class="solr.PatternReplaceCharFilterFactory" pattern="([^a-z])" replacement=""
- *                 maxBlockChars="10000" blockDelimiters="|"/>
+ *     <charFilter class="solr.PatternReplaceCharFilterFactory" 
+ *                    pattern="([^a-z])" replacement=""/>
  *     <tokenizer class="solr.KeywordTokenizerFactory"/>
  *   </analyzer>
  * </fieldType>
          
@@ -44,8 +42,6 @@ public class PatternReplaceCharFilterFactory extends CharFilterFactory {
   
   private Pattern p;
   private String replacement;
-  private int maxBlockChars;
-  private String blockDelimiters;
 
   @Override
   public void init(Map args) {
@@ -54,11 +50,10 @@ public class PatternReplaceCharFilterFactory extends CharFilterFactory {
     replacement = args.get( "replacement" );
     if( replacement == null )
       replacement = "";
-    maxBlockChars = getInt( "maxBlockChars", PatternReplaceCharFilter.DEFAULT_MAX_BLOCK_CHARS );
-    blockDelimiters = args.get( "blockDelimiters" );
+    // TODO: throw exception if you set maxBlockChars or blockDelimiters ?
   }
 
   public CharStream create(CharStream input) {
-    return new PatternReplaceCharFilter( p, replacement, maxBlockChars, blockDelimiters, input );
+    return new PatternReplaceCharFilter( p, replacement, input );
   }
 }
diff --git a/solr/core/src/java/org/apache/solr/analysis/PatternReplaceFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/PatternReplaceFilterFactory.java
index 40654b5..c71b794 100644
--- a/solr/core/src/java/org/apache/solr/analysis/PatternReplaceFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/PatternReplaceFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/PatternTokenizerFactory.java b/solr/core/src/java/org/apache/solr/analysis/PatternTokenizerFactory.java
index 5fac340..a11e9fd 100644
--- a/solr/core/src/java/org/apache/solr/analysis/PatternTokenizerFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/PatternTokenizerFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/PersianCharFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/PersianCharFilterFactory.java
index e136992..6bcadcd 100644
--- a/solr/core/src/java/org/apache/solr/analysis/PersianCharFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/PersianCharFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/PersianNormalizationFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/PersianNormalizationFilterFactory.java
index aedc4cc..a0ccd43 100644
--- a/solr/core/src/java/org/apache/solr/analysis/PersianNormalizationFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/PersianNormalizationFilterFactory.java
@@ -1,5 +1,5 @@
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/PhoneticFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/PhoneticFilterFactory.java
index c107b77..5ecfcc3 100644
--- a/solr/core/src/java/org/apache/solr/analysis/PhoneticFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/PhoneticFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/PorterStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/PorterStemFilterFactory.java
index 04b0dcb..c6dea10 100644
--- a/solr/core/src/java/org/apache/solr/analysis/PorterStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/PorterStemFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/PortugueseLightStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/PortugueseLightStemFilterFactory.java
index 89c3a05..63f8902 100644
--- a/solr/core/src/java/org/apache/solr/analysis/PortugueseLightStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/PortugueseLightStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/PortugueseMinimalStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/PortugueseMinimalStemFilterFactory.java
index 57f86d6..9148435 100644
--- a/solr/core/src/java/org/apache/solr/analysis/PortugueseMinimalStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/PortugueseMinimalStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/PortugueseStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/PortugueseStemFilterFactory.java
index 9b0f9b7..4ddb7d9 100644
--- a/solr/core/src/java/org/apache/solr/analysis/PortugueseStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/PortugueseStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/PositionFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/PositionFilterFactory.java
index 444503a..1fcc294 100644
--- a/solr/core/src/java/org/apache/solr/analysis/PositionFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/PositionFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/RemoveDuplicatesTokenFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/RemoveDuplicatesTokenFilterFactory.java
index 56aefb2..30d51bb 100644
--- a/solr/core/src/java/org/apache/solr/analysis/RemoveDuplicatesTokenFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/RemoveDuplicatesTokenFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/ReverseStringFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/ReverseStringFilterFactory.java
index d23127a..f090c9d 100644
--- a/solr/core/src/java/org/apache/solr/analysis/ReverseStringFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/ReverseStringFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/ReversedWildcardFilter.java b/solr/core/src/java/org/apache/solr/analysis/ReversedWildcardFilter.java
index 4a595e0..c1a9794 100644
--- a/solr/core/src/java/org/apache/solr/analysis/ReversedWildcardFilter.java
+++ b/solr/core/src/java/org/apache/solr/analysis/ReversedWildcardFilter.java
@@ -1,5 +1,5 @@
 package org.apache.solr.analysis;
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/ReversedWildcardFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/ReversedWildcardFilterFactory.java
index 22500fe..1e9e5dd 100644
--- a/solr/core/src/java/org/apache/solr/analysis/ReversedWildcardFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/ReversedWildcardFilterFactory.java
@@ -1,5 +1,5 @@
 package org.apache.solr.analysis;
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/RussianLetterTokenizerFactory.java b/solr/core/src/java/org/apache/solr/analysis/RussianLetterTokenizerFactory.java
deleted file mode 100644
index 340dfc0..0000000
--- a/solr/core/src/java/org/apache/solr/analysis/RussianLetterTokenizerFactory.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.analysis;
-
-import java.io.Reader;
-import java.util.Map;
-
-import org.apache.lucene.analysis.ru.RussianLetterTokenizer;
-import org.apache.lucene.analysis.util.InitializationException;
-import org.apache.lucene.analysis.util.TokenizerFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/** @deprecated Use {@link StandardTokenizerFactory} instead.
- *  This tokenizer has no Russian-specific functionality.
- */
-@Deprecated
-public class RussianLetterTokenizerFactory extends TokenizerFactory {
-
-  private static final Logger log = LoggerFactory.getLogger(RussianLetterTokenizerFactory.class);
-
-  @Override
-  public void init(Map args) {
-    super.init(args);
-    if (args.containsKey("charset"))
-      throw new InitializationException(
-          "The charset parameter is no longer supported.  "
-          + "Please process your documents as Unicode instead.");
-    assureMatchVersion();
-    log.warn(getClass().getSimpleName() + " is deprecated. Use StandardTokenizerFactory instead.");
-  }
-
-  public RussianLetterTokenizer create(Reader in) {
-    return new RussianLetterTokenizer(luceneMatchVersion,in);
-  }
-}
-
diff --git a/solr/core/src/java/org/apache/solr/analysis/RussianLightStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/RussianLightStemFilterFactory.java
index 0ff5d7c..6b0411b 100644
--- a/solr/core/src/java/org/apache/solr/analysis/RussianLightStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/RussianLightStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/ShingleFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/ShingleFilterFactory.java
index e1932d0..ba329a0 100644
--- a/solr/core/src/java/org/apache/solr/analysis/ShingleFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/ShingleFilterFactory.java
@@ -1,5 +1,5 @@
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/SlowSynonymFilter.java b/solr/core/src/java/org/apache/solr/analysis/SlowSynonymFilter.java
deleted file mode 100644
index d97cacd..0000000
--- a/solr/core/src/java/org/apache/solr/analysis/SlowSynonymFilter.java
+++ /dev/null
@@ -1,261 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.analysis;
-
-import org.apache.lucene.analysis.Token;
-import org.apache.lucene.analysis.TokenFilter;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
-import org.apache.lucene.util.AttributeSource;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.LinkedList;
-
-/** SynonymFilter handles multi-token synonyms with variable position increment offsets.
- * 
          
- * The matched tokens from the input stream may be optionally passed through (includeOrig=true)
- * or discarded.  If the original tokens are included, the position increments may be modified
- * to retain absolute positions after merging with the synonym tokenstream.
- * 
          
- * Generated synonyms will start at the same position as the first matched source token.
- * @deprecated (3.4) use {@link SynonymFilterFactory} instead. only for precise index backwards compatibility. this factory will be removed in Lucene 5.0
- */
-@Deprecated
-final class SlowSynonymFilter extends TokenFilter {
-
-  private final SlowSynonymMap map;  // Map
-  private Iterator replacement;  // iterator over generated tokens
-
-  public SlowSynonymFilter(TokenStream in, SlowSynonymMap map) {
-    super(in);
-    if (map == null)
-      throw new IllegalArgumentException("map is required");
-
-    this.map = map;
-    // just ensuring these attributes exist...
-    addAttribute(CharTermAttribute.class);
-    addAttribute(PositionIncrementAttribute.class);
-    addAttribute(OffsetAttribute.class);
-    addAttribute(TypeAttribute.class);
-  }
-
-
-  /*
-   * Need to worry about multiple scenarios:
-   *  - need to go for the longest match
-   *    a b => foo      #shouldn't match if "a b" is followed by "c d"
-   *    a b c d => bar
-   *  - need to backtrack - retry matches for tokens already read
-   *     a b c d => foo
-   *       b c => bar
-   *     If the input stream is "a b c x", one will consume "a b c d"
-   *     trying to match the first rule... all but "a" should be
-   *     pushed back so a match may be made on "b c".
-   *  - don't try and match generated tokens (thus need separate queue)
-   *    matching is not recursive.
-   *  - handle optional generation of original tokens in all these cases,
-   *    merging token streams to preserve token positions.
-   *  - preserve original positionIncrement of first matched token
-   */
-  @Override
-  public boolean incrementToken() throws IOException {
-    while (true) {
-      // if there are any generated tokens, return them... don't try any
-      // matches against them, as we specifically don't want recursion.
-      if (replacement!=null && replacement.hasNext()) {
-        copy(this, replacement.next());
-        return true;
-      }
-
-      // common case fast-path of first token not matching anything
-      AttributeSource firstTok = nextTok();
-      if (firstTok == null) return false;
-      CharTermAttribute termAtt = firstTok.addAttribute(CharTermAttribute.class);
-      SlowSynonymMap result = map.submap!=null ? map.submap.get(termAtt.buffer(), 0, termAtt.length()) : null;
-      if (result == null) {
-        copy(this, firstTok);
-        return true;
-      }
-
-      // fast-path failed, clone ourselves if needed
-      if (firstTok == this)
-        firstTok = cloneAttributes();
-      // OK, we matched a token, so find the longest match.
-
-      matched = new LinkedList();
-
-      result = match(result);
-
-      if (result==null) {
-        // no match, simply return the first token read.
-        copy(this, firstTok);
-        return true;
-      }
-
-      // reuse, or create new one each time?
-      ArrayList generated = new ArrayList(result.synonyms.length + matched.size() + 1);
-
-      //
-      // there was a match... let's generate the new tokens, merging
-      // in the matched tokens (position increments need adjusting)
-      //
-      AttributeSource lastTok = matched.isEmpty() ? firstTok : matched.getLast();
-      boolean includeOrig = result.includeOrig();
-
-      AttributeSource origTok = includeOrig ? firstTok : null;
-      PositionIncrementAttribute firstPosIncAtt = firstTok.addAttribute(PositionIncrementAttribute.class);
-      int origPos = firstPosIncAtt.getPositionIncrement();  // position of origTok in the original stream
-      int repPos=0; // curr position in replacement token stream
-      int pos=0;  // current position in merged token stream
-
-      for (int i=0; i  foo/0
-      // should I re-create the gap on the next buffered token?
-
-      replacement = generated.iterator();
-      // Now return to the top of the loop to read and return the first
-      // generated token.. The reason this is done is that we may have generated
-      // nothing at all, and may need to continue with more matching logic.
-    }
-  }
-
-
-  //
-  // Defer creation of the buffer until the first time it is used to
-  // optimize short fields with no matches.
-  //
-  private LinkedList buffer;
-  private LinkedList matched;
-
-  private boolean exhausted;
-
-  private AttributeSource nextTok() throws IOException {
-    if (buffer!=null && !buffer.isEmpty()) {
-      return buffer.removeFirst();
-    } else {
-      if (!exhausted && input.incrementToken()) {
-        return this;
-      } else {
-        exhausted = true;
-        return null;
-      }
-    }
-  }
-
-  private void pushTok(AttributeSource t) {
-    if (buffer==null) buffer=new LinkedList();
-    buffer.addFirst(t);
-  }
-
-  private SlowSynonymMap match(SlowSynonymMap map) throws IOException {
-    SlowSynonymMap result = null;
-
-    if (map.submap != null) {
-      AttributeSource tok = nextTok();
-      if (tok != null) {
-        // clone ourselves.
-        if (tok == this)
-          tok = cloneAttributes();
-        // check for positionIncrement!=1?  if>1, should not match, if==0, check multiple at this level?
-        CharTermAttribute termAtt = tok.getAttribute(CharTermAttribute.class);
-        SlowSynonymMap subMap = map.submap.get(termAtt.buffer(), 0, termAtt.length());
-
-        if (subMap != null) {
-          // recurse
-          result = match(subMap);
-        }
-
-        if (result != null) {
-          matched.addFirst(tok);
-        } else {
-          // push back unmatched token
-          pushTok(tok);
-        }
-      }
-    }
-
-    // if no longer sequence matched, so if this node has synonyms, it's the match.
-    if (result==null && map.synonyms!=null) {
-      result = map;
-    }
-
-    return result;
-  }
-
-  private void copy(AttributeSource target, AttributeSource source) {
-    if (target != source)
-      source.copyTo(target);
-  }
-
-  @Override
-  public void reset() throws IOException {
-    input.reset();
-    replacement = null;
-    exhausted = false;
-  }
-}
diff --git a/solr/core/src/java/org/apache/solr/analysis/SlowSynonymFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/SlowSynonymFilterFactory.java
deleted file mode 100644
index b898a03..0000000
--- a/solr/core/src/java/org/apache/solr/analysis/SlowSynonymFilterFactory.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.analysis;
-
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.util.*;
-import org.apache.solr.common.util.StrUtils;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Factory for {@link SlowSynonymFilter} (only used with luceneMatchVersion < 3.4)
- * 
          - * <fieldType name="text_synonym" class="solr.TextField" positionIncrementGap="100">
- *   <analyzer>
- *     <tokenizer class="solr.WhitespaceTokenizerFactory"/>
- *     <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="false"
- *             expand="true" tokenizerFactory="solr.WhitespaceTokenizerFactory"/>
- *   </analyzer>
- * </fieldType>
          
- * @deprecated (3.4) use {@link SynonymFilterFactory} instead. only for precise index backwards compatibility. this factory will be removed in Lucene 5.0
- */
-@Deprecated
-final class SlowSynonymFilterFactory extends TokenFilterFactory implements ResourceLoaderAware {
-
-  public void inform(ResourceLoader loader) {
-    String synonyms = args.get("synonyms");
-    if (synonyms == null)
-      throw new InitializationException("Missing required argument 'synonyms'.");
-    boolean ignoreCase = getBoolean("ignoreCase", false);
-    boolean expand = getBoolean("expand", true);
-
-    String tf = args.get("tokenizerFactory");
-    TokenizerFactory tokFactory = null;
-    if( tf != null ){
-      tokFactory = loadTokenizerFactory(loader, tf);
-    }
-
-    Iterable wlist=loadRules( synonyms, loader );
-    
-    synMap = new SlowSynonymMap(ignoreCase);
-    parseRules(wlist, synMap, "=>", ",", expand,tokFactory);
-  }
-  
-  /**
-   * @return a list of all rules
-   */
-  protected Iterable loadRules( String synonyms, ResourceLoader loader ) {
-    List wlist=null;
-    try {
-      File synonymFile = new File(synonyms);
-      if (synonymFile.exists()) {
-        wlist = loader.getLines(synonyms);
-      } else  {
-        List files = StrUtils.splitFileNames(synonyms);
-        wlist = new ArrayList();
-        for (String file : files) {
-          List lines = loader.getLines(file.trim());
-          wlist.addAll(lines);
-        }
-      }
-    } catch (IOException e) {
-      throw new InitializationException("IOException thrown while loading synonym rules", e);
-    }
-    return wlist;
-  }
-
-  private SlowSynonymMap synMap;
-
-  static void parseRules(Iterable rules, SlowSynonymMap map, String mappingSep,
-    String synSep, boolean expansion, TokenizerFactory tokFactory) {
-    int count=0;
-    for (String rule : rules) {
-      // To use regexes, we need an expression that specifies an odd number of chars.
-      // This can't really be done with string.split(), and since we need to
-      // do unescaping at some point anyway, we wouldn't be saving any effort
-      // by using regexes.
-
-      List mapping = StrUtils.splitSmart(rule, mappingSep, false);
-
-      List> source;
-      List> target;
-
-      if (mapping.size() > 2) {
-        throw new InitializationException("Invalid Synonym Rule:" + rule);
-      } else if (mapping.size()==2) {
-        source = getSynList(mapping.get(0), synSep, tokFactory);
-        target = getSynList(mapping.get(1), synSep, tokFactory);
-      } else {
-        source = getSynList(mapping.get(0), synSep, tokFactory);
-        if (expansion) {
-          // expand to all arguments
-          target = source;
-        } else {
-          // reduce to first argument
-          target = new ArrayList>(1);
-          target.add(source.get(0));
-        }
-      }
-
-      boolean includeOrig=false;
-      for (List fromToks : source) {
-        count++;
-        for (List toToks : target) {
-          map.add(fromToks,
-                  SlowSynonymMap.makeTokens(toToks),
-                  includeOrig,
-                  true
-          );
-        }
-      }
-    }
-  }
-
-  // a , b c , d e f => [[a],[b,c],[d,e,f]]
-  private static List> getSynList(String str, String separator, TokenizerFactory tokFactory) {
-    List strList = StrUtils.splitSmart(str, separator, false);
-    // now split on whitespace to get a list of token strings
-    List> synList = new ArrayList>();
-    for (String toks : strList) {
-      List tokList = tokFactory == null ?
-        StrUtils.splitWS(toks, true) : splitByTokenizer(toks, tokFactory);
-      synList.add(tokList);
-    }
-    return synList;
-  }
-
-  private static List splitByTokenizer(String source, TokenizerFactory tokFactory){
-    StringReader reader = new StringReader( source );
-    TokenStream ts = loadTokenizer(tokFactory, reader);
-    List tokList = new ArrayList();
-    try {
-      CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
-      while (ts.incrementToken()){
-        if( termAtt.length() > 0 )
-          tokList.add( termAtt.toString() );
-      }
-    } catch (IOException e) {
-      throw new InitializationException("IOException thrown while tokenizing source", e);
-    }
-    finally{
-      reader.close();
-    }
-    return tokList;
-  }
-
-  private TokenizerFactory loadTokenizerFactory(ResourceLoader loader, String cname) {
-    TokenizerFactory tokFactory = loader.newInstance(cname, TokenizerFactory.class);
-    tokFactory.setLuceneMatchVersion(luceneMatchVersion);
-    tokFactory.init( args );
-    if (tokFactory instanceof ResourceLoaderAware) {
-      ((ResourceLoaderAware) tokFactory).inform(loader);
-    }
-    return tokFactory;
-  }
-
-  private static TokenStream loadTokenizer(TokenizerFactory tokFactory, Reader reader){
-    return tokFactory.create( reader );
-  }
-
-  public SlowSynonymMap getSynonymMap() {
-    return synMap;
-  }
-
-  public SlowSynonymFilter create(TokenStream input) {
-    return new SlowSynonymFilter(input,synMap);
-  }
-}
diff --git a/solr/core/src/java/org/apache/solr/analysis/SlowSynonymMap.java b/solr/core/src/java/org/apache/solr/analysis/SlowSynonymMap.java
deleted file mode 100644
index 8082281..0000000
--- a/solr/core/src/java/org/apache/solr/analysis/SlowSynonymMap.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.analysis;
-
-import org.apache.lucene.analysis.Token;
-import org.apache.lucene.analysis.util.CharArrayMap;
-import org.apache.lucene.analysis.util.InitializationException;
-import org.apache.lucene.util.Version;
-
-import java.util.*;
-
-/** Mapping rules for use with {@link SlowSynonymFilter}
- * @deprecated (3.4) use {@link SynonymFilterFactory} instead. only for precise index backwards compatibility. this factory will be removed in Lucene 5.0
- */
-@Deprecated
-class SlowSynonymMap {
-  /** @lucene.internal */
-  public CharArrayMap submap; // recursive: Map
-  /** @lucene.internal */
-  public Token[] synonyms;
-  int flags;
-
-  static final int INCLUDE_ORIG=0x01;
-  static final int IGNORE_CASE=0x02;
-
-  public SlowSynonymMap() {}
-  public SlowSynonymMap(boolean ignoreCase) {
-    if (ignoreCase) flags |= IGNORE_CASE;
-  }
-
-  public boolean includeOrig() { return (flags & INCLUDE_ORIG) != 0; }
-  public boolean ignoreCase() { return (flags & IGNORE_CASE) != 0; }
-
-  /**
-   * @param singleMatch  List, the sequence of strings to match
-   * @param replacement  List the list of tokens to use on a match
-   * @param includeOrig  sets a flag on this mapping signaling the generation of matched tokens in addition to the replacement tokens
-   * @param mergeExisting merge the replacement tokens with any other mappings that exist
-   */
-  public void add(List singleMatch, List replacement, boolean includeOrig, boolean mergeExisting) {
-    SlowSynonymMap currMap = this;
-    for (String str : singleMatch) {
-      if (currMap.submap==null) {
-        // for now hardcode at 4.0, as its what the old code did.
-        // would be nice to fix, but shouldn't store a version in each submap!!!
-        currMap.submap = new CharArrayMap(Version.LUCENE_40, 1, ignoreCase());
-      }
-
-      SlowSynonymMap map = currMap.submap.get(str);
-      if (map==null) {
-        map = new SlowSynonymMap();
-        map.flags |= flags & IGNORE_CASE;
-        currMap.submap.put(str, map);
-      }
-
-      currMap = map;
-    }
-
-    if (currMap.synonyms != null && !mergeExisting) {
-      throw new InitializationException("SynonymFilter: there is already a mapping for " + singleMatch);
-    }
-    List superset = currMap.synonyms==null ? replacement :
-          mergeTokens(Arrays.asList(currMap.synonyms), replacement);
-    currMap.synonyms = superset.toArray(new Token[superset.size()]);
-    if (includeOrig) currMap.flags |= INCLUDE_ORIG;
-  }
-
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder("<");
-    if (synonyms!=null) {
-      sb.append("[");
-      for (int i=0; i");
-    return sb.toString();
-  }
-
-
-
-  /** Produces a List from a List */
-  public static List makeTokens(List strings) {
-    List ret = new ArrayList(strings.size());
-    for (String str : strings) {
-      //Token newTok = new Token(str,0,0,"SYNONYM");
-      Token newTok = new Token(str, 0,0,"SYNONYM");
-      ret.add(newTok);
-    }
-    return ret;
-  }
-
-
-  /**
-   * Merge two lists of tokens, producing a single list with manipulated positionIncrements so that
-   * the tokens end up at the same position.
-   *
-   * Example:  [a b] merged with [c d] produces [a/b c/d]  ('/' denotes tokens in the same position)
-   * Example:  [a,5 b,2] merged with [c d,4 e,4] produces [c a,5/d b,2 e,2]  (a,n means a has posInc=n)
-   *
-   */
-  public static List mergeTokens(List lst1, List lst2) {
-    ArrayList result = new ArrayList();
-    if (lst1 ==null || lst2 ==null) {
-      if (lst2 != null) result.addAll(lst2);
-      if (lst1 != null) result.addAll(lst1);
-      return result;
-    }
-
-    int pos=0;
-    Iterator iter1=lst1.iterator();
-    Iterator iter2=lst2.iterator();
-    Token tok1 = iter1.hasNext() ? iter1.next() : null;
-    Token tok2 = iter2.hasNext() ? iter2.next() : null;
-    int pos1 = tok1!=null ? tok1.getPositionIncrement() : 0;
-    int pos2 = tok2!=null ? tok2.getPositionIncrement() : 0;
-    while(tok1!=null || tok2!=null) {
-      while (tok1 != null && (pos1 <= pos2 || tok2==null)) {
-        Token tok = new Token(tok1.startOffset(), tok1.endOffset(), tok1.type());
-        tok.copyBuffer(tok1.buffer(), 0, tok1.length());
-        tok.setPositionIncrement(pos1-pos);
-        result.add(tok);
-        pos=pos1;
-        tok1 = iter1.hasNext() ? iter1.next() : null;
-        pos1 += tok1!=null ? tok1.getPositionIncrement() : 0;
-      }
-      while (tok2 != null && (pos2 <= pos1 || tok1==null)) {
-        Token tok = new Token(tok2.startOffset(), tok2.endOffset(), tok2.type());
-        tok.copyBuffer(tok2.buffer(), 0, tok2.length());
-        tok.setPositionIncrement(pos2-pos);
-        result.add(tok);
-        pos=pos2;
-        tok2 = iter2.hasNext() ? iter2.next() : null;
-        pos2 += tok2!=null ? tok2.getPositionIncrement() : 0;
-      }
-    }
-    return result;
-  }
-
-}
diff --git a/solr/core/src/java/org/apache/solr/analysis/SnowballPorterFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/SnowballPorterFilterFactory.java
index 17711ff..dc897cb 100644
--- a/solr/core/src/java/org/apache/solr/analysis/SnowballPorterFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/SnowballPorterFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/SolrAnalyzer.java b/solr/core/src/java/org/apache/solr/analysis/SolrAnalyzer.java
index 6ab3fac..28c2bee 100644
--- a/solr/core/src/java/org/apache/solr/analysis/SolrAnalyzer.java
+++ b/solr/core/src/java/org/apache/solr/analysis/SolrAnalyzer.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -44,7 +44,7 @@ public abstract class SolrAnalyzer extends Analyzer {
   }
 
   @Override
-  protected Reader initReader(Reader reader) {
+  protected Reader initReader(String fieldName, Reader reader) {
     return charStream(reader);
   }
 }
diff --git a/solr/core/src/java/org/apache/solr/analysis/SpanishLightStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/SpanishLightStemFilterFactory.java
index 0f40a1c..61c883b 100644
--- a/solr/core/src/java/org/apache/solr/analysis/SpanishLightStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/SpanishLightStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/StandardFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/StandardFilterFactory.java
index 1943a19..965eb08 100644
--- a/solr/core/src/java/org/apache/solr/analysis/StandardFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/StandardFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/StandardTokenizerFactory.java b/solr/core/src/java/org/apache/solr/analysis/StandardTokenizerFactory.java
index b4fbd30..4385057 100644
--- a/solr/core/src/java/org/apache/solr/analysis/StandardTokenizerFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/StandardTokenizerFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/StemmerOverrideFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/StemmerOverrideFilterFactory.java
index e6e861d..c9bec3f 100644
--- a/solr/core/src/java/org/apache/solr/analysis/StemmerOverrideFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/StemmerOverrideFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/StopFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/StopFilterFactory.java
index 4dd04b9..7e49e39 100644
--- a/solr/core/src/java/org/apache/solr/analysis/StopFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/StopFilterFactory.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/SwedishLightStemFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/SwedishLightStemFilterFactory.java
index 119618e..78f65da 100644
--- a/solr/core/src/java/org/apache/solr/analysis/SwedishLightStemFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/SwedishLightStemFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/solr/core/src/java/org/apache/solr/analysis/SynonymFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/SynonymFilterFactory.java
index df0a93e..cddaf64 100644
--- a/solr/core/src/java/org/apache/solr/analysis/SynonymFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/analysis/SynonymFilterFactory.java
@@ -1,6 +1,6 @@
 package org.apache.solr.analysis;
 
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -17,15 +17,30 @@ package org.apache.solr.analysis;
  * limitations under the License.
  */
 
-import java.util.Map;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CodingErrorAction;
+import java.text.ParseException;
+import java.util.List;
 
+import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.LowerCaseFilter;
+import org.apache.lucene.analysis.core.WhitespaceTokenizer;
 import org.apache.lucene.analysis.synonym.SynonymFilter;
-import org.apache.lucene.analysis.util.InitializationException;
+import org.apache.lucene.analysis.synonym.SynonymMap;
+import org.apache.lucene.analysis.synonym.SolrSynonymParser;
+import org.apache.lucene.analysis.synonym.WordnetSynonymParser;
+import org.apache.lucene.analysis.util.*;
 import org.apache.lucene.util.Version;
-import org.apache.lucene.analysis.util.ResourceLoader;
-import org.apache.lucene.analysis.util.ResourceLoaderAware;
-import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.solr.common.util.StrUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Factory for {@link SynonymFilter}.
@@ -40,34 +55,120 @@ import org.apache.lucene.analysis.util.TokenFilterFactory;
  * </fieldType>
      */ public class SynonymFilterFactory extends TokenFilterFactory implements ResourceLoaderAware { - private TokenFilterFactory delegator; - @Override - public void init(Map args) { - super.init(args); - assureMatchVersion(); - if (luceneMatchVersion.onOrAfter(Version.LUCENE_34)) { - delegator = new FSTSynonymFilterFactory(); - } else { - // check if you use the new optional arg "format". this makes no sense for the old one, - // as its wired to solr's synonyms format only. - if (args.containsKey("format") && !args.get("format").equals("solr")) { - throw new InitializationException("You must specify luceneMatchVersion >= 3.4 to use alternate synonyms formats"); - } - delegator = new SlowSynonymFilterFactory(); - } - delegator.init(args); - } + public static final Logger log = LoggerFactory.getLogger(SynonymFilterFactory.class); + private SynonymMap map; + private boolean ignoreCase; + @Override public TokenStream create(TokenStream input) { - assert delegator != null : "init() was not called!"; - return delegator.create(input); + // if the fst is null, it means there's actually no synonyms... just return the original stream + // as there is nothing to do here. + return map.fst == null ? input : new SynonymFilter(input, map, ignoreCase); } @Override public void inform(ResourceLoader loader) { - assert delegator != null : "init() was not called!"; - ((ResourceLoaderAware) delegator).inform(loader); + final boolean ignoreCase = getBoolean("ignoreCase", false); + this.ignoreCase = ignoreCase; + + String tf = args.get("tokenizerFactory"); + + final TokenizerFactory factory = tf == null ? null : loadTokenizerFactory(loader, tf); + + Analyzer analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName, Reader reader) { + Tokenizer tokenizer = factory == null ? new WhitespaceTokenizer(Version.LUCENE_50, reader) : factory.create(reader); + TokenStream stream = ignoreCase ? new LowerCaseFilter(Version.LUCENE_50, tokenizer) : tokenizer; + return new TokenStreamComponents(tokenizer, stream); + } + }; + + String format = args.get("format"); + try { + if (format == null || format.equals("solr")) { + // TODO: expose dedup as a parameter? + map = loadSolrSynonyms(loader, true, analyzer); + } else if (format.equals("wordnet")) { + map = loadWordnetSynonyms(loader, true, analyzer); + } else { + // TODO: somehow make this more pluggable + throw new InitializationException("Unrecognized synonyms format: " + format); + } + } catch (Exception e) { + throw new InitializationException("Exception thrown while loading synonyms", e); + } + + if (map.fst == null) { + log.warn("Synonyms loaded with " + args + " has empty rule set!"); + } + } + + /** + * Load synonyms from the solr format, "format=solr". + */ + private SynonymMap loadSolrSynonyms(ResourceLoader loader, boolean dedup, Analyzer analyzer) throws IOException, ParseException { + final boolean expand = getBoolean("expand", true); + String synonyms = args.get("synonyms"); + if (synonyms == null) + throw new InitializationException("Missing required argument 'synonyms'."); + + CharsetDecoder decoder = Charset.forName("UTF-8").newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT); + + SolrSynonymParser parser = new SolrSynonymParser(dedup, expand, analyzer); + File synonymFile = new File(synonyms); + if (synonymFile.exists()) { + decoder.reset(); + parser.add(new InputStreamReader(loader.openResource(synonyms), decoder)); + } else { + List files = StrUtils.splitFileNames(synonyms); + for (String file : files) { + decoder.reset(); + parser.add(new InputStreamReader(loader.openResource(file), decoder)); + } + } + return parser.build(); + } + + /** + * Load synonyms from the wordnet format, "format=wordnet". + */ + private SynonymMap loadWordnetSynonyms(ResourceLoader loader, boolean dedup, Analyzer analyzer) throws IOException, ParseException { + final boolean expand = getBoolean("expand", true); + String synonyms = args.get("synonyms"); + if (synonyms == null) + throw new InitializationException("Missing required argument 'synonyms'."); + + CharsetDecoder decoder = Charset.forName("UTF-8").newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT); + + WordnetSynonymParser parser = new WordnetSynonymParser(dedup, expand, analyzer); + File synonymFile = new File(synonyms); + if (synonymFile.exists()) { + decoder.reset(); + parser.add(new InputStreamReader(loader.openResource(synonyms), decoder)); + } else { + List files = StrUtils.splitFileNames(synonyms); + for (String file : files) { + decoder.reset(); + parser.add(new InputStreamReader(loader.openResource(file), decoder)); + } + } + return parser.build(); + } + + private TokenizerFactory loadTokenizerFactory(ResourceLoader loader, String cname){ + TokenizerFactory tokFactory = loader.newInstance(cname, TokenizerFactory.class); + tokFactory.setLuceneMatchVersion(luceneMatchVersion); + tokFactory.init(args); + if (tokFactory instanceof ResourceLoaderAware) { + ((ResourceLoaderAware) tokFactory).inform(loader); + } + return tokFactory; } } diff --git a/solr/core/src/java/org/apache/solr/analysis/ThaiWordFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/ThaiWordFilterFactory.java index 474d89b..22d3953 100644 --- a/solr/core/src/java/org/apache/solr/analysis/ThaiWordFilterFactory.java +++ b/solr/core/src/java/org/apache/solr/analysis/ThaiWordFilterFactory.java @@ -1,5 +1,5 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/analysis/TokenOffsetPayloadTokenFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/TokenOffsetPayloadTokenFilterFactory.java index 9a5e3e9..6d866e1 100644 --- a/solr/core/src/java/org/apache/solr/analysis/TokenOffsetPayloadTokenFilterFactory.java +++ b/solr/core/src/java/org/apache/solr/analysis/TokenOffsetPayloadTokenFilterFactory.java @@ -1,5 +1,5 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java b/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java index 8643ec2..6188469 100644 --- a/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java +++ b/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -57,18 +57,19 @@ public final class TokenizerChain extends SolrAnalyzer { super(source, result); } + // TODO: what is going on here? @Override protected void reset(Reader reader) throws IOException { // the tokenizers are currently reset by the indexing process, so only // the tokenizer needs to be reset. - Reader r = initReader(reader); + Reader r = initReader(null, reader); super.reset(r); } } @Override - public Reader initReader(Reader reader) { + public Reader initReader(String fieldName, Reader reader) { if (charFilters != null && charFilters.length > 0) { CharStream cs = CharReader.get( reader ); for (CharFilterFactory charFilter : charFilters) { @@ -81,7 +82,7 @@ public final class TokenizerChain extends SolrAnalyzer { @Override protected TokenStreamComponents createComponents(String fieldName, Reader aReader) { - Tokenizer tk = tokenizer.create( initReader(aReader) ); + Tokenizer tk = tokenizer.create( initReader(fieldName, aReader) ); TokenStream ts = tk; for (TokenFilterFactory filter : filters) { ts = filter.create(ts); diff --git a/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java b/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java index e017811..b760a2e 100644 --- a/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java +++ b/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -73,15 +73,11 @@ final class TrieTokenizer extends Tokenizer { this.precisionStep = precisionStep; this.ts = ts; - try { - reset(input); - } catch (IOException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to create TrieIndexTokenizer", e); - } + reset(input); } @Override - public void reset(Reader input) throws IOException { + public void reset(Reader input) { try { super.reset(input); input = super.input; @@ -132,7 +128,7 @@ final class TrieTokenizer extends Tokenizer { } @Override - public boolean incrementToken() throws IOException { + public boolean incrementToken() { if (ts.incrementToken()) { ofsAtt.setOffset(startOfs, endOfs); return true; diff --git a/solr/core/src/java/org/apache/solr/analysis/TrimFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/TrimFilterFactory.java index 917dd79..ef7fbae 100644 --- a/solr/core/src/java/org/apache/solr/analysis/TrimFilterFactory.java +++ b/solr/core/src/java/org/apache/solr/analysis/TrimFilterFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/analysis/TurkishLowerCaseFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/TurkishLowerCaseFilterFactory.java index 6d5e7b4..72fa83b 100644 --- a/solr/core/src/java/org/apache/solr/analysis/TurkishLowerCaseFilterFactory.java +++ b/solr/core/src/java/org/apache/solr/analysis/TurkishLowerCaseFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/analysis/TypeAsPayloadTokenFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/TypeAsPayloadTokenFilterFactory.java index 550bcc9..9a67fa9 100644 --- a/solr/core/src/java/org/apache/solr/analysis/TypeAsPayloadTokenFilterFactory.java +++ b/solr/core/src/java/org/apache/solr/analysis/TypeAsPayloadTokenFilterFactory.java @@ -1,5 +1,5 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/analysis/UAX29URLEmailTokenizerFactory.java b/solr/core/src/java/org/apache/solr/analysis/UAX29URLEmailTokenizerFactory.java index 3a6a171..4c7076f 100644 --- a/solr/core/src/java/org/apache/solr/analysis/UAX29URLEmailTokenizerFactory.java +++ b/solr/core/src/java/org/apache/solr/analysis/UAX29URLEmailTokenizerFactory.java @@ -1,7 +1,7 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/analysis/WhitespaceTokenizerFactory.java b/solr/core/src/java/org/apache/solr/analysis/WhitespaceTokenizerFactory.java index dd1ef98..47bf213 100644 --- a/solr/core/src/java/org/apache/solr/analysis/WhitespaceTokenizerFactory.java +++ b/solr/core/src/java/org/apache/solr/analysis/WhitespaceTokenizerFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/analysis/WikipediaTokenizerFactory.java b/solr/core/src/java/org/apache/solr/analysis/WikipediaTokenizerFactory.java index a1eb583..e77b058 100644 --- a/solr/core/src/java/org/apache/solr/analysis/WikipediaTokenizerFactory.java +++ b/solr/core/src/java/org/apache/solr/analysis/WikipediaTokenizerFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/analysis/WordDelimiterFilterFactory.java b/solr/core/src/java/org/apache/solr/analysis/WordDelimiterFilterFactory.java index 0cd5779..86239eb 100644 --- a/solr/core/src/java/org/apache/solr/analysis/WordDelimiterFilterFactory.java +++ b/solr/core/src/java/org/apache/solr/analysis/WordDelimiterFilterFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java b/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java index 3c5c909..a67a9c3 100644 --- a/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java +++ b/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,9 +21,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.util.Set; -import org.apache.lucene.document.Document; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; @@ -45,9 +43,6 @@ import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.response.BinaryResponseWriter; import org.apache.solr.response.ResultContext; import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.response.transform.DocTransformer; -import org.apache.solr.search.DocIterator; -import org.apache.solr.search.DocList; import org.apache.solr.servlet.SolrRequestParsers; /** @@ -187,7 +182,7 @@ public class EmbeddedSolrServer extends SolrServer new JavaBinCodec(resolver) { @Override - public void writeSolrDocument(SolrDocument doc) throws IOException { + public void writeSolrDocument(SolrDocument doc) { callback.streamSolrDocument( doc ); //super.writeSolrDocument( doc, fields ); } @@ -252,6 +247,7 @@ public class EmbeddedSolrServer extends SolrServer /** * Shutdown all cores within the EmbeddedSolrServer instance */ + @Override public void shutdown() { coreContainer.shutdown(); } diff --git a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java index 944c933..0729704 100644 --- a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java +++ b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/cloud/AssignShard.java b/solr/core/src/java/org/apache/solr/cloud/AssignShard.java index b7fa396..9dcd09d 100644 --- a/solr/core/src/java/org/apache/solr/cloud/AssignShard.java +++ b/solr/core/src/java/org/apache/solr/cloud/AssignShard.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java b/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java index c29b7c3..041439d 100644 --- a/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java +++ b/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/cloud/CurrentCoreDescriptorProvider.java b/solr/core/src/java/org/apache/solr/cloud/CurrentCoreDescriptorProvider.java index 2349df3..db15281 100644 --- a/solr/core/src/java/org/apache/solr/cloud/CurrentCoreDescriptorProvider.java +++ b/solr/core/src/java/org/apache/solr/cloud/CurrentCoreDescriptorProvider.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java b/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java new file mode 100644 index 0000000..143324d --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java @@ -0,0 +1,323 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + +import java.util.List; +import java.util.NoSuchElementException; +import java.util.TreeMap; +import java.util.concurrent.CountDownLatch; + +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.data.ACL; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A distributed queue from zk recipes. + */ +public class DistributedQueue { + private static final Logger LOG = LoggerFactory + .getLogger(DistributedQueue.class); + + private final String dir; + + private ZooKeeper zookeeper; + private List acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; + + private final String prefix = "qn-"; + + public DistributedQueue(ZooKeeper zookeeper, String dir, List acl) { + this.dir = dir; + + if (acl != null) { + this.acl = acl; + } + this.zookeeper = zookeeper; + + } + + /** + * Returns a Map of the children, ordered by id. + * + * @param watcher + * optional watcher on getChildren() operation. + * @return map from id to child name for all children + */ + private TreeMap orderedChildren(Watcher watcher) + throws KeeperException, InterruptedException { + TreeMap orderedChildren = new TreeMap(); + + List childNames = null; + try { + childNames = zookeeper.getChildren(dir, watcher); + } catch (KeeperException.NoNodeException e) { + throw e; + } + + for (String childName : childNames) { + try { + // Check format + if (!childName.regionMatches(0, prefix, 0, prefix.length())) { + LOG.warn("Found child node with improper name: " + childName); + continue; + } + String suffix = childName.substring(prefix.length()); + Long childId = new Long(suffix); + orderedChildren.put(childId, childName); + } catch (NumberFormatException e) { + LOG.warn("Found child node with improper format : " + childName + " " + + e, e); + } + } + + return orderedChildren; + } + + /** + * Return the head of the queue without modifying the queue. + * + * @return the data at the head of the queue. + * @throws NoSuchElementException + * @throws KeeperException + * @throws InterruptedException + */ + public byte[] element() throws NoSuchElementException, KeeperException, + InterruptedException { + TreeMap orderedChildren; + + // element, take, and remove follow the same pattern. + // We want to return the child node with the smallest sequence number. + // Since other clients are remove()ing and take()ing nodes concurrently, + // the child with the smallest sequence number in orderedChildren might be + // gone by the time we check. + // We don't call getChildren again until we have tried the rest of the nodes + // in sequence order. + while (true) { + try { + orderedChildren = orderedChildren(null); + } catch (KeeperException.NoNodeException e) { + throw new NoSuchElementException(); + } + if (orderedChildren.size() == 0) throw new NoSuchElementException(); + + for (String headNode : orderedChildren.values()) { + if (headNode != null) { + try { + return zookeeper.getData(dir + "/" + headNode, false, null); + } catch (KeeperException.NoNodeException e) { + // Another client removed the node first, try next + } + } + } + } + } + + /** + * Attempts to remove the head of the queue and return it. + * + * @return The former head of the queue + * @throws NoSuchElementException + * @throws KeeperException + * @throws InterruptedException + */ + public byte[] remove() throws NoSuchElementException, KeeperException, + InterruptedException { + TreeMap orderedChildren; + // Same as for element. Should refactor this. + while (true) { + try { + orderedChildren = orderedChildren(null); + } catch (KeeperException.NoNodeException e) { + throw new NoSuchElementException(); + } + if (orderedChildren.size() == 0) throw new NoSuchElementException(); + + for (String headNode : orderedChildren.values()) { + String path = dir + "/" + headNode; + try { + byte[] data = zookeeper.getData(path, false, null); + zookeeper.delete(path, -1); + return data; + } catch (KeeperException.NoNodeException e) { + // Another client deleted the node first. + } + } + + } + } + + private class LatchChildWatcher implements Watcher { + + CountDownLatch latch; + + public LatchChildWatcher() { + latch = new CountDownLatch(1); + } + + public void process(WatchedEvent event) { + LOG.debug("Watcher fired on path: " + event.getPath() + " state: " + + event.getState() + " type " + event.getType()); + latch.countDown(); + } + + public void await() throws InterruptedException { + latch.await(); + } + } + + /** + * Removes the head of the queue and returns it, blocks until it succeeds. + * + * @return The former head of the queue + * @throws NoSuchElementException + * @throws KeeperException + * @throws InterruptedException + */ + public byte[] take() throws KeeperException, InterruptedException { + TreeMap orderedChildren; + // Same as for element. Should refactor this. + while (true) { + LatchChildWatcher childWatcher = new LatchChildWatcher(); + try { + orderedChildren = orderedChildren(childWatcher); + } catch (KeeperException.NoNodeException e) { + zookeeper.create(dir, new byte[0], acl, CreateMode.PERSISTENT); + continue; + } + if (orderedChildren.size() == 0) { + childWatcher.await(); + continue; + } + + for (String headNode : orderedChildren.values()) { + String path = dir + "/" + headNode; + try { + byte[] data = zookeeper.getData(path, false, null); + zookeeper.delete(path, -1); + return data; + } catch (KeeperException.NoNodeException e) { + // Another client deleted the node first. + } + } + } + } + + /** + * Inserts data into queue. + * + * @param data + * @return true if data was successfully added + */ + public boolean offer(byte[] data) throws KeeperException, + InterruptedException { + for (;;) { + try { + zookeeper.create(dir + "/" + prefix, data, acl, + CreateMode.PERSISTENT_SEQUENTIAL); + return true; + } catch (KeeperException.NoNodeException e) { + try { + zookeeper.create(dir, new byte[0], acl, CreateMode.PERSISTENT); + } catch (KeeperException.NodeExistsException ne) { + //someone created it + } + } + } + + + + } + + /** + * Returns the data at the first element of the queue, or null if the queue is + * empty. + * + * @return data at the first element of the queue, or null. + * @throws KeeperException + * @throws InterruptedException + */ + public byte[] peek() throws KeeperException, InterruptedException { + try { + return element(); + } catch (NoSuchElementException e) { + return null; + } + } + + /** + * Returns the data at the first element of the queue, or null if the queue is + * empty. + * + * @return data at the first element of the queue, or null. + * @throws KeeperException + * @throws InterruptedException + */ + public byte[] peek(boolean block) throws KeeperException, InterruptedException { + if (!block) { + return peek(); + } + + TreeMap orderedChildren; + while (true) { + LatchChildWatcher childWatcher = new LatchChildWatcher(); + try { + orderedChildren = orderedChildren(childWatcher); + } catch (KeeperException.NoNodeException e) { + zookeeper.create(dir, new byte[0], acl, CreateMode.PERSISTENT); + continue; + } + if (orderedChildren.size() == 0) { + childWatcher.await(); + continue; + } + + for (String headNode : orderedChildren.values()) { + String path = dir + "/" + headNode; + try { + byte[] data = zookeeper.getData(path, false, null); + return data; + } catch (KeeperException.NoNodeException e) { + // Another client deleted the node first. + } + } + } + } + + /** + * Attempts to remove the head of the queue and return it. Returns null if the + * queue is empty. + * + * @return Head of the queue or null. + * @throws KeeperException + * @throws InterruptedException + */ + public byte[] poll() throws KeeperException, InterruptedException { + try { + return remove(); + } catch (NoSuchElementException e) { + return null; + } + } + +} diff --git a/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java index 4128926..9d20d14 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java @@ -8,17 +8,17 @@ import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.CloudState; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkClientConnectionStrategy; import org.apache.solr.common.cloud.ZkCoreNodeProps; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.component.ShardHandler; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.NodeExistsException; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -96,6 +96,15 @@ class ShardLeaderElectionContextBase extends ElectionContext { leaderProps == null ? null : ZkStateReader.toJSON(leaderProps), CreateMode.EPHEMERAL, true); } + + // TODO: above we make it looks like leaderProps could be true, but here + // you would get an NPE if it was. + ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, + "leader", ZkStateReader.SHARD_ID_PROP, shardId, + ZkStateReader.COLLECTION_PROP, collection, ZkStateReader.BASE_URL_PROP, + leaderProps.getProperties().get(ZkStateReader.BASE_URL_PROP), + ZkStateReader.CORE_NAME_PROP, leaderProps.getProperties().get(ZkStateReader.CORE_NAME_PROP)); + Overseer.getInQueue(zkClient).offer(ZkStateReader.toJSON(m)); } } @@ -239,11 +248,15 @@ final class OverseerElectionContext extends ElectionContext { private final SolrZkClient zkClient; private final ZkStateReader stateReader; + private ShardHandler shardHandler; + private String adminPath; - public OverseerElectionContext(final String zkNodeName, SolrZkClient zkClient, ZkStateReader stateReader) { + public OverseerElectionContext(ShardHandler shardHandler, String adminPath, final String zkNodeName, ZkStateReader stateReader) { super(zkNodeName, "/overseer_elect", "/overseer_elect/leader", null, stateReader.getZkClient()); - this.zkClient = zkClient; this.stateReader = stateReader; + this.shardHandler = shardHandler; + this.adminPath = adminPath; + this.zkClient = stateReader.getZkClient(); } @Override @@ -265,7 +278,7 @@ final class OverseerElectionContext extends ElectionContext { CreateMode.EPHEMERAL, true); } - new Overseer(zkClient, stateReader, id); + new Overseer(shardHandler, adminPath, stateReader, id); } } diff --git a/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java b/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java index cc89917..12dd86f 100644 --- a/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java +++ b/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -30,12 +30,12 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkCmdExecutor; import org.apache.solr.common.cloud.ZooKeeperException; -import org.apache.solr.core.SolrCore; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.ConnectionLossException; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.Watcher.Event.EventType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -115,6 +115,11 @@ public class LeaderElector { @Override public void process(WatchedEvent event) { + // session events are not change events, + // and do not remove the watcher + if (EventType.None.equals(event.getType())) { + return; + } // am I the next leader? try { checkIfIamLeader(seq, context, true); diff --git a/solr/core/src/java/org/apache/solr/cloud/NodeStateWatcher.java b/solr/core/src/java/org/apache/solr/cloud/NodeStateWatcher.java deleted file mode 100644 index ab339c8..0000000 --- a/solr/core/src/java/org/apache/solr/cloud/NodeStateWatcher.java +++ /dev/null @@ -1,140 +0,0 @@ -package org.apache.solr.cloud; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.apache.solr.common.cloud.CoreState; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Watcher for node state changes. - */ -public class NodeStateWatcher implements Watcher { - - private static Logger log = LoggerFactory.getLogger(NodeStateWatcher.class); - - public static interface NodeStateChangeListener { - void coreChanged(String nodeName, Set states) - throws KeeperException, InterruptedException; - void coreDeleted(String nodeName, Collection states) - throws KeeperException, InterruptedException; - } - - private final SolrZkClient zkClient; - private final String path; - private volatile Set currentState = new HashSet(); - private final NodeStateChangeListener listener; - private final String nodeName; - - - public Set getCurrentState() { - return currentState; - } - - public NodeStateWatcher(SolrZkClient zkClient, String nodeName, String path, - NodeStateChangeListener listener) throws KeeperException, InterruptedException { - this.nodeName = nodeName; - this.zkClient = zkClient; - this.path = path; - this.listener = listener; - processStateChange(); - } - - @Override - public void process(WatchedEvent event) { - try { - processStateChange(); - } catch (InterruptedException e) { - // Restore the interrupted status - Thread.currentThread().interrupt(); - return; - } catch (Exception e) { - log.warn("Error processing state change", e); - } - } - - private void processStateChange() throws KeeperException, InterruptedException { - byte[] data = zkClient.getData(path, this, null, true); - - if (data != null) { - CoreState[] states = CoreState.load(data); - List stateList = Arrays.asList(states); - HashSet modifiedCores = new HashSet(); - modifiedCores.addAll(stateList); - modifiedCores.removeAll(currentState); - - HashSet newState = new HashSet(); - newState.addAll(stateList); - - HashMap lookup = new HashMap(); - for(CoreState state: states) { - lookup.put(state.getCoreName(), state); - } - - //check for status change - for(CoreState state: currentState) { - if(lookup.containsKey(state.getCoreName())) { - if(!state.getProperties().equals(lookup.get(state.getCoreName()).getProperties())) { - modifiedCores.add(lookup.get(state.getCoreName())); - } - } - } - - HashMap deletedCores = new HashMap(); - for(CoreState state: currentState) { - deletedCores.put(state.getCoreNodeName(), state); - } - - for(CoreState state: stateList) { - deletedCores.remove(state.getCoreNodeName()); - } - - if (deletedCores.size() > 0) { - listener.coreDeleted(nodeName, deletedCores.values()); - } - - currentState = Collections.unmodifiableSet(newState); - - if (modifiedCores.size() > 0) { - try { - listener.coreChanged(nodeName, Collections.unmodifiableSet(modifiedCores)); - } catch (KeeperException e) { - log.warn("Could not talk to ZK", e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - log.warn("Could not talk to ZK", e); - } - } - - } else { - // ignore null state - } - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java index 59bd291..54bb825 100644 --- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java +++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF @@ -17,157 +17,170 @@ package org.apache.solr.cloud; * the License. */ -import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.Set; -import org.apache.solr.cloud.NodeStateWatcher.NodeStateChangeListener; -import org.apache.solr.cloud.ShardLeaderWatcher.ShardLeaderListener; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.CloudState; -import org.apache.solr.common.cloud.CoreState; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkCmdExecutor; import org.apache.solr.common.cloud.ZkCoreNodeProps; import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.cloud.ZkOperation; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.cloud.ZooKeeperException; +import org.apache.solr.handler.component.ShardHandler; +import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.KeeperException.Code; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Cluster leader. Responsible node assignments, cluster state file? */ -public class Overseer implements NodeStateChangeListener, ShardLeaderListener { +public class Overseer { + public static final String QUEUE_OPERATION = "operation"; private static final int STATE_UPDATE_DELAY = 500; // delay between cloud state updates - static enum Op { - LeaderChange, StateChange, CoreDeleted; - } - - private final class CloudStateUpdateRequest { - - final Op operation; - final Object[] args; - - CloudStateUpdateRequest(final Op operation, final Object... args) { - this.operation = operation; - this.args = args; - } - } - - public static final String STATES_NODE = "/node_states"; private static Logger log = LoggerFactory.getLogger(Overseer.class); - private final SolrZkClient zkClient; - - // pooled updates - private final LinkedBlockingQueue fifo = new LinkedBlockingQueue(); - - // node stateWatches - private HashMap nodeStateWatches = new HashMap(); - - // shard leader watchers (collection->slice->watcher) - private HashMap> shardLeaderWatches = new HashMap>(); - private ZkCmdExecutor zkCmdExecutor; - private static class CloudStateUpdater implements Runnable { - private final LinkedBlockingQueue fifo; + private static final String DELETECORE = "deletecore"; private final ZkStateReader reader; private final SolrZkClient zkClient; private final String myId; + //queue where everybody can throw tasks + private final DistributedQueue stateUpdateQueue; + //Internal queue where overseer stores events that have not yet been published into cloudstate + //If Overseer dies while extracting the main queue a new overseer will start from this queue + private final DistributedQueue workQueue; - public CloudStateUpdater(final LinkedBlockingQueue fifo, final ZkStateReader reader, final SolrZkClient zkClient, final String myId) { - this.fifo = fifo; + public CloudStateUpdater(final ZkStateReader reader, final String myId) { + this.zkClient = reader.getZkClient(); + this.stateUpdateQueue = getInQueue(zkClient); + this.workQueue = getInternalQueue(zkClient); this.myId = myId; this.reader = reader; - this.zkClient = zkClient; } - @Override - public void run() { - while (amILeader()) { - - - LinkedList requests = new LinkedList(); - while (!fifo.isEmpty()) { - // collect all queued requests - CloudStateUpdateRequest req; - req = fifo.poll(); - if (req == null) { - break; - } - requests.add(req); - } - - if (requests.size() > 0) { - // process updates - synchronized (reader.getUpdateLock()) { - try { + + @Override + public void run() { + + if(amILeader()) { + // see if there's something left from the previous Overseer and re + // process all events that were not persisted into cloud state + synchronized (reader.getUpdateLock()) { //XXX this only protects against edits inside single node + try { + byte[] head = workQueue.peek(); + + if (head != null) { reader.updateCloudState(true); CloudState cloudState = reader.getCloudState(); - for (CloudStateUpdateRequest request : requests) { - - switch (request.operation) { - case LeaderChange: - cloudState = setShardLeader(cloudState, - (String) request.args[0], (String) request.args[1], - (String) request.args[2]); - - break; - case StateChange: - cloudState = updateState(cloudState, - (String) request.args[0], (CoreState) request.args[1]); - break; - - case CoreDeleted: - cloudState = removeCore(cloudState, (String) request.args[0], (String) request.args[1]); - break; - } - } - - log.info("Announcing new cluster state"); - zkClient.setData(ZkStateReader.CLUSTER_STATE, - ZkStateReader.toJSON(cloudState), true); - - } catch (KeeperException e) { - if (e.code() == KeeperException.Code.SESSIONEXPIRED - || e.code() == KeeperException.Code.CONNECTIONLOSS) { - log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK"); - return; + log.info("Replaying operations from work queue."); + + while (head != null && amILeader()) { + final ZkNodeProps message = ZkNodeProps.load(head); + final String operation = message + .get(QUEUE_OPERATION); + cloudState = processMessage(cloudState, message, operation); + zkClient.setData(ZkStateReader.CLUSTER_STATE, + ZkStateReader.toJSON(cloudState), true); + workQueue.remove(); + head = workQueue.peek(); } - SolrException.log(log, "", e); - throw new ZooKeeperException( - SolrException.ErrorCode.SERVER_ERROR, "", e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + } + } catch (KeeperException e) { + if (e.code() == KeeperException.Code.SESSIONEXPIRED + || e.code() == KeeperException.Code.CONNECTIONLOSS) { + log.warn("Solr cannot talk to ZK"); return; } + SolrException.log(log, "", e); + throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, + "", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; } } - + } + + log.info("Starting to work on the main queue"); + while (amILeader()) { + synchronized (reader.getUpdateLock()) { try { - Thread.sleep(STATE_UPDATE_DELAY); + byte[] head = stateUpdateQueue.peek(); + + if (head != null) { + reader.updateCloudState(true); + CloudState cloudState = reader.getCloudState(); + + while (head != null) { + final ZkNodeProps message = ZkNodeProps.load(head); + final String operation = message.get(QUEUE_OPERATION); + + cloudState = processMessage(cloudState, message, operation); + byte[] processed = stateUpdateQueue.remove(); + workQueue.offer(processed); + head = stateUpdateQueue.peek(); + } + zkClient.setData(ZkStateReader.CLUSTER_STATE, + ZkStateReader.toJSON(cloudState), true); + } + // clean work queue + while (workQueue.poll() != null); + + } catch (KeeperException e) { + if (e.code() == KeeperException.Code.SESSIONEXPIRED + || e.code() == KeeperException.Code.CONNECTIONLOSS) { + log.warn("Overseer cannot talk to ZK"); + return; + } + SolrException.log(log, "", e); + throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, + "", e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); + return; } } + + try { + Thread.sleep(STATE_UPDATE_DELAY); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } } + } + + private CloudState processMessage(CloudState cloudState, + final ZkNodeProps message, final String operation) { + if ("state".equals(operation)) { + cloudState = updateState(cloudState, message); + } else if (DELETECORE.equals(operation)) { + cloudState = removeCore(cloudState, message); + } else if (ZkStateReader.LEADER_PROP.equals(operation)) { + StringBuilder sb = new StringBuilder(); + String baseUrl = message.get(ZkStateReader.BASE_URL_PROP); + String coreName = message.get(ZkStateReader.CORE_NAME_PROP); + sb.append(baseUrl); + if (!baseUrl.endsWith("/")) sb.append("/"); + sb.append(coreName == null ? "" : coreName); + if (!(sb.substring(sb.length() - 1).equals("/"))) sb + .append("/"); + cloudState = setShardLeader(cloudState, + message.get(ZkStateReader.COLLECTION_PROP), + message.get(ZkStateReader.SHARD_ID_PROP), sb.toString()); + } else { + throw new RuntimeException("unknown operation:" + operation + + " contents:" + message.getProperties()); + } + return cloudState; + } private boolean amILeader() { try { @@ -185,35 +198,36 @@ public class Overseer implements NodeStateChangeListener, ShardLeaderListener { } /** * Try to assign core to the cluster. - * @throws KeeperException - * @throws InterruptedException */ - private CloudState updateState(CloudState state, String nodeName, CoreState coreState) throws KeeperException, InterruptedException { - String collection = coreState.getCollectionName(); - String zkCoreNodeName = coreState.getCoreNodeName(); + private CloudState updateState(CloudState state, final ZkNodeProps message) { + final String collection = message.get(ZkStateReader.COLLECTION_PROP); + final String zkCoreNodeName = message.get(ZkStateReader.NODE_NAME_PROP) + "_" + message.get(ZkStateReader.CORE_NAME_PROP); + final Integer numShards = message.get(ZkStateReader.NUM_SHARDS_PROP)!=null?Integer.parseInt(message.get(ZkStateReader.NUM_SHARDS_PROP)):null; //collection does not yet exist, create placeholders if num shards is specified - if (!state.getCollections().contains(coreState.getCollectionName()) - && coreState.getNumShards() != null) { - state = createCollection(state, collection, coreState.getNumShards()); + if (!state.getCollections().contains(collection) + && numShards!=null) { + state = createCollection(state, collection, numShards); } // use the provided non null shardId - String shardId = coreState.getProperties().get(ZkStateReader.SHARD_ID_PROP); - if(shardId==null) { - //use shardId from CloudState - shardId = getAssignedId(state, nodeName, coreState); + String shardId = message.get(ZkStateReader.SHARD_ID_PROP); + if (shardId == null) { + String nodeName = message.get(ZkStateReader.NODE_NAME_PROP); + //get shardId from CloudState + shardId = getAssignedId(state, nodeName, message); } - if(shardId==null) { + if(shardId == null) { //request new shardId - shardId = AssignShard.assignShard(collection, state, coreState.getNumShards()); + shardId = AssignShard.assignShard(collection, state, numShards); } Map props = new HashMap(); - Map coreProps = new HashMap(coreState.getProperties().size()); - coreProps.putAll(coreState.getProperties()); + Map coreProps = new HashMap(message.getProperties().size()); + coreProps.putAll(message.getProperties()); // we don't put num_shards in the clusterstate - coreProps.remove("num_shards"); + coreProps.remove(ZkStateReader.NUM_SHARDS_PROP); + coreProps.remove(QUEUE_OPERATION); for (Entry entry : coreProps.entrySet()) { props.put(entry.getKey(), entry.getValue()); } @@ -249,9 +263,9 @@ public class Overseer implements NodeStateChangeListener, ShardLeaderListener { * Return an already assigned id or null if not assigned */ private String getAssignedId(final CloudState state, final String nodeName, - final CoreState coreState) { - final String key = coreState.getProperties().get(ZkStateReader.NODE_NAME_PROP) + "_" + coreState.getProperties().get(ZkStateReader.CORE_NAME_PROP); - Map slices = state.getSlices(coreState.getCollectionName()); + final ZkNodeProps coreState) { + final String key = coreState.get(ZkStateReader.NODE_NAME_PROP) + "_" + coreState.get(ZkStateReader.CORE_NAME_PROP); + Map slices = state.getSlices(coreState.get(ZkStateReader.COLLECTION_PROP)); if (slices != null) { for (Slice slice : slices.values()) { if (slice.getShards().get(key) != null) { @@ -303,12 +317,12 @@ public class Overseer implements NodeStateChangeListener, ShardLeaderListener { final Map slices = newStates.get(collection); if(slices==null) { - log.error("Could not mark shard leader for non existing collection."); + log.error("Could not mark shard leader for non existing collection:" + collection); return state; } if (!slices.containsKey(sliceName)) { - log.error("Could not mark leader for non existing slice."); + log.error("Could not mark leader for non existing slice:" + sliceName); return state; } else { final Map newShards = new LinkedHashMap(); @@ -333,7 +347,11 @@ public class Overseer implements NodeStateChangeListener, ShardLeaderListener { /* * Remove core from cloudstate */ - private CloudState removeCore(final CloudState cloudState, final String collection, final String coreNodeName) { + private CloudState removeCore(final CloudState cloudState, ZkNodeProps message) { + + final String coreNodeName = message.get(ZkStateReader.NODE_NAME_PROP) + "_" + message.get(ZkStateReader.CORE_NAME_PROP); + final String collection = message.get(ZkStateReader.COLLECTION_PROP); + final LinkedHashMap> newStates = new LinkedHashMap>(); for(String collectionName: cloudState.getCollections()) { if(collection.equals(collectionName)) { @@ -344,13 +362,36 @@ public class Overseer implements NodeStateChangeListener, ShardLeaderListener { LinkedHashMap newShards = new LinkedHashMap(); newShards.putAll(slice.getShards()); newShards.remove(coreNodeName); + Slice newSlice = new Slice(slice.getName(), newShards); newSlices.put(slice.getName(), newSlice); + } else { newSlices.put(slice.getName(), slice); } } - newStates.put(collectionName, newSlices); + int cnt = 0; + for (Slice slice : newSlices.values()) { + cnt+=slice.getShards().size(); + } + // TODO: if no nodes are left after this unload + // remove from zk - do we have a race where Overseer + // see's registered nodes and publishes though? + if (cnt > 0) { + newStates.put(collectionName, newSlices); + } else { + // TODO: it might be better logically to have this in ZkController + // but for tests (it's easier) it seems better for the moment to leave CoreContainer and/or + // ZkController out of the Overseer. + try { + zkClient.clean("/collections/" + collectionName); + } catch (InterruptedException e) { + SolrException.log(log, "Cleaning up collection in zk was interrupted:" + collectionName, e); + Thread.currentThread().interrupt(); + } catch (KeeperException e) { + SolrException.log(log, "Problem cleaning up collection in zk:" + collectionName, e); + } + } } else { newStates.put(collectionName, cloudState.getSlices(collectionName)); } @@ -358,257 +399,57 @@ public class Overseer implements NodeStateChangeListener, ShardLeaderListener { CloudState newState = new CloudState(cloudState.getLiveNodes(), newStates); return newState; } + } - public Overseer(final SolrZkClient zkClient, final ZkStateReader reader, String id) throws KeeperException, InterruptedException { - log.info("Constructing new Overseer id=" + id); - this.zkClient = zkClient; - this.zkCmdExecutor = new ZkCmdExecutor(); - createWatches(); - + public Overseer(ShardHandler shardHandler, String adminPath, final ZkStateReader reader, final String id) throws KeeperException, InterruptedException { + log.info("Overseer (id=" + id + ") starting"); + createOverseerNode(reader.getZkClient()); //launch cluster state updater thread - ThreadGroup tg = new ThreadGroup("Overseer delayed state updater"); - Thread updaterThread = new Thread(tg, new CloudStateUpdater(fifo, reader, zkClient, id)); + ThreadGroup tg = new ThreadGroup("Overseer state updater."); + Thread updaterThread = new Thread(tg, new CloudStateUpdater(reader, id)); updaterThread.setDaemon(true); updaterThread.start(); - } - - public synchronized void createWatches() - throws KeeperException, InterruptedException { - addCollectionsWatch(); - addLiveNodesWatch(); - } - - /* - * Watch for collections so we can add watches for its shard leaders. - */ - private void addCollectionsWatch() throws KeeperException, - InterruptedException { - zkCmdExecutor.ensureExists(ZkStateReader.COLLECTIONS_ZKNODE, zkClient); - - List collections = zkClient.getChildren(ZkStateReader.COLLECTIONS_ZKNODE, new Watcher(){ - @Override - public void process(WatchedEvent event) { - try { - List collections = zkClient.getChildren(ZkStateReader.COLLECTIONS_ZKNODE, this, true); - collectionsChanged(collections); - } catch (KeeperException e) { - if (e.code() == Code.CONNECTIONLOSS || e.code() == Code.SESSIONEXPIRED) { - log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK"); - return; - } - } catch (InterruptedException e) { - // Restore the interrupted status - Thread.currentThread().interrupt(); - log.warn("", e); - } - } - }, true); - - collectionsChanged(collections); - } - - private void collectionsChanged(Collection collections) throws KeeperException, InterruptedException { - synchronized (shardLeaderWatches) { - for(String collection: collections) { - if(!shardLeaderWatches.containsKey(collection)) { - shardLeaderWatches.put(collection, new HashMap()); - addShardLeadersWatch(collection); - } - } - //XXX not handling delete collections.. - } + ThreadGroup ccTg = new ThreadGroup("Overseer collection creation process."); + Thread ccThread = new Thread(ccTg, new OverseerCollectionProcessor(reader, id, shardHandler, adminPath)); + ccThread.setDaemon(true); + ccThread.start(); } /** - * Add a watch for node containing shard leaders for a collection - * @param collection - * @throws KeeperException - * @throws InterruptedException + * Get queue that can be used to send messages to Overseer. */ - private void addShardLeadersWatch(final String collection) throws KeeperException, - InterruptedException { - - zkCmdExecutor.ensureExists(ZkStateReader.getShardLeadersPath(collection, null), zkClient); - - final List leaderNodes = zkClient.getChildren( - ZkStateReader.getShardLeadersPath(collection, null), new Watcher() { - - @Override - public void process(WatchedEvent event) { - try { - List leaderNodes = zkClient.getChildren( - ZkStateReader.getShardLeadersPath(collection, null), this, true); - - processLeaderNodesChanged(collection, leaderNodes); - } catch (KeeperException e) { - if (e.code() == KeeperException.Code.SESSIONEXPIRED - || e.code() == KeeperException.Code.CONNECTIONLOSS) { - log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK"); - return; - } - SolrException.log(log, "", e); - throw new ZooKeeperException( - SolrException.ErrorCode.SERVER_ERROR, "", e); - } catch (InterruptedException e) { - // Restore the interrupted status - Thread.currentThread().interrupt(); - } - } - }, true); - - processLeaderNodesChanged(collection, leaderNodes); - } - - /** - * Process change in shard leaders. Make sure we have watches for each leader. - */ - private void processLeaderNodesChanged(final String collection, final Collection shardIds) { - if(log.isInfoEnabled()) { - log.info("Leader nodes changed for collection: " + collection + " nodes now:" + shardIds); - } - - Map watches = shardLeaderWatches.get(collection); - Set currentWatches = new HashSet(); - currentWatches.addAll(watches.keySet()); - - Set newLeaders = complement(shardIds, currentWatches); - - Set lostLeaders = complement(currentWatches, shardIds); - //remove watches for lost shards - for (String shardId : lostLeaders) { - ShardLeaderWatcher watcher = watches.remove(shardId); - if (watcher != null) { - watcher.close(); - } - } - - //add watches for the new shards - for(String shardId: newLeaders) { - try { - ShardLeaderWatcher watcher = new ShardLeaderWatcher(shardId, collection, zkClient, this); - watches.put(shardId, watcher); - } catch (KeeperException e) { - log.error("Failed to create watcher for shard leader col:" + collection + " shard:" + shardId + ", exception: " + e.getClass()); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - log.error("Failed to create watcher for shard leader col:" + collection + " shard:" + shardId + ", exception: " + e.getClass()); - } - } + public static DistributedQueue getInQueue(final SolrZkClient zkClient) { + createOverseerNode(zkClient); + return new DistributedQueue(zkClient.getSolrZooKeeper(), "/overseer/queue", null); } - private void addLiveNodesWatch() throws KeeperException, - InterruptedException { - List liveNodes = zkCmdExecutor.retryOperation(new ZkOperation() { - - @Override - public Object execute() throws KeeperException, InterruptedException { - return zkClient.getChildren( - ZkStateReader.LIVE_NODES_ZKNODE, new Watcher() { - - @Override - public void process(WatchedEvent event) { - try { - List liveNodes = zkClient.getChildren( - ZkStateReader.LIVE_NODES_ZKNODE, this, true); - synchronized (nodeStateWatches) { - processLiveNodesChanged(nodeStateWatches.keySet(), liveNodes); - } - } catch (KeeperException e) { - if (e.code() == KeeperException.Code.SESSIONEXPIRED - || e.code() == KeeperException.Code.CONNECTIONLOSS) { - log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK"); - return; - } - SolrException.log(log, "", e); - throw new ZooKeeperException( - SolrException.ErrorCode.SERVER_ERROR, "", e); - } catch (InterruptedException e) { - // Restore the interrupted status - Thread.currentThread().interrupt(); - } - } - }, true); - } - }); - - processLiveNodesChanged(Collections.emptySet(), liveNodes); + /* Internal queue, not to be used outside of Overseer */ + static DistributedQueue getInternalQueue(final SolrZkClient zkClient) { + createOverseerNode(zkClient); + return new DistributedQueue(zkClient.getSolrZooKeeper(), "/overseer/queue-work", null); } - private void processLiveNodesChanged(Collection oldLiveNodes, - Collection liveNodes) throws InterruptedException, KeeperException { - - Set upNodes = complement(liveNodes, oldLiveNodes); - if (upNodes.size() > 0) { - addNodeStateWatches(upNodes); - } - - Set downNodes = complement(oldLiveNodes, liveNodes); - for(String node: downNodes) { - synchronized (nodeStateWatches) { - NodeStateWatcher watcher = nodeStateWatches.remove(node); - } - log.debug("Removed NodeStateWatcher for node:" + node); - } + /* Collection creation queue */ + static DistributedQueue getCollectionQueue(final SolrZkClient zkClient) { + createOverseerNode(zkClient); + return new DistributedQueue(zkClient.getSolrZooKeeper(), "/overseer/collection-queue-work", null); } - private void addNodeStateWatches(Set nodeNames) throws InterruptedException, KeeperException { - - for (String nodeName : nodeNames) { - final String path = STATES_NODE + "/" + nodeName; - synchronized (nodeStateWatches) { - if (!nodeStateWatches.containsKey(nodeName)) { - zkCmdExecutor.ensureExists(path, zkClient); - nodeStateWatches.put(nodeName, new NodeStateWatcher(zkClient, nodeName, path, this)); - log.debug("Added NodeStateWatcher for node " + nodeName); - } else { - log.debug("watch already added"); - } - } - } - } - - private Set complement(Collection next, - Collection prev) { - Set downCollections = new HashSet(); - downCollections.addAll(next); - downCollections.removeAll(prev); - return downCollections; - } - - @Override - public void coreChanged(final String nodeName, final Set states) - throws KeeperException, InterruptedException { - log.info("Core change pooled: " + nodeName + " states:" + states); - for (CoreState state : states) { - fifo.add(new CloudStateUpdateRequest(Op.StateChange, nodeName, state)); + private static void createOverseerNode(final SolrZkClient zkClient) { + try { + zkClient.create("/overseer", new byte[0], CreateMode.PERSISTENT, true); + } catch (KeeperException.NodeExistsException e) { + //ok + } catch (InterruptedException e) { + log.error("Could not create Overseer node: " + e.getClass() + ":" + e.getMessage()); + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } catch (KeeperException e) { + log.error("Could not create Overseer node: " + e.getClass() + ":" + e.getMessage()); + throw new RuntimeException(e); } } - @Override - public void coreDeleted(String nodeName, Collection states) - throws KeeperException, InterruptedException { - for (CoreState state : states) { - fifo.add(new CloudStateUpdateRequest(Op.CoreDeleted, state.getCollectionName(), state.getCoreNodeName())); - } - } - - public static void createClientNodes(SolrZkClient zkClient, String nodeName) throws KeeperException, InterruptedException { - final String node = STATES_NODE + "/" + nodeName; - if (log.isInfoEnabled()) { - log.info("creating node:" + node); - } - - ZkCmdExecutor zkCmdExecutor = new ZkCmdExecutor(); - zkCmdExecutor.ensureExists(node, zkClient); - } - - @Override - public void announceLeader(String collection, String shardId, ZkCoreNodeProps props) { - String coreUrl = props.getCoreUrl(); - log.info("Leader change pooled: " + coreUrl); - fifo.add(new CloudStateUpdateRequest(Op.LeaderChange, collection, shardId, coreUrl)); - } - } diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java new file mode 100644 index 0000000..090b8fe --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java @@ -0,0 +1,282 @@ +package org.apache.solr.cloud; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.cloud.CloudState; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.cloud.ZooKeeperException; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.handler.component.ShardHandler; +import org.apache.solr.handler.component.ShardRequest; +import org.apache.solr.handler.component.ShardResponse; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class OverseerCollectionProcessor implements Runnable { + public static final String DELETECOLLECTION = "deletecollection"; + + public static final String CREATECOLLECTION = "createcollection"; + + // TODO: use from Overseer? + private static final String QUEUE_OPERATION = "operation"; + + private static Logger log = LoggerFactory + .getLogger(OverseerCollectionProcessor.class); + + private DistributedQueue workQueue; + + private String myId; + + private ShardHandler shardHandler; + + private String adminPath; + + private ZkStateReader zkStateReader; + + public OverseerCollectionProcessor(ZkStateReader zkStateReader, String myId, ShardHandler shardHandler, String adminPath) { + this.zkStateReader = zkStateReader; + this.myId = myId; + this.shardHandler = shardHandler; + this.adminPath = adminPath; + workQueue = Overseer.getCollectionQueue(zkStateReader.getZkClient()); + } + + @Override + public void run() { + log.info("Process current queue of collection creations"); + while (amILeader()) { + try { + byte[] head = workQueue.peek(true); + + //if (head != null) { // should not happen since we block above + final ZkNodeProps message = ZkNodeProps.load(head); + final String operation = message.get(QUEUE_OPERATION); + + boolean success = processMessage(message, operation); + if (!success) { + // TODO: what to do on failure / partial failure + // if we fail, do we clean up then ? + SolrException.log(log, "Collection creation of " + message.get("name") + " failed"); + } + //} + workQueue.remove(); + } catch (KeeperException e) { + if (e.code() == KeeperException.Code.SESSIONEXPIRED + || e.code() == KeeperException.Code.CONNECTIONLOSS) { + log.warn("Overseer cannot talk to ZK"); + return; + } + SolrException.log(log, "", e); + throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", + e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } + + private boolean amILeader() { + try { + ZkNodeProps props = ZkNodeProps.load(zkStateReader.getZkClient().getData( + "/overseer_elect/leader", null, null, true)); + if (myId.equals(props.get("id"))) { + return true; + } + } catch (KeeperException e) { + log.warn("", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + log.info("According to ZK I (id=" + myId + ") am no longer a leader."); + return false; + } + + private boolean processMessage(ZkNodeProps message, String operation) { + if (CREATECOLLECTION.equals(operation)) { + return createCollection(zkStateReader.getCloudState(), message); + } else if (DELETECOLLECTION.equals(operation)) { + return deleteCollection(zkStateReader.getCloudState(), message); + } + // unknown command, toss it from our queue + return true; + } + + private boolean deleteCollection(CloudState cloudState, ZkNodeProps message) { + + String name = message.get("name"); + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(CoreAdminParams.ACTION, CoreAdminAction.UNLOAD.toString()); + + Map slices = cloudState.getCollectionStates().get(name); + + if (slices == null) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection:" + name); + } + + for (Map.Entry entry : slices.entrySet()) { + Slice slice = entry.getValue(); + Map shards = slice.getShards(); + Set> shardEntries = shards.entrySet(); + for (Map.Entry shardEntry : shardEntries) { + final ZkNodeProps node = shardEntry.getValue(); + if (cloudState.liveNodesContain(node.get(ZkStateReader.NODE_NAME_PROP))) { + params.set(CoreAdminParams.CORE, name); + params.set(CoreAdminParams.DELETE_INSTANCE_DIR, true); + + String replica = node.get(ZkStateReader.BASE_URL_PROP); + ShardRequest sreq = new ShardRequest(); + // yes, they must use same admin handler path everywhere... + params.set("qt", adminPath); + + sreq.purpose = 1; + // TODO: this sucks + if (replica.startsWith("http://")) replica = replica.substring(7); + sreq.shards = new String[] {replica}; + sreq.actualShards = sreq.shards; + sreq.params = params; + + shardHandler.submit(sreq, replica, sreq.params); + } + } + } + + int failed = 0; + ShardResponse srsp; + do { + srsp = shardHandler.takeCompletedOrError(); + if (srsp != null) { + Throwable e = srsp.getException(); + if (e != null) { + // should we retry? + // TODO: we should return errors to the client + // TODO: what if one fails and others succeed? + failed++; + log.error("Error talking to shard: " + srsp.getShard(), e); + } + } + } while (srsp != null); + + + // if all calls succeeded, return true + if (failed > 0) { + return false; + } + return true; + } + + // TODO: bad name conflict with another method + private boolean createCollection(CloudState cloudState, ZkNodeProps message) { + + // look at the replication factor and see if it matches reality + // if it does not, find best nodes to create more cores + + String numReplicasString = message.get("numReplicas"); + int numReplicas; + try { + numReplicas = numReplicasString == null ? 0 : Integer.parseInt(numReplicasString); + } catch (Exception ex) { + SolrException.log(log, "Could not parse numReplicas", ex); + return false; + } + String numShardsString = message.get("numShards"); + int numShards; + try { + numShards = numShardsString == null ? 0 : Integer.parseInt(numShardsString); + } catch (Exception ex) { + SolrException.log(log, "Could not parse numShards", ex); + return false; + } + + String name = message.get("name"); + String configName = message.get("collection.configName"); + + // we need to look at every node and see how many cores it serves + // add our new cores to existing nodes serving the least number of cores + // but (for now) require that each core goes on a distinct node. + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(CoreAdminParams.ACTION, CoreAdminAction.CREATE.toString()); + + + // TODO: add smarter options that look at the current number of cores per node? + // for now we just go random + Set nodes = cloudState.getLiveNodes(); + List nodeList = new ArrayList(nodes.size()); + nodeList.addAll(nodes); + Collections.shuffle(nodeList); + + int numNodes = numShards * (numReplicas + 1); + List createOnNodes = nodeList.subList(0, Math.min(nodeList.size() -1, numNodes - 1)); + + for (String replica : createOnNodes) { + // TODO: this does not work if original url had _ in it + replica = replica.replaceAll("_", "/"); + params.set(CoreAdminParams.NAME, name); + params.set("collection.configName", configName); + params.set("numShards", numShards); + ShardRequest sreq = new ShardRequest(); + params.set("qt", adminPath); + sreq.purpose = 1; + // TODO: this sucks + if (replica.startsWith("http://")) replica = replica.substring(7); + sreq.shards = new String[] {replica}; + sreq.actualShards = sreq.shards; + sreq.params = params; + + shardHandler.submit(sreq, replica, sreq.params); + } + + int failed = 0; + ShardResponse srsp; + do { + srsp = shardHandler.takeCompletedOrError(); + if (srsp != null) { + Throwable e = srsp.getException(); + if (e != null) { + // should we retry? + // TODO: we should return errors to the client + // TODO: what if one fails and others succeed? + failed++; + log.error("Error talking to shard: " + srsp.getShard(), e); + } + } + } while (srsp != null); + + + // if all calls succeeded, return true + if (failed > 0) { + return false; + } + return true; + } +} diff --git a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java index 134b94e..8928e12 100644 --- a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java +++ b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,12 +18,10 @@ package org.apache.solr.cloud; */ import java.io.IOException; -import java.net.MalformedURLException; import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -import java.util.concurrent.TimeoutException; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.HttpSolrServer; @@ -36,6 +34,7 @@ import org.apache.solr.common.cloud.SafeStopThread; import org.apache.solr.common.cloud.ZkCoreNodeProps; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.cloud.ZooKeeperException; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreDescriptor; @@ -52,6 +51,7 @@ import org.apache.solr.update.PeerSync; import org.apache.solr.update.UpdateLog; import org.apache.solr.update.UpdateLog.RecoveryInfo; import org.apache.solr.update.processor.DistributedUpdateProcessor; +import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -98,11 +98,10 @@ public class RecoveryStrategy extends Thread implements SafeStopThread { private void recoveryFailed(final SolrCore core, final ZkController zkController, final String baseUrl, - final String shardZkNodeName, final CoreDescriptor cd) { + final String shardZkNodeName, final CoreDescriptor cd) throws KeeperException, InterruptedException { SolrException.log(log, "Recovery failed - I give up."); try { - zkController.publishAsRecoveryFailed(baseUrl, cd, - shardZkNodeName, core.getName()); + zkController.publish(cd, ZkStateReader.RECOVERY_FAILED); } finally { close(); } @@ -136,7 +135,7 @@ public class RecoveryStrategy extends Thread implements SafeStopThread { } ModifiableSolrParams solrParams = new ModifiableSolrParams(); - solrParams.set(ReplicationHandler.MASTER_URL, leaderUrl + "replication"); + solrParams.set(ReplicationHandler.MASTER_URL, leaderUrl); if (isClosed()) retries = INTERRUPTED; boolean success = replicationHandler.doFetch(solrParams, true); // TODO: look into making sure force=true does not download files we already have @@ -161,8 +160,7 @@ public class RecoveryStrategy extends Thread implements SafeStopThread { } } - private void commitOnLeader(String leaderUrl) throws MalformedURLException, - SolrServerException, IOException { + private void commitOnLeader(String leaderUrl) throws SolrServerException, IOException { HttpSolrServer server = new HttpSolrServer(leaderUrl); server.setConnectionTimeout(30000); server.setSoTimeout(30000); @@ -175,7 +173,7 @@ public class RecoveryStrategy extends Thread implements SafeStopThread { } private void sendPrepRecoveryCmd(String leaderBaseUrl, - String leaderCoreName) throws MalformedURLException, SolrServerException, + String leaderCoreName) throws SolrServerException, IOException { HttpSolrServer server = new HttpSolrServer(leaderBaseUrl); server.setConnectionTimeout(45000); @@ -208,7 +206,18 @@ public class RecoveryStrategy extends Thread implements SafeStopThread { log.info("Starting recovery process. recoveringAfterStartup=" + recoveringAfterStartup); - doRecovery(core); + try { + doRecovery(core); + } catch (KeeperException e) { + log.error("", e); + throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, + "", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + SolrException.log(log, "", e); + throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", + e); + } } finally { if (core != null) core.close(); SolrRequestInfo.clearRequestInfo(); @@ -216,7 +225,7 @@ public class RecoveryStrategy extends Thread implements SafeStopThread { } // TODO: perhaps make this grab a new core each time through the loop to handle core reloads? - public void doRecovery(SolrCore core) { + public void doRecovery(SolrCore core) throws KeeperException, InterruptedException { boolean replayed = false; boolean successfulRecovery = false; @@ -327,8 +336,8 @@ public class RecoveryStrategy extends Thread implements SafeStopThread { // } // sync success - register as active and return - zkController.publishAsActive(baseUrl, core.getCoreDescriptor(), - coreZkNodeName, coreName); + zkController.publish(core.getCoreDescriptor(), + ZkStateReader.ACTIVE); successfulRecovery = true; close = true; return; @@ -352,8 +361,7 @@ public class RecoveryStrategy extends Thread implements SafeStopThread { log.info("Recovery was successful - registering as Active"); // if there are pending recovery requests, don't advert as active - zkController.publishAsActive(baseUrl, core.getCoreDescriptor(), - coreZkNodeName, coreName); + zkController.publish(core.getCoreDescriptor(), ZkStateReader.ACTIVE); close = true; successfulRecovery = true; } catch (InterruptedException e) { @@ -419,7 +427,7 @@ public class RecoveryStrategy extends Thread implements SafeStopThread { } private Future replay(UpdateLog ulog) - throws InterruptedException, ExecutionException, TimeoutException { + throws InterruptedException, ExecutionException { Future future = ulog.applyBufferedUpdates(); if (future == null) { // no replay needed\ diff --git a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderWatcher.java b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderWatcher.java deleted file mode 100644 index d15d117..0000000 --- a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderWatcher.java +++ /dev/null @@ -1,90 +0,0 @@ -package org.apache.solr.cloud; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkCoreNodeProps; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.KeeperException.Code; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A watcher for shard leader. - */ -public class ShardLeaderWatcher implements Watcher { - private static Logger logger = LoggerFactory.getLogger(ShardLeaderWatcher.class); - static interface ShardLeaderListener { - void announceLeader(String collection, String shardId, ZkCoreNodeProps props); - } - - private final String shard; - private final String collection; - private final String path; - private final SolrZkClient zkClient; - private volatile boolean closed = false; - private final ShardLeaderListener listener; - - public ShardLeaderWatcher(String shard, String collection, - SolrZkClient zkClient, ShardLeaderListener listener) throws KeeperException, InterruptedException { - this.shard = shard; - this.collection = collection; - this.path = ZkStateReader.getShardLeadersPath(collection, shard); - this.zkClient = zkClient; - this.listener = listener; - processLeaderChange(); - } - - private void processLeaderChange() throws KeeperException, InterruptedException { - if(closed) return; - try { - byte[] data = zkClient.getData(path, this, null, true); - if (data != null) { - final ZkCoreNodeProps leaderProps = new ZkCoreNodeProps(ZkNodeProps.load(data)); - listener.announceLeader(collection, shard, leaderProps); - } - } catch (KeeperException ke) { - //check if we lost connection or the node was gone - if (ke.code() != Code.CONNECTIONLOSS && ke.code() != Code.SESSIONEXPIRED - && ke.code() != Code.NONODE) { - throw ke; - } - } - } - - @Override - public void process(WatchedEvent event) { - try { - processLeaderChange(); - } catch (KeeperException e) { - logger.warn("Shard leader watch triggered but Solr cannot talk to zk."); - } catch (InterruptedException e) { - Thread.interrupted(); - logger.warn("Shard leader watch triggered but Solr cannot talk to zk."); - } - } - - public void close() { - closed = true; - } - -} diff --git a/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java b/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java index 2da89d9..203d347 100644 --- a/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java +++ b/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java b/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java index 9ca9eec..3034d12 100644 --- a/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java +++ b/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,17 +17,12 @@ package org.apache.solr.cloud; * limitations under the License. */ -import java.io.IOException; -import java.net.MalformedURLException; import java.util.ArrayList; import java.util.List; import java.util.Map; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; -import org.apache.http.params.CoreConnectionPNames; -import org.apache.solr.client.solrj.SolrServerException; +import org.apache.http.client.HttpClient; +import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.client.solrj.request.CoreAdminRequest.RequestRecovery; import org.apache.solr.common.SolrException; @@ -51,25 +46,21 @@ import org.slf4j.LoggerFactory; public class SyncStrategy { protected final Logger log = LoggerFactory.getLogger(getClass()); - private HttpShardHandlerFactory shardHandlerFactory; - - private ShardHandler shardHandler; + private final ShardHandler shardHandler; - private static ThreadSafeClientConnManager mgr = new ThreadSafeClientConnManager(); - private static DefaultHttpClient client = new DefaultHttpClient(mgr); + private final static HttpClient client; static { - mgr.setDefaultMaxPerRoute(20); - mgr.setMaxTotal(10000); - client.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 30000); - client.getParams().setIntParameter(CoreConnectionPNames.SO_TIMEOUT, 30000); - // prevent retries (note: this didn't work when set on mgr.. needed to be set on client) - DefaultHttpRequestRetryHandler retryhandler = new DefaultHttpRequestRetryHandler(0, false); - client.setHttpRequestRetryHandler(retryhandler); + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 10000); + params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 20); + params.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, 30000); + params.set(HttpClientUtil.PROP_SO_TIMEOUT, 30000); + params.set(HttpClientUtil.PROP_USE_RETRY, false); + client = HttpClientUtil.createClient(params); } public SyncStrategy() { - shardHandlerFactory = new HttpShardHandlerFactory(); - shardHandler = shardHandlerFactory.getShardHandler(client); + shardHandler = new HttpShardHandlerFactory().getShardHandler(client); } private static class SyncShardRequest extends ShardRequest { @@ -166,8 +157,7 @@ public class SyncStrategy { } private boolean syncWithReplicas(ZkController zkController, SolrCore core, - ZkNodeProps props, String collection, String shardId) - throws MalformedURLException, SolrServerException, IOException { + ZkNodeProps props, String collection, String shardId) { List nodes = zkController.getZkStateReader() .getReplicaProps(collection, shardId, props.get(ZkStateReader.NODE_NAME_PROP), @@ -197,8 +187,7 @@ public class SyncStrategy { } private void syncToMe(ZkController zkController, String collection, - String shardId, ZkNodeProps leaderProps) throws MalformedURLException, - SolrServerException, IOException { + String shardId, ZkNodeProps leaderProps) { // sync everyone else // TODO: we should do this in parallel at least diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index 3b92460..cbc67f4 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -26,7 +26,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Properties; -import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeoutException; import java.util.regex.Matcher; @@ -39,7 +38,6 @@ import org.apache.solr.client.solrj.request.CoreAdminRequest.WaitForState; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.CloudState; -import org.apache.solr.common.cloud.CoreState; import org.apache.solr.common.cloud.OnReconnect; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkCmdExecutor; @@ -53,6 +51,8 @@ import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreDescriptor; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.handler.component.HttpShardHandlerFactory; +import org.apache.solr.handler.component.ShardHandler; import org.apache.solr.update.UpdateLog; import org.apache.solr.util.DOMUtil; import org.apache.zookeeper.CreateMode; @@ -86,6 +86,9 @@ public final class ZkController { private final boolean SKIP_AUTO_RECOVERY = Boolean.getBoolean("solrcloud.skip.autorecovery"); + private final DistributedQueue overseerJobQueue; + private final DistributedQueue overseerCollectionQueue; + // package private for tests static final String CONFIGS_ZKNODE = "/configs"; @@ -93,11 +96,6 @@ public final class ZkController { public final static String COLLECTION_PARAM_PREFIX="collection."; public final static String CONFIGNAME_PROP="configName"; - private Map coreStates = new HashMap(); // key is the local core name - private long coreStatesVersion; // bumped by 1 each time we serialize coreStates... sync on coreStates - private long coreStatesPublishedVersion; // last version published to ZK... sync on coreStatesPublishLock - private Object coreStatesPublishLock = new Object(); // only publish one at a time - private final Map electionContexts = Collections.synchronizedMap(new HashMap()); private SolrZkClient zkClient; @@ -119,7 +117,8 @@ public final class ZkController { private LeaderElector overseerElector; - // this can be null in which case recovery will be inactive + // for now, this can be null in tests, in which case recovery will be inactive, and other features + // may accept defaults or use mocks rather than pulling things from a CoreContainer private CoreContainer cc; /** @@ -182,7 +181,7 @@ public final class ZkController { * @throws TimeoutException * @throws IOException */ - public ZkController(CoreContainer cc, String zkServerAddress, int zkClientTimeout, int zkClientConnectTimeout, String localHost, String locaHostPort, + public ZkController(final CoreContainer cc, String zkServerAddress, int zkClientTimeout, int zkClientConnectTimeout, String localHost, String locaHostPort, String localHostContext, final CurrentCoreDescriptorProvider registerOnReconnect) throws InterruptedException, TimeoutException, IOException { this.cc = cc; @@ -209,8 +208,19 @@ public final class ZkController { // seems we dont need to do this again... //Overseer.createClientNodes(zkClient, getNodeName()); - - ElectionContext context = new OverseerElectionContext(getNodeName(), zkClient, zkStateReader); + ShardHandler shardHandler; + String adminPath; + if (cc == null) { + shardHandler = new HttpShardHandlerFactory().getShardHandler(); + adminPath = "/admin/cores"; + } else { + shardHandler = cc.getShardHandlerFactory().getShardHandler(); + adminPath = cc.getAdminPath(); + } + + ElectionContext context = new OverseerElectionContext( + shardHandler, adminPath, + getNodeName(), zkStateReader); overseerElector.joinElection(context); zkStateReader.createClusterStateWatchersAndUpdate(); @@ -223,8 +233,7 @@ public final class ZkController { final String coreZkNodeName = getNodeName() + "_" + descriptor.getName(); try { - publishAsDown(getBaseUrl(), descriptor, coreZkNodeName, - descriptor.getName()); + publish(descriptor, ZkStateReader.DOWN); waitForLeaderToSeeDownState(descriptor, coreZkNodeName); } catch (Exception e) { SolrException.log(log, "", e); @@ -261,6 +270,8 @@ public final class ZkController { }); + this.overseerJobQueue = Overseer.getInQueue(zkClient); + this.overseerCollectionQueue = Overseer.getCollectionQueue(zkClient); cmdExecutor = new ZkCmdExecutor(); leaderElector = new LeaderElector(zkClient); zkStateReader = new ZkStateReader(zkClient); @@ -302,13 +313,6 @@ public final class ZkController { return zkStateReader.getCloudState(); } - /** @return the CoreState for the core, which may not yet be visible to ZooKeeper or other nodes in the cluster */ - public CoreState getCoreState(String coreName) { - synchronized (coreStates) { - return coreStates.get(coreName); - } - } - /** * @param zkConfigName * @param fileName @@ -387,14 +391,22 @@ public final class ZkController { // makes nodes zkNode cmdExecutor.ensureExists(ZkStateReader.LIVE_NODES_ZKNODE, zkClient); - Overseer.createClientNodes(zkClient, getNodeName()); createEphemeralLiveNode(); cmdExecutor.ensureExists(ZkStateReader.COLLECTIONS_ZKNODE, zkClient); - syncNodeState(); - + ShardHandler shardHandler; + String adminPath; + if (cc == null) { + shardHandler = new HttpShardHandlerFactory().getShardHandler(); + adminPath = "/admin/cores"; + } else { + shardHandler = cc.getShardHandlerFactory().getShardHandler(); + adminPath = cc.getAdminPath(); + } + overseerElector = new LeaderElector(zkClient); - ElectionContext context = new OverseerElectionContext(getNodeName(), zkClient, zkStateReader); + ElectionContext context = new OverseerElectionContext(shardHandler, + adminPath, getNodeName(), zkStateReader); overseerElector.setup(context); overseerElector.joinElection(context); zkStateReader.createClusterStateWatchersAndUpdate(); @@ -416,27 +428,6 @@ public final class ZkController { } } - - /* - * sync internal state with zk on startup - */ - private void syncNodeState() throws KeeperException, InterruptedException { - log.debug("Syncing internal state with zk. Current: " + coreStates); - final String path = Overseer.STATES_NODE + "/" + getNodeName(); - - final byte[] data = zkClient.getData(path, null, null, true); - - if (data != null) { - CoreState[] states = CoreState.load(data); - synchronized (coreStates) { - coreStates.clear(); // TODO: should we do this? - for(CoreState coreState: states) { - coreStates.put(coreState.getCoreName(), coreState); - } - } - } - log.debug("after sync: " + coreStates); - } public boolean isConnected() { return zkClient.isConnected(); @@ -496,10 +487,9 @@ public final class ZkController { * @return config value * @throws KeeperException * @throws InterruptedException - * @throws IOException */ public String readConfigName(String collection) throws KeeperException, - InterruptedException, IOException { + InterruptedException { String configName = null; @@ -640,7 +630,7 @@ public final class ZkController { boolean didRecovery = checkRecovery(coreName, desc, recoverReloadedCores, isLeader, cloudDesc, collection, coreZkNodeName, shardId, leaderProps, core, cc); if (!didRecovery) { - publishAsActive(baseUrl, desc, coreZkNodeName, coreName); + publish(desc, ZkStateReader.ACTIVE); } } finally { if (core != null) { @@ -648,7 +638,7 @@ public final class ZkController { } } } else { - publishAsActive(baseUrl, desc, coreZkNodeName, coreName); + publish(desc, ZkStateReader.ACTIVE); } // make sure we have an update cluster state right away @@ -721,17 +711,12 @@ public final class ZkController { * @param core * @param cc * @return whether or not a recovery was started - * @throws InterruptedException - * @throws KeeperException - * @throws IOException - * @throws ExecutionException */ private boolean checkRecovery(String coreName, final CoreDescriptor desc, boolean recoverReloadedCores, final boolean isLeader, final CloudDescriptor cloudDesc, final String collection, final String shardZkNodeName, String shardId, ZkNodeProps leaderProps, - SolrCore core, CoreContainer cc) throws InterruptedException, - KeeperException, IOException, ExecutionException { + SolrCore core, CoreContainer cc) { if (SKIP_AUTO_RECOVERY) { log.warn("Skipping recovery according to sys prop solrcloud.skip.autorecovery"); return false; @@ -760,50 +745,34 @@ public final class ZkController { return baseURL; } - - void publishAsActive(String shardUrl, - final CoreDescriptor cd, String shardZkNodeName, String coreName) { - Map finalProps = new HashMap(); - finalProps.put(ZkStateReader.BASE_URL_PROP, shardUrl); - finalProps.put(ZkStateReader.CORE_NAME_PROP, coreName); - finalProps.put(ZkStateReader.NODE_NAME_PROP, getNodeName()); - finalProps.put(ZkStateReader.STATE_PROP, ZkStateReader.ACTIVE); - - publishState(cd, shardZkNodeName, coreName, finalProps); - } - - public void publish(CoreDescriptor cd, String state) { - Map finalProps = new HashMap(); - finalProps.put(ZkStateReader.BASE_URL_PROP, getBaseUrl()); - finalProps.put(ZkStateReader.CORE_NAME_PROP, cd.getName()); - finalProps.put(ZkStateReader.NODE_NAME_PROP, getNodeName()); - finalProps.put(ZkStateReader.STATE_PROP, state); - publishState(cd, getNodeName() + "_" + cd.getName(), - cd.getName(), finalProps); - } - - void publishAsDown(String baseUrl, - final CoreDescriptor cd, String shardZkNodeName, String coreName) { - Map finalProps = new HashMap(); - finalProps.put(ZkStateReader.BASE_URL_PROP, baseUrl); - finalProps.put(ZkStateReader.CORE_NAME_PROP, coreName); - finalProps.put(ZkStateReader.NODE_NAME_PROP, getNodeName()); - finalProps.put(ZkStateReader.STATE_PROP, ZkStateReader.DOWN); - - publishState(cd, shardZkNodeName, coreName, finalProps); - } - - void publishAsRecoveryFailed(String baseUrl, - final CoreDescriptor cd, String shardZkNodeName, String coreName) { - Map finalProps = new HashMap(); - finalProps.put(ZkStateReader.BASE_URL_PROP, baseUrl); - finalProps.put(ZkStateReader.CORE_NAME_PROP, coreName); - finalProps.put(ZkStateReader.NODE_NAME_PROP, getNodeName()); - finalProps.put(ZkStateReader.STATE_PROP, ZkStateReader.RECOVERY_FAILED); - publishState(cd, shardZkNodeName, coreName, finalProps); + /** + * Publish core state to overseer. + * @param cd + * @param state + * @throws KeeperException + * @throws InterruptedException + */ + public void publish(final CoreDescriptor cd, final String state) throws KeeperException, InterruptedException { + //System.out.println(Thread.currentThread().getStackTrace()[3]); + Integer numShards = cd.getCloudDescriptor().getNumShards(); + if (numShards == null) { //XXX sys prop hack + numShards = Integer.getInteger(ZkStateReader.NUM_SHARDS_PROP); + } + + ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, "state", + ZkStateReader.STATE_PROP, state, + ZkStateReader.BASE_URL_PROP, getBaseUrl(), + ZkStateReader.CORE_NAME_PROP, cd.getName(), + ZkStateReader.ROLES_PROP, cd.getCloudDescriptor().getRoles(), + ZkStateReader.NODE_NAME_PROP, getNodeName(), + ZkStateReader.SHARD_ID_PROP, cd.getCloudDescriptor().getShardId(), + ZkStateReader.COLLECTION_PROP, cd.getCloudDescriptor() + .getCollectionName(), ZkStateReader.STATE_PROP, state, + ZkStateReader.NUM_SHARDS_PROP, numShards != null ? numShards.toString() + : null); + overseerJobQueue.offer(ZkStateReader.toJSON(m)); } - private boolean needsToBeAssignedShardId(final CoreDescriptor desc, final CloudState state, final String shardZkNodeName) { @@ -826,16 +795,26 @@ public final class ZkController { */ public void unregister(String coreName, CloudDescriptor cloudDesc) throws InterruptedException, KeeperException { - synchronized (coreStates) { - coreStates.remove(coreName); - } - publishState(); + ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, + "deletecore", ZkStateReader.CORE_NAME_PROP, coreName, + ZkStateReader.NODE_NAME_PROP, getNodeName(), + ZkStateReader.COLLECTION_PROP, cloudDesc.getCollectionName()); + overseerJobQueue.offer(ZkStateReader.toJSON(m)); + final String zkNodeName = getNodeName() + "_" + coreName; ElectionContext context = electionContexts.remove(zkNodeName); if (context != null) { context.cancelElection(); } } + + public void createCollection(String collection) throws KeeperException, + InterruptedException { + ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, + "createcollection", ZkStateReader.NODE_NAME_PROP, getNodeName(), + ZkStateReader.COLLECTION_PROP, collection); + overseerJobQueue.offer(ZkStateReader.toJSON(m)); + } /** * @param dir @@ -864,7 +843,7 @@ public final class ZkController { zkClient.printLayoutToStdOut(); } - public void createCollectionZkNode(CloudDescriptor cd) throws KeeperException, InterruptedException, IOException { + public void createCollectionZkNode(CloudDescriptor cd) throws KeeperException, InterruptedException { String collection = cd.getCollectionName(); log.info("Check for collection zkNode:" + collection); @@ -993,85 +972,7 @@ public final class ZkController { return zkStateReader; } - - private void publishState(CoreDescriptor cd, String shardZkNodeName, String coreName, - Map props) { - CloudDescriptor cloudDesc = cd.getCloudDescriptor(); - if (cloudDesc.getRoles() != null) { - props.put(ZkStateReader.ROLES_PROP, cloudDesc.getRoles()); - } - - if (cloudDesc.getShardId() == null && needsToBeAssignedShardId(cd, zkStateReader.getCloudState(), shardZkNodeName)) { - // publish with no shard id so we are assigned one, and then look for it - doPublish(shardZkNodeName, coreName, props, cloudDesc); - String shardId; - try { - shardId = doGetShardIdProcess(coreName, cloudDesc); - } catch (InterruptedException e) { - throw new SolrException(ErrorCode.SERVER_ERROR, "Interrupted"); - } - cloudDesc.setShardId(shardId); - } - - - if (!props.containsKey(ZkStateReader.SHARD_ID_PROP) && cloudDesc.getShardId() != null) { - props.put(ZkStateReader.SHARD_ID_PROP, cloudDesc.getShardId()); - } - - doPublish(shardZkNodeName, coreName, props, cloudDesc); - } - - - private void doPublish(String shardZkNodeName, String coreName, - Map props, CloudDescriptor cloudDesc) { - Integer numShards = cloudDesc.getNumShards(); - if (numShards == null) { - numShards = Integer.getInteger(ZkStateReader.NUM_SHARDS_PROP); - } - CoreState coreState = new CoreState(coreName, - cloudDesc.getCollectionName(), props, numShards); - - synchronized (coreStates) { - coreStates.put(coreName, coreState); - } - - publishState(); - } - - private void publishState() { - final String nodePath = "/node_states/" + getNodeName(); - - long version; - byte[] coreStatesData; - synchronized (coreStates) { - version = ++coreStatesVersion; - coreStatesData = ZkStateReader.toJSON(coreStates.values()); - } - - // if multiple threads are trying to publish state, make sure that we never write - // an older version after a newer version. - synchronized (coreStatesPublishLock) { - try { - if (version < coreStatesPublishedVersion) { - log.info("Another thread already published a newer coreStates: ours="+version + " lastPublished=" + coreStatesPublishedVersion); - } else { - zkClient.setData(nodePath, coreStatesData, true); - coreStatesPublishedVersion = version; // put it after so it won't be set if there's an exception - } - } catch (KeeperException e) { - throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, - "could not publish node state", e); - } catch (InterruptedException e) { - // Restore the interrupted status - Thread.currentThread().interrupt(); - throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, - "could not publish node state", e); - } - } - } - - private String doGetShardIdProcess(String coreName, CloudDescriptor descriptor) - throws InterruptedException { + private String doGetShardIdProcess(String coreName, CloudDescriptor descriptor) { final String shardZkNodeName = getNodeName() + "_" + coreName; int retryCount = 120; while (retryCount-- > 0) { @@ -1086,6 +987,7 @@ public final class ZkController { Thread.currentThread().interrupt(); } } + throw new SolrException(ErrorCode.SERVER_ERROR, "Could not get shard_id for core: " + coreName); } @@ -1106,14 +1008,26 @@ public final class ZkController { } } + private String getCoreNodeName(CoreDescriptor descriptor){ + return getNodeName() + "_" + + descriptor.getName(); + } + public static void uploadConfigDir(SolrZkClient zkClient, File dir, String configName) throws IOException, KeeperException, InterruptedException { uploadToZK(zkClient, dir, ZkController.CONFIGS_ZKNODE + "/" + configName); } - public void preRegister(CoreDescriptor cd) { + public void preRegister(CoreDescriptor cd) throws KeeperException, InterruptedException { // before becoming available, make sure we are not live and active // this also gets us our assigned shard id if it was not specified publish(cd, ZkStateReader.DOWN); + String shardZkNodeName = getCoreNodeName(cd); + if (cd.getCloudDescriptor().getShardId() == null && needsToBeAssignedShardId(cd, zkStateReader.getCloudState(), shardZkNodeName)) { + String shardId; + shardId = doGetShardIdProcess(cd.getName(), cd.getCloudDescriptor()); + cd.getCloudDescriptor().setShardId(shardId); + } + } private ZkCoreNodeProps waitForLeaderToSeeDownState( @@ -1215,4 +1129,12 @@ public final class ZkController { } } + public DistributedQueue getOverseerJobQueue() { + return overseerJobQueue; + } + + public DistributedQueue getOverseerCollectionQueue() { + return overseerCollectionQueue; + } + } diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java b/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java index 46739f9..42a9a8c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/AbstractSolrEventListener.java b/solr/core/src/java/org/apache/solr/core/AbstractSolrEventListener.java index 2634ef7..6d66d43 100644 --- a/solr/core/src/java/org/apache/solr/core/AbstractSolrEventListener.java +++ b/solr/core/src/java/org/apache/solr/core/AbstractSolrEventListener.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java index 8d86342..92b980d 100644 --- a/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/CloseHook.java b/solr/core/src/java/org/apache/solr/core/CloseHook.java index 1885014..7431c1f 100644 --- a/solr/core/src/java/org/apache/solr/core/CloseHook.java +++ b/solr/core/src/java/org/apache/solr/core/CloseHook.java @@ -1,5 +1,5 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/CodecFactory.java b/solr/core/src/java/org/apache/solr/core/CodecFactory.java index 68b9089..32db7da 100644 --- a/solr/core/src/java/org/apache/solr/core/CodecFactory.java +++ b/solr/core/src/java/org/apache/solr/core/CodecFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/Config.java b/solr/core/src/java/org/apache/solr/core/Config.java index f238a35..4ecb3ba 100644 --- a/solr/core/src/java/org/apache/solr/core/Config.java +++ b/solr/core/src/java/org/apache/solr/core/Config.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index b09d8cc..b132dd6 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -62,6 +62,7 @@ import org.apache.solr.util.FileUtils; import org.apache.solr.util.SystemIdResolver; import org.apache.solr.core.SolrXMLSerializer.SolrCoreXMLDef; import org.apache.solr.core.SolrXMLSerializer.SolrXMLDef; +import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.handler.admin.CoreAdminHandler; import org.apache.solr.handler.component.HttpShardHandlerFactory; import org.apache.solr.handler.component.ShardHandlerFactory; @@ -103,6 +104,7 @@ public class CoreContainer protected String hostContext; protected String host; protected CoreAdminHandler coreAdminHandler = null; + protected CollectionsHandler collectionsHandler = null; protected File configFile = null; protected String libDir = null; protected ClassLoader libLoader = null; @@ -453,6 +455,8 @@ public class CoreContainer } } + collectionsHandler = new CollectionsHandler(this); + try { containerProperties = readProperties(cfg, ((NodeList) cfg.evaluate(DEFAULT_HOST_CONTEXT, XPathConstants.NODESET)).item(0)); } catch (Throwable e) { @@ -629,7 +633,18 @@ public class CoreContainer if (zkController != null) { // this happens before we can receive requests - zkController.preRegister(core.getCoreDescriptor()); + try { + zkController.preRegister(core.getCoreDescriptor()); + } catch (KeeperException e) { + log.error("", e); + throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, + "", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.error("", e); + throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, + "", e); + } } SolrCore old = null; @@ -662,7 +677,6 @@ public class CoreContainer } } - private void registerInZk(SolrCore core) { if (zkController != null) { try { @@ -676,7 +690,18 @@ public class CoreContainer } catch (Exception e) { // if register fails, this is really bad - close the zkController to // minimize any damage we can cause - zkController.publish(core.getCoreDescriptor(), ZkStateReader.DOWN); + try { + zkController.publish(core.getCoreDescriptor(), ZkStateReader.DOWN); + } catch (KeeperException e1) { + log.error("", e); + throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, + "", e); + } catch (InterruptedException e1) { + Thread.currentThread().interrupt(); + log.error("", e); + throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, + "", e); + } SolrException.log(log, "", e); throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e); @@ -1002,6 +1027,10 @@ public class CoreContainer return coreAdminHandler; } + public CollectionsHandler getCollectionsHandler() { + return collectionsHandler; + } + /** * the default core name, or null if there is no default core name */ @@ -1300,7 +1329,7 @@ public class CoreContainer private static final String DEF_SOLR_XML ="\n" + "\n" + " \n" + - " \n" + + " \n" + " \n" + ""; } diff --git a/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java b/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java index 390a036..27ba112 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java +++ b/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/DefaultCodecFactory.java b/solr/core/src/java/org/apache/solr/core/DefaultCodecFactory.java index 432d63f..54d3473 100644 --- a/solr/core/src/java/org/apache/solr/core/DefaultCodecFactory.java +++ b/solr/core/src/java/org/apache/solr/core/DefaultCodecFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java index 0b4a710..e3b8fe3 100644 --- a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java b/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java index 6b33e9c..38930be 100644 --- a/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java +++ b/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java @@ -1,5 +1,5 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java b/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java index b9e7409..c7c8143 100644 --- a/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java +++ b/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java @@ -1,5 +1,5 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java b/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java index 4cdd48d..818b062 100644 --- a/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java +++ b/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java index b78a5ea..95e44d2 100644 --- a/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java @@ -1,5 +1,5 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/NIOFSDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/NIOFSDirectoryFactory.java index 8b9d803..4906674 100644 --- a/solr/core/src/java/org/apache/solr/core/NIOFSDirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/NIOFSDirectoryFactory.java @@ -1,5 +1,5 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/NRTCachingDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/NRTCachingDirectoryFactory.java index 25b8769..957d3a2 100644 --- a/solr/core/src/java/org/apache/solr/core/NRTCachingDirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/NRTCachingDirectoryFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/PluginInfo.java b/solr/core/src/java/org/apache/solr/core/PluginInfo.java index 682a9b1..c6b6e3b 100644 --- a/solr/core/src/java/org/apache/solr/core/PluginInfo.java +++ b/solr/core/src/java/org/apache/solr/core/PluginInfo.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java b/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java index 3903f91..6d0b1fe 100644 --- a/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java +++ b/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/RAMDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/RAMDirectoryFactory.java index f264d36..546f52e 100644 --- a/solr/core/src/java/org/apache/solr/core/RAMDirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/RAMDirectoryFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/RequestHandlers.java b/solr/core/src/java/org/apache/solr/core/RequestHandlers.java index 2a47953..1246ef0 100644 --- a/solr/core/src/java/org/apache/solr/core/RequestHandlers.java +++ b/solr/core/src/java/org/apache/solr/core/RequestHandlers.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/RunExecutableListener.java b/solr/core/src/java/org/apache/solr/core/RunExecutableListener.java index 62f554e..43fbc58 100644 --- a/solr/core/src/java/org/apache/solr/core/RunExecutableListener.java +++ b/solr/core/src/java/org/apache/solr/core/RunExecutableListener.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/SimpleFSDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/SimpleFSDirectoryFactory.java index 80202e9..69da63b 100644 --- a/solr/core/src/java/org/apache/solr/core/SimpleFSDirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/SimpleFSDirectoryFactory.java @@ -1,5 +1,5 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java index fec79ad..c8d81bf 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -112,7 +112,7 @@ public class SolrConfig extends Config { *@param name the configuration name *@param is the configuration stream */ - SolrConfig(SolrResourceLoader loader, String name, InputSource is) + public SolrConfig(SolrResourceLoader loader, String name, InputSource is) throws ParserConfigurationException, IOException, SAXException { super(loader, name, is, "/config/"); initLibs(); diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 59760fc..834be65 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -25,18 +25,35 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.store.LockObtainFailedException; +import org.apache.solr.cloud.CloudDescriptor; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.CommonParams.EchoParamStyle; -import org.apache.solr.common.params.ShardParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.handler.admin.ShowFileRequestHandler; -import org.apache.solr.handler.component.*; +import org.apache.solr.handler.component.DebugComponent; +import org.apache.solr.handler.component.FacetComponent; +import org.apache.solr.handler.component.HighlightComponent; +import org.apache.solr.handler.component.MoreLikeThisComponent; +import org.apache.solr.handler.component.QueryComponent; +import org.apache.solr.handler.component.RealTimeGetComponent; +import org.apache.solr.handler.component.SearchComponent; +import org.apache.solr.handler.component.StatsComponent; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; -import org.apache.solr.response.*; +import org.apache.solr.response.BinaryResponseWriter; +import org.apache.solr.response.CSVResponseWriter; +import org.apache.solr.response.JSONResponseWriter; +import org.apache.solr.response.PHPResponseWriter; +import org.apache.solr.response.PHPSerializedResponseWriter; +import org.apache.solr.response.PythonResponseWriter; +import org.apache.solr.response.QueryResponseWriter; +import org.apache.solr.response.RawResponseWriter; +import org.apache.solr.response.RubyResponseWriter; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.response.XMLResponseWriter; import org.apache.solr.response.transform.TransformerFactory; import org.apache.solr.schema.IndexSchema; import org.apache.solr.search.QParserPlugin; @@ -46,7 +63,11 @@ import org.apache.solr.search.ValueSourceParser; import org.apache.solr.update.DirectUpdateHandler2; import org.apache.solr.update.SolrIndexWriter; import org.apache.solr.update.UpdateHandler; -import org.apache.solr.update.processor.*; +import org.apache.solr.update.processor.DistributedUpdateProcessorFactory; +import org.apache.solr.update.processor.LogUpdateProcessorFactory; +import org.apache.solr.update.processor.RunUpdateProcessorFactory; +import org.apache.solr.update.processor.UpdateRequestProcessorChain; +import org.apache.solr.update.processor.UpdateRequestProcessorFactory; import org.apache.solr.util.RefCounted; import org.apache.solr.util.plugin.NamedListInitializedPlugin; import org.apache.solr.util.plugin.PluginInfoInitialized; @@ -56,11 +77,35 @@ import org.slf4j.LoggerFactory; import org.xml.sax.SAXException; import javax.xml.parsers.ParserConfigurationException; -import java.io.*; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.Writer; import java.lang.reflect.Constructor; import java.net.URL; -import java.util.*; -import java.util.concurrent.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.StringTokenizer; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantLock; @@ -75,7 +120,7 @@ public final class SolrCore implements SolrInfoMBean { // These should *only* be used for debugging or monitoring purposes public static final AtomicLong numOpens = new AtomicLong(); public static final AtomicLong numCloses = new AtomicLong(); - public static Map openHandles = Collections.synchronizedMap(new IdentityHashMap()); + public static Map openHandles = Collections.synchronizedMap(new IdentityHashMap()); public static Logger log = LoggerFactory.getLogger(SolrCore.class); @@ -198,8 +243,14 @@ public final class SolrCore implements SolrInfoMBean { result = dataDir + s; } } + if (!result.equals(lastNewIndexDir)) { + log.warn("New index directory detected: old="+lastNewIndexDir + " new=" + result); + } + lastNewIndexDir = result; return result; } + private String lastNewIndexDir; // for debugging purposes only... access not synchronized, but that's ok + public DirectoryFactory getDirectoryFactory() { return directoryFactory; @@ -831,13 +882,14 @@ public final class SolrCore implements SolrInfoMBean { public static void verbose(Object... args) { if (!VERBOSE) return; StringBuilder sb = new StringBuilder("VERBOSE:"); - sb.append(Thread.currentThread().getName()); - sb.append(':'); +// sb.append(Thread.currentThread().getName()); +// sb.append(':'); for (Object o : args) { sb.append(' '); sb.append(o==null ? "(null)" : o.toString()); } - System.out.println(sb.toString()); + // System.out.println(sb.toString()); + log.info(sb.toString()); } @@ -997,12 +1049,7 @@ public final class SolrCore implements SolrInfoMBean { * {@link org.apache.solr.request.SolrQueryRequest#getSearcher()} instead. */ public RefCounted getSearcher() { - try { - return getSearcher(false,true,null); - } catch (IOException e) { - SolrException.log(log,null,e); - return null; - } + return getSearcher(false,true,null); } /** @@ -1066,7 +1113,7 @@ public final class SolrCore implements SolrInfoMBean { } - public RefCounted getSearcher(boolean forceNew, boolean returnSearcher, final Future[] waitSearcher) throws IOException { + public RefCounted getSearcher(boolean forceNew, boolean returnSearcher, final Future[] waitSearcher) { return getSearcher(forceNew, returnSearcher, waitSearcher, false); } @@ -1211,9 +1258,8 @@ public final class SolrCore implements SolrInfoMBean { * @param returnSearcher if true, returns a {@link SolrIndexSearcher} holder with the refcount already incremented. * @param waitSearcher if non-null, will be filled in with a {@link Future} that will return after the new searcher is registered. * @param updateHandlerReopens if true, the UpdateHandler will be used when reopening a {@link SolrIndexSearcher}. - * @throws IOException */ - public RefCounted getSearcher(boolean forceNew, boolean returnSearcher, final Future[] waitSearcher, boolean updateHandlerReopens) throws IOException { + public RefCounted getSearcher(boolean forceNew, boolean returnSearcher, final Future[] waitSearcher, boolean updateHandlerReopens) { // it may take some time to open an index.... we may need to make // sure that two threads aren't trying to open one at the same time // if it isn't necessary. @@ -1470,7 +1516,7 @@ public final class SolrCore implements SolrInfoMBean { // // onDeckSearchers will also be decremented (it should have been incremented // as a result of opening a new searcher). - private void registerSearcher(RefCounted newSearcherHolder) throws IOException { + private void registerSearcher(RefCounted newSearcherHolder) { synchronized (searcherLock) { try { if (_searcher != null) { @@ -1747,7 +1793,7 @@ public final class SolrCore implements SolrInfoMBean { public QParserPlugin getQueryPlugin(String parserName) { QParserPlugin plugin = qParserPlugins.get(parserName); if (plugin != null) return plugin; - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown query type '"+parserName+"'"); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown query parser '"+parserName+"'"); } private final HashMap valueSourceParsers = new HashMap(); @@ -1935,9 +1981,18 @@ public final class SolrCore implements SolrInfoMBean { lst.add("startTime", new Date(startTime)); lst.add("refCount", getOpenCount()); - if (null != getCoreDescriptor() && null != getCoreDescriptor().getCoreContainer()) { - lst.add("aliases", getCoreDescriptor().getCoreContainer().getCoreNames(this)); + CoreDescriptor cd = getCoreDescriptor(); + if (cd != null) { + if (null != cd && cd.getCoreContainer() != null) { + lst.add("aliases", getCoreDescriptor().getCoreContainer().getCoreNames(this)); + } + CloudDescriptor cloudDesc = cd.getCloudDescriptor(); + if (cloudDesc != null) { + lst.add("collection", cloudDesc.getCollectionName()); + lst.add("shard", cloudDesc.getShardId()); + } } + return lst; } diff --git a/solr/core/src/java/org/apache/solr/core/SolrDeletionPolicy.java b/solr/core/src/java/org/apache/solr/core/SolrDeletionPolicy.java index d8035e4..6ac3f0a 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrDeletionPolicy.java +++ b/solr/core/src/java/org/apache/solr/core/SolrDeletionPolicy.java @@ -1,5 +1,5 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/SolrEventListener.java b/solr/core/src/java/org/apache/solr/core/SolrEventListener.java index 1cc131e..0c67f76 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrEventListener.java +++ b/solr/core/src/java/org/apache/solr/core/SolrEventListener.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java b/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java index f5b1bd1..8458e1f 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java +++ b/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java index 8cf3446..e4df2a0 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java +++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/SolrXMLSerializer.java b/solr/core/src/java/org/apache/solr/core/SolrXMLSerializer.java index 266caaa..484fe42 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrXMLSerializer.java +++ b/solr/core/src/java/org/apache/solr/core/SolrXMLSerializer.java @@ -1,6 +1,6 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java index 8afbce3..87617f8 100644 --- a/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java @@ -1,5 +1,5 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java b/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java index 615e9d9..133eba8 100644 --- a/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java +++ b/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java @@ -1,5 +1,5 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java index b59b781..6bfdc24 100644 --- a/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java +++ b/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -26,7 +26,6 @@ import org.apache.lucene.analysis.util.CharFilterFactory; import org.apache.lucene.analysis.util.TokenFilterFactory; import org.apache.lucene.analysis.util.TokenizerFactory; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.index.Payload; import org.apache.lucene.util.Attribute; import org.apache.lucene.util.AttributeImpl; import org.apache.lucene.util.AttributeSource; @@ -113,7 +112,7 @@ public abstract class AnalysisRequestHandlerBase extends RequestHandlerBase { } } - TokenStream tokenStream = tfac.create(tokenizerChain.initReader(new StringReader(value))); + TokenStream tokenStream = tfac.create(tokenizerChain.initReader(null, new StringReader(value))); List tokens = analyzeTokenStream(tokenStream); namedList.add(tokenStream.getClass().getName(), convertTokensToNamedLists(tokens, context)); @@ -273,9 +272,9 @@ public abstract class AnalysisRequestHandlerBase extends RequestHandlerBase { k = ATTRIBUTE_MAPPING.get(k); } - if (value instanceof Payload) { - final Payload p = (Payload) value; - value = new BytesRef(p.getData()).toString(); + if (value instanceof BytesRef) { + final BytesRef p = (BytesRef) value; + value = p.toString(); } tokenNamedList.add(k, value); @@ -327,7 +326,7 @@ public abstract class AnalysisRequestHandlerBase extends RequestHandlerBase { } @Override - public boolean incrementToken() throws IOException { + public boolean incrementToken() { if (tokenIterator.hasNext()) { clearAttributes(); AttributeSource next = tokenIterator.next(); diff --git a/solr/core/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java index 70d3179..cac839e 100644 --- a/solr/core/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java index cf6a95d..5adce48 100755 --- a/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/ContentStreamHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/ContentStreamHandlerBase.java index 4bd2011..125c613 100644 --- a/solr/core/src/java/org/apache/solr/handler/ContentStreamHandlerBase.java +++ b/solr/core/src/java/org/apache/solr/handler/ContentStreamHandlerBase.java @@ -1,5 +1,5 @@ package org.apache.solr.handler; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/ContentStreamLoader.java b/solr/core/src/java/org/apache/solr/handler/ContentStreamLoader.java index cc37681..dd980d1 100644 --- a/solr/core/src/java/org/apache/solr/handler/ContentStreamLoader.java +++ b/solr/core/src/java/org/apache/solr/handler/ContentStreamLoader.java @@ -1,5 +1,5 @@ package org.apache.solr.handler; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java index 6ef0b9a..c1602a0 100644 --- a/solr/core/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/DumpRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/DumpRequestHandler.java index 2b974c2..7556c30 100644 --- a/solr/core/src/java/org/apache/solr/handler/DumpRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/DumpRequestHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java index 9333d06..d7f0ca7 100644 --- a/solr/core/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java index c280be5..f4254a2 100644 --- a/solr/core/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java index f8f459f..7a5a0fa 100644 --- a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/PingRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/PingRequestHandler.java index 4206f63..68e0606 100644 --- a/solr/core/src/java/org/apache/solr/handler/PingRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/PingRequestHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/RealTimeGetHandler.java b/solr/core/src/java/org/apache/solr/handler/RealTimeGetHandler.java index 2d9a466..03ff291 100644 --- a/solr/core/src/java/org/apache/solr/handler/RealTimeGetHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/RealTimeGetHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java index f954118..74dd04f 100644 --- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -42,7 +42,6 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexDeletionPolicy; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.DirectoryReader; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; @@ -602,10 +601,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw if (showSlaveDetails && snapPuller != null) { Properties props = loadReplicationProperties(); try { - NamedList command = new NamedList(); - command.add(COMMAND, CMD_DETAILS); - command.add("slave", "false"); - NamedList nl = snapPuller.getCommandResponse(command); + NamedList nl = snapPuller.getDetails(); slave.add("masterDetails", nl.get(CMD_DETAILS)); } catch (Exception e) { LOG.warn("Exception while invoking 'details' method for replication on master ", e); @@ -823,7 +819,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw List files = Arrays.asList(includeConfFiles.split(",")); for (String file : files) { if (file.trim().length() == 0) continue; - String[] strs = file.split(":"); + String[] strs = file.trim().split(":"); // if there is an alias add it or it is null confFileNameAlias.add(strs[0], strs.length > 1 ? strs[1] : null); } @@ -946,7 +942,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw stream.write(out); } - public void write(Writer writer, SolrQueryRequest request, SolrQueryResponse response) throws IOException { + public void write(Writer writer, SolrQueryRequest request, SolrQueryResponse response) { throw new RuntimeException("This is a binary writer , Cannot write to a characterstream"); } diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java index 2a533f0..20d05d0 100644 --- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java +++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java b/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java index c84f68d..fb5e8e4 100755 --- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java +++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -90,8 +90,7 @@ public class RequestHandlerUtils /** * Modify UpdateCommand based on request parameters */ - public static void updateCommit(CommitUpdateCommand cmd, SolrParams params) throws IOException - { + public static void updateCommit(CommitUpdateCommand cmd, SolrParams params) { if( params == null ) return; cmd.openSearcher = params.getBool( UpdateParams.OPEN_SEARCHER, cmd.openSearcher ); diff --git a/solr/core/src/java/org/apache/solr/handler/SnapPuller.java b/solr/core/src/java/org/apache/solr/handler/SnapPuller.java index 9179fff..a0518f4 100644 --- a/solr/core/src/java/org/apache/solr/handler/SnapPuller.java +++ b/solr/core/src/java/org/apache/solr/handler/SnapPuller.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,26 +17,19 @@ package org.apache.solr.handler; import org.apache.commons.io.IOUtils; -import org.apache.http.Header; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.HttpClient; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; -import org.apache.http.message.AbstractHttpMessage; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.params.CoreConnectionPNames; -import org.apache.http.util.EntityUtils; import org.apache.lucene.index.IndexCommit; +import org.apache.solr.client.solrj.SolrServer; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpClientUtil; +import org.apache.solr.client.solrj.impl.HttpSolrServer; +import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.FastInputStream; import org.apache.solr.util.FileUtils; -import org.apache.solr.common.util.JavaBinCodec; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; import org.apache.solr.core.IndexDeletionPolicyWrapper; @@ -61,7 +54,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.zip.Adler32; import java.util.zip.Checksum; -import java.util.zip.GZIPInputStream; import java.util.zip.InflaterInputStream; /** @@ -120,43 +112,47 @@ public class SnapPuller { // HttpClient for this instance if connectionTimeout or readTimeout has been specified private final HttpClient myHttpClient; - private static synchronized HttpClient createHttpClient(String connTimeout, String readTimeout, String httpBasicAuthUser, String httpBasicAuthPassword) { + private static synchronized HttpClient createHttpClient(String connTimeout, String readTimeout, String httpBasicAuthUser, String httpBasicAuthPassword, boolean useCompression) { if (connTimeout == null && readTimeout == null && client != null) return client; - ThreadSafeClientConnManager mgr = new ThreadSafeClientConnManager(); + final ModifiableSolrParams httpClientParams = new ModifiableSolrParams(); + httpClientParams.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, connTimeout != null ? connTimeout : "5000"); + httpClientParams.set(HttpClientUtil.PROP_SO_TIMEOUT, readTimeout != null ? readTimeout : "20000"); + httpClientParams.set(HttpClientUtil.PROP_BASIC_AUTH_USER, httpBasicAuthUser); + httpClientParams.set(HttpClientUtil.PROP_BASIC_AUTH_PASS, httpBasicAuthPassword); + httpClientParams.set(HttpClientUtil.PROP_ALLOW_COMPRESSION, useCompression); // Keeping a very high number so that if you have a large number of cores // no requests are kept waiting for an idle connection. - mgr.setDefaultMaxPerRoute(10000); - mgr.setMaxTotal(10000); - DefaultHttpClient httpClient = new DefaultHttpClient(mgr); - httpClient.getParams().setIntParameter(CoreConnectionPNames.SO_TIMEOUT, readTimeout == null ? 20000 : Integer.parseInt(readTimeout)); //20 secs - httpClient.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, connTimeout == null ? 5000 : Integer.parseInt(connTimeout)); //5 secs + httpClientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 10000); + httpClientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 10000); + HttpClient httpClient = HttpClientUtil.createClient(httpClientParams); if (client == null && connTimeout == null && readTimeout == null) client = httpClient; - - if (httpBasicAuthUser != null && httpBasicAuthPassword != null) { - httpClient.getCredentialsProvider().setCredentials(AuthScope.ANY, - new UsernamePasswordCredentials(httpBasicAuthUser, httpBasicAuthPassword)); - } - return httpClient; } - public SnapPuller(NamedList initArgs, ReplicationHandler handler, SolrCore sc) { + public SnapPuller(final NamedList initArgs, final ReplicationHandler handler, final SolrCore sc) { solrCore = sc; - masterUrl = (String) initArgs.get(MASTER_URL); + final SolrParams params = SolrParams.toSolrParams(initArgs); + String masterUrl = (String) initArgs.get(MASTER_URL); if (masterUrl == null) throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "'masterUrl' is required for a slave"); + if (masterUrl.endsWith("/replication")) { + masterUrl = masterUrl.substring(0, masterUrl.length()-12); + LOG.warn("'masterUrl' must be specified without the /replication suffix"); + } + this.masterUrl = masterUrl; + this.replicationHandler = handler; pollIntervalStr = (String) initArgs.get(POLL_INTERVAL); pollInterval = readInterval(pollIntervalStr); String compress = (String) initArgs.get(COMPRESSION); useInternal = INTERNAL.equals(compress); useExternal = EXTERNAL.equals(compress); - String connTimeout = (String) initArgs.get(HTTP_CONN_TIMEOUT); - String readTimeout = (String) initArgs.get(HTTP_READ_TIMEOUT); - String httpBasicAuthUser = (String) initArgs.get(HTTP_BASIC_AUTH_USER); - String httpBasicAuthPassword = (String) initArgs.get(HTTP_BASIC_AUTH_PASSWORD); - myHttpClient = createHttpClient(connTimeout, readTimeout, httpBasicAuthUser, httpBasicAuthPassword); + String connTimeout = (String) initArgs.get(HttpClientUtil.PROP_CONNECTION_TIMEOUT); + String readTimeout = (String) initArgs.get(HttpClientUtil.PROP_SO_TIMEOUT); + String httpBasicAuthUser = (String) initArgs.get(HttpClientUtil.PROP_BASIC_AUTH_USER); + String httpBasicAuthPassword = (String) initArgs.get(HttpClientUtil.PROP_BASIC_AUTH_PASS); + myHttpClient = createHttpClient(connTimeout, readTimeout, httpBasicAuthUser, httpBasicAuthPassword, useExternal); if (pollInterval != null && pollInterval > 0) { startExecutorService(); } else { @@ -190,95 +186,63 @@ public class SnapPuller { */ @SuppressWarnings("unchecked") NamedList getLatestVersion() throws IOException { - HttpPost post = new HttpPost(masterUrl); - List formparams = new ArrayList(); - formparams.add(new BasicNameValuePair("wt", "javabin")); - formparams.add(new BasicNameValuePair(COMMAND, CMD_INDEX_VERSION)); - UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); - post.setEntity(entity); - return getNamedListResponse(post); - } - - NamedList getCommandResponse(NamedList commands) throws IOException { - - HttpPost post = new HttpPost(masterUrl); - - List formparams = new ArrayList(); - formparams.add(new BasicNameValuePair("wt", "javabin")); - - for (Map.Entry c : commands) { - formparams.add(new BasicNameValuePair(c.getKey(), c.getValue())); - } - UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); - post.setEntity(entity); - return getNamedListResponse(post); - } - - private NamedList getNamedListResponse(HttpPost method) throws IOException { - InputStream input = null; - NamedList result = null; + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(COMMAND, CMD_INDEX_VERSION); + params.set(CommonParams.WT, "javabin"); + params.set(CommonParams.QT, "/replication"); + QueryRequest req = new QueryRequest(params); + SolrServer server = new HttpSolrServer(masterUrl, myHttpClient); //XXX modify to use shardhandler try { - HttpResponse response = myHttpClient.execute(method); - int status = response.getStatusLine().getStatusCode(); - if (status != HttpStatus.SC_OK) { - throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, - "Request failed for the url " + method); - } - input = response.getEntity().getContent(); - result = (NamedList)new JavaBinCodec().unmarshal(input); - } finally { - try { - if (input != null) { - input.close(); - } - } catch (Exception e) { - } + return server.request(req); + } catch (SolrServerException e) { + throw new IOException(e); } - return result; } /** - * Fetches the list of files in a given index commit point + * Fetches the list of files in a given index commit point and updates internal list of files to download. */ - void fetchFileList(long gen) throws IOException { - HttpPost post = new HttpPost(masterUrl); - - List formparams = new ArrayList(); - formparams.add(new BasicNameValuePair("wt", "javabin")); - formparams.add(new BasicNameValuePair(COMMAND, CMD_GET_FILE_LIST)); - formparams.add(new BasicNameValuePair(GENERATION, String.valueOf(gen))); - - UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); - post.setEntity(entity); - - @SuppressWarnings("unchecked") - NamedList>> nl - = (NamedList>>) getNamedListResponse(post); - - List> f = nl.get(CMD_GET_FILE_LIST); - if (f != null) - filesToDownload = Collections.synchronizedList(f); - else { - filesToDownload = Collections.emptyList(); - LOG.error("No files to download for index generation: "+ gen); - } + private void fetchFileList(long gen) throws IOException { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(COMMAND, CMD_GET_FILE_LIST); + params.set(GENERATION, String.valueOf(gen)); + params.set(CommonParams.WT, "javabin"); + params.set(CommonParams.QT, "/replication"); + QueryRequest req = new QueryRequest(params); + SolrServer server = new HttpSolrServer(masterUrl, myHttpClient); //XXX modify to use shardhandler + + try { + NamedList response = server.request(req); + + List> files = (List>) response.get(CMD_GET_FILE_LIST); + if (files != null) + filesToDownload = Collections.synchronizedList(files); + else { + filesToDownload = Collections.emptyList(); + LOG.error("No files to download for index generation: "+ gen); + } - f = nl.get(CONF_FILES); - if (f != null) - confFilesToDownload = Collections.synchronizedList(f); + files = (List>) response.get(CONF_FILES); + if (files != null) + confFilesToDownload = Collections.synchronizedList(files); + + } catch (SolrServerException e) { + throw new IOException(e); + } } + private boolean successfulInstall = false; + /** * This command downloads all the necessary files from master to install a index commit point. Only changed files are * downloaded. It also downloads the conf files (if they are modified). * * @param core the SolrCore + * @param force force a replication in all cases * @return true on success, false if slave is already in sync * @throws IOException if an exception occurs */ @SuppressWarnings("unchecked") - boolean successfulInstall = false; - boolean fetchLatestIndex(SolrCore core, boolean force) throws IOException, InterruptedException { successfulInstall = false; replicationStartTime = System.currentTimeMillis(); @@ -348,6 +312,7 @@ public class SnapPuller { File tmpIndexDir = createTempindexDir(core); if (isIndexStale()) isFullCopyNeeded = true; + LOG.info("Starting download to " + tmpIndexDir + " fullCopy=" + isFullCopyNeeded); successfulInstall = false; boolean deleteTmpIdxDir = true; File indexDir = null ; @@ -394,8 +359,13 @@ public class SnapPuller { } catch (Exception e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Index fetch failed : ", e); } finally { - if (deleteTmpIdxDir) delTree(tmpIndexDir); - else delTree(indexDir); + if (deleteTmpIdxDir) { + LOG.info("removing temporary index download directory " + tmpIndexDir); + delTree(tmpIndexDir); + } else { + LOG.info("removing old index directory " + indexDir); + delTree(indexDir); + } } } finally { if (!successfulInstall) { @@ -613,8 +583,9 @@ public class SnapPuller { * @param latestGeneration the version number */ private void downloadIndexFiles(boolean downloadCompleteIndex, File tmpIdxDir, long latestGeneration) throws Exception { + String indexDir = solrCore.getIndexDir(); for (Map file : filesToDownload) { - File localIndexFile = new File(solrCore.getIndexDir(), (String) file.get(NAME)); + File localIndexFile = new File(indexDir, (String) file.get(NAME)); if (!localIndexFile.exists() || downloadCompleteIndex) { fileFetcher = new FileFetcher(tmpIdxDir, file, (String) file.get(NAME), false, latestGeneration); currentFile = file; @@ -660,6 +631,18 @@ public class SnapPuller { + " to: " + indexFileInIndex + " Trying to do a copy"); FileUtils.copyFile(indexFileInTmpDir,indexFileInIndex); success = true; + } catch (FileNotFoundException e) { + if (!indexDir.exists()) { + File parent = indexDir.getParentFile(); + String[] children = null; + if (parent != null) { + children = parent.list(); + } + LOG.error("The index directory does not exist: " + indexDir.getAbsolutePath() + + " dirs found: " + (children == null ? "none could be found" : Arrays.asList(children))); + } + LOG.error("Unable to copy index file from: " + indexFileInTmpDir + + " to: " + indexFileInIndex , e); } catch (IOException e) { LOG.error("Unable to copy index file from: " + indexFileInTmpDir + " to: " + indexFileInIndex , e); @@ -681,7 +664,7 @@ public class SnapPuller { /** * Copy all index files from the temp index dir to the actual index. The segments_N file is copied last. */ - private boolean copyIndexFiles(File tmpIdxDir, File indexDir) throws IOException { + private boolean copyIndexFiles(File tmpIdxDir, File indexDir) { String segmentsFile = null; List copiedfiles = new ArrayList(); for (Map f : filesDownloaded) { @@ -708,7 +691,7 @@ public class SnapPuller { /** * The conf files are copied to the tmp dir to the conf dir. A backup of the old file is maintained */ - private void copyTmpConfFiles2Conf(File tmpconfDir) throws IOException { + private void copyTmpConfFiles2Conf(File tmpconfDir) { File confDir = new File(solrCore.getResourceLoader().getConfigDir()); for (File file : tmpconfDir.listFiles()) { File oldFile = new File(confDir, file.getName()); @@ -737,7 +720,7 @@ public class SnapPuller { * If the index is stale by any chance, load index from a different dir in the data dir. */ private boolean modifyIndexProps(String tmpIdxDirName) { - LOG.info("New index installed. Updating index properties..."); + LOG.info("New index installed. Updating index properties... index="+tmpIdxDirName); File idxprops = new File(solrCore.getDataDir() + "index.properties"); Properties p = new Properties(); if (idxprops.exists()) { @@ -941,8 +924,6 @@ public class SnapPuller { private boolean isConf; - private HttpPost post; - private boolean aborted = false; private Long indexGen; @@ -1118,79 +1099,62 @@ public class SnapPuller { * Open a new stream using HttpClient */ FastInputStream getStream() throws IOException { - post = new HttpPost(masterUrl); - //the method is command=filecontent - - List formparams = new ArrayList(); - - formparams.add(new BasicNameValuePair(COMMAND, CMD_GET_FILE)); + SolrServer s = new HttpSolrServer(masterUrl, myHttpClient, null); //XXX use shardhandler + ModifiableSolrParams params = new ModifiableSolrParams(); +// //the method is command=filecontent + params.set(COMMAND, CMD_GET_FILE); + params.set(GENERATION, Long.toString(indexGen)); + params.set(CommonParams.QT, "/replication"); //add the version to download. This is used to reserve the download - formparams.add(new BasicNameValuePair(GENERATION, indexGen.toString())); if (isConf) { //set cf instead of file for config file - formparams.add(new BasicNameValuePair(CONF_FILE_SHORT, fileName)); + params.set(CONF_FILE_SHORT, fileName); } else { - formparams.add(new BasicNameValuePair(FILE, fileName)); + params.set(FILE, fileName); } if (useInternal) { - formparams.add(new BasicNameValuePair(COMPRESSION, "true")); - } - if (useExternal) { - formparams.add(new BasicNameValuePair("Accept-Encoding", "gzip,deflate")); + params.set(COMPRESSION, "internal"); } //use checksum - if (this.includeChecksum) - formparams.add(new BasicNameValuePair(CHECKSUM, "true")); + if (this.includeChecksum) { + params.set(CHECKSUM, true); + } //wt=filestream this is a custom protocol - formparams.add(new BasicNameValuePair("wt", FILE_STREAM)); - // This happen if there is a failure there is a retry. the offset= ensures that - // the server starts from the offset + params.set(CommonParams.WT, FILE_STREAM); + // This happen if there is a failure there is a retry. the offset= ensures that + // the server starts from the offset if (bytesDownloaded > 0) { - formparams.add(new BasicNameValuePair(OFFSET, "" + bytesDownloaded)); + params.set(OFFSET, Long.toString(bytesDownloaded)); } - UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); - post.setEntity(entity); - - HttpResponse response = myHttpClient.execute(post); - InputStream is = response.getEntity().getContent(); - //wrap it using FastInputStream - if (useInternal) { - is = new InflaterInputStream(is); - } else if (useExternal) { - is = checkCompressed(post, is); - } - return new FastInputStream(is); - } - } - /* - * This is copied from CommonsHttpSolrServer - */ - private InputStream checkCompressed(AbstractHttpMessage method, InputStream respBody) throws IOException { - Header contentEncodingHeader = method.getFirstHeader("Content-Encoding"); - if (contentEncodingHeader != null) { - String contentEncoding = contentEncodingHeader.getValue(); - if (contentEncoding.contains("gzip")) { - respBody = new GZIPInputStream(respBody); - } else if (contentEncoding.contains("deflate")) { - respBody = new InflaterInputStream(respBody); - } - } else { - Header contentTypeHeader = method.getFirstHeader("Content-Type"); - if (contentTypeHeader != null) { - String contentType = contentTypeHeader.getValue(); - if (contentType != null) { - if (contentType.startsWith("application/x-gzip-compressed")) { - respBody = new GZIPInputStream(respBody); - } else if (contentType.startsWith("application/x-deflate")) { - respBody = new InflaterInputStream(respBody); - } + NamedList response; + InputStream is = null; + try { + QueryRequest req = new QueryRequest(params); + response = s.request(req); + is = (InputStream) response.get("stream"); + if(useInternal) { + is = new InflaterInputStream(is); } + return new FastInputStream(is); + } catch (Throwable t) { + //close stream on error + IOUtils.closeQuietly(is); + throw new IOException("Could not download file '" + fileName + "'", t); } } - return respBody; + } + + NamedList getDetails() throws IOException, SolrServerException { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(COMMAND, CMD_DETAILS); + params.set("slave", false); + params.set(CommonParams.QT, "/replication"); + SolrServer server = new HttpSolrServer(masterUrl, myHttpClient); //XXX use shardhandler + QueryRequest request = new QueryRequest(params); + return server.request(request); } static Integer readInterval(String interval) { @@ -1251,14 +1215,6 @@ public class SnapPuller { private static final Pattern INTERVAL_PATTERN = Pattern.compile("(\\d*?):(\\d*?):(\\d*)"); - private static final String HTTP_CONN_TIMEOUT = "httpConnTimeout"; - - private static final String HTTP_READ_TIMEOUT = "httpReadTimeout"; - - private static final String HTTP_BASIC_AUTH_USER = "httpBasicAuthUser"; - - private static final String HTTP_BASIC_AUTH_PASSWORD = "httpBasicAuthPassword"; - static final String INDEX_REPLICATED_AT = "indexReplicatedAt"; static final String TIMES_INDEX_REPLICATED = "timesIndexReplicated"; diff --git a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java b/solr/core/src/java/org/apache/solr/handler/SnapShooter.java index ec470cd..45fdfd7 100644 --- a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java +++ b/solr/core/src/java/org/apache/solr/handler/SnapShooter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -53,7 +53,7 @@ public class SnapShooter { private SolrCore solrCore; private SimpleFSLockFactory lockFactory; - public SnapShooter(SolrCore core, String location) throws IOException { + public SnapShooter(SolrCore core, String location) { solrCore = core; if (location == null) snapDir = core.getDataDir(); else { @@ -165,7 +165,7 @@ public class SnapShooter { } public static final String SNAP_DIR = "snapDir"; - public static final String DATE_FMT = "yyyyMMddHHmmss"; + public static final String DATE_FMT = "yyyyMMddHHmmssSSS"; private class FileCopier { diff --git a/solr/core/src/java/org/apache/solr/handler/StandardRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/StandardRequestHandler.java index f94fa09..5352538 100644 --- a/solr/core/src/java/org/apache/solr/handler/StandardRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/StandardRequestHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java index 5c37316..473c154 100644 --- a/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java index 65f5d0b..da5c079 100644 --- a/solr/core/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/XsltUpdateRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/XsltUpdateRequestHandler.java index 20ada96..c12bc66 100644 --- a/solr/core/src/java/org/apache/solr/handler/XsltUpdateRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/XsltUpdateRequestHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java b/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java index 6ad01bc..c96cfb7 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java new file mode 100644 index 0000000..4dbfcac --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -0,0 +1,163 @@ +package org.apache.solr.handler.admin; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.solr.cloud.Overseer; +import org.apache.solr.cloud.OverseerCollectionProcessor; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CollectionParams.CollectionAction; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.handler.RequestHandlerBase; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class CollectionsHandler extends RequestHandlerBase { + protected static Logger log = LoggerFactory.getLogger(CollectionsHandler.class); + protected final CoreContainer coreContainer; + + public CollectionsHandler() { + super(); + // Unlike most request handlers, CoreContainer initialization + // should happen in the constructor... + this.coreContainer = null; + } + + + /** + * Overloaded ctor to inject CoreContainer into the handler. + * + * @param coreContainer Core Container of the solr webapp installed. + */ + public CollectionsHandler(final CoreContainer coreContainer) { + this.coreContainer = coreContainer; + } + + + @Override + final public void init(NamedList args) { + + } + + /** + * The instance of CoreContainer this handler handles. This should be the CoreContainer instance that created this + * handler. + * + * @return a CoreContainer instance + */ + public CoreContainer getCoreContainer() { + return this.coreContainer; + } + + @Override + public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { + // Make sure the cores is enabled + CoreContainer cores = getCoreContainer(); + if (cores == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Core container instance missing"); + } + + // Pick the action + SolrParams params = req.getParams(); + CollectionAction action = null; + String a = params.get(CoreAdminParams.ACTION); + if (a != null) { + action = CollectionAction.get(a); + } + if (action != null) { + switch (action) { + case CREATE: { + this.handleCreateAction(req, rsp); + break; + } + case DELETE: { + this.handleDeleteAction(req, rsp); + break; + } + + default: { + throw new RuntimeException("Unknown action: " + action); + } + } + } + + rsp.setHttpCaching(false); + } + + private void handleDeleteAction(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException { + String name = req.getParams().required().get("name"); + + ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, + OverseerCollectionProcessor.DELETECOLLECTION, "name", name); + + // TODO: what if you want to block until the collection is available? + coreContainer.getZkController().getOverseerCollectionQueue().offer(ZkStateReader.toJSON(m)); + } + + + // very simple currently, you can pass a template collection, and the new collection is created on + // every node the template collection is on + // there is a lot more to add - you should also be able to create with an explicit server list + // we might also want to think about error handling (add the request to a zk queue and involve overseer?) + // as well as specific replicas= options + private void handleCreateAction(SolrQueryRequest req, + SolrQueryResponse rsp) throws InterruptedException, KeeperException { + + Integer numReplicas = req.getParams().getInt("numReplicas", 0); + String name = req.getParams().required().get("name"); + String configName = req.getParams().get("collection.configName"); + String numShards = req.getParams().get("numShards"); + + ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, + OverseerCollectionProcessor.CREATECOLLECTION, "numReplicas", numReplicas.toString(), "name", name, + "collection.configName", configName, "numShards", numShards); + + // TODO: what if you want to block until the collection is available? + coreContainer.getZkController().getOverseerCollectionQueue().offer(ZkStateReader.toJSON(m)); + } + + public static ModifiableSolrParams params(String... params) { + ModifiableSolrParams msp = new ModifiableSolrParams(); + for (int i=0; i slices = cloudState.getCollectionStates().get(collection); - for (Map.Entry entry : slices.entrySet()) { - Slice slice = entry.getValue(); - Map shards = slice.getShards(); - Set> shardEntries = shards.entrySet(); - for (Map.Entry shardEntry : shardEntries) { - final ZkNodeProps node = shardEntry.getValue(); - if (cloudState.liveNodesContain(node.get(ZkStateReader.NODE_NAME_PROP))) { - newParams.set(CoreAdminParams.CORE, node.get(ZkStateReader.CORE_NAME_PROP)); - String replica = node.get(ZkStateReader.BASE_URL_PROP); - ShardRequest sreq = new ShardRequest(); - newParams.set("qt", "/admin/cores"); - sreq.purpose = 1; - // TODO: this sucks - if (replica.startsWith("http://")) - replica = replica.substring(7); - sreq.shards = new String[]{replica}; - sreq.actualShards = sreq.shards; - sreq.params = newParams; - - shardHandler.submit(sreq, replica, sreq.params); - } - } - } - - ShardResponse srsp; - do { - srsp = shardHandler.takeCompletedOrError(); - if (srsp != null) { - Throwable e = srsp.getException(); - if (e != null) { - log.error("Error talking to shard: " + srsp.getShard(), e); - } - } - } while(srsp != null); - - } protected NamedList getCoreStatus(CoreContainer cores, String cname) throws IOException { NamedList info = new SimpleOrderedMap(); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/LoggingHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/LoggingHandler.java index ee83329..f95f03c 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/LoggingHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/LoggingHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java index 5839edd..16eb4cb 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -544,7 +544,7 @@ public class LukeRequestHandler extends RequestHandlerBase indexInfo.add("maxDoc", reader.maxDoc()); indexInfo.add("version", reader.getVersion()); // TODO? Is this different then: IndexReader.getCurrentVersion( dir )? - indexInfo.add("segmentCount", reader.getSequentialSubReaders().length); + indexInfo.add("segmentCount", reader.getTopReaderContext().leaves().size()); indexInfo.add("current", reader.isCurrent() ); indexInfo.add("hasDeletions", reader.hasDeletions() ); indexInfo.add("directory", dir ); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/PluginInfoHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/PluginInfoHandler.java index 8ef0717..c8a64f9 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/PluginInfoHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/PluginInfoHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/admin/PropertiesRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/PropertiesRequestHandler.java index 0b90530..b8a7890 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/PropertiesRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/PropertiesRequestHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java index db6b19c..b919fe4 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -195,8 +195,7 @@ public class ShowFileRequestHandler extends RequestHandlerBase rsp.setHttpCaching(false); } - private void showFromFileSystem(SolrQueryRequest req, SolrQueryResponse rsp) - throws IOException { + private void showFromFileSystem(SolrQueryRequest req, SolrQueryResponse rsp) { File adminFile = null; final SolrResourceLoader loader = req.getCore().getResourceLoader(); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/SolrInfoMBeanHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/SolrInfoMBeanHandler.java index 4b47488..56068b2 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/SolrInfoMBeanHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/SolrInfoMBeanHandler.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.admin; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java index 47510b1..7b50054 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -81,8 +81,7 @@ public class SystemInfoHandler extends RequestHandlerBase /** * Get system info */ - private SimpleOrderedMap getCoreInfo( SolrCore core ) throws Exception - { + private SimpleOrderedMap getCoreInfo( SolrCore core ) { SimpleOrderedMap info = new SimpleOrderedMap(); IndexSchema schema = core.getSchema(); @@ -110,8 +109,7 @@ public class SystemInfoHandler extends RequestHandlerBase /** * Get system info */ - public static SimpleOrderedMap getSystemInfo() throws Exception - { + public static SimpleOrderedMap getSystemInfo() { SimpleOrderedMap info = new SimpleOrderedMap(); OperatingSystemMXBean os = ManagementFactory.getOperatingSystemMXBean(); @@ -255,8 +253,7 @@ public class SystemInfoHandler extends RequestHandlerBase return jvm; } - private static SimpleOrderedMap getLuceneInfo() throws Exception - { + private static SimpleOrderedMap getLuceneInfo() { SimpleOrderedMap info = new SimpleOrderedMap(); Package p = SolrCore.class.getPackage(); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ThreadDumpHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/ThreadDumpHandler.java index 8ca29d6..b9df00c 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/ThreadDumpHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/ThreadDumpHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -79,8 +79,7 @@ public class ThreadDumpHandler extends RequestHandlerBase //-------------------------------------------------------------------------------- //-------------------------------------------------------------------------------- - private static SimpleOrderedMap getThreadInfo( ThreadInfo ti, ThreadMXBean tmbean ) throws IOException - { + private static SimpleOrderedMap getThreadInfo( ThreadInfo ti, ThreadMXBean tmbean ) { SimpleOrderedMap info = new SimpleOrderedMap(); long tid = ti.getThreadId(); diff --git a/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java b/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java index 03d9bd0..8f96236 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java index 5d27e3b..090cd05 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,9 +20,11 @@ package org.apache.solr.handler.component; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.util.OpenBitSet; import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.FacetParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.ShardParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; @@ -291,7 +293,16 @@ public class FacetComponent extends SearchComponent for (ShardResponse srsp: sreq.responses) { int shardNum = rb.getShardNum(srsp.getShard()); - NamedList facet_counts = (NamedList)srsp.getSolrResponse().getResponse().get("facet_counts"); + NamedList facet_counts = null; + try { + facet_counts = (NamedList)srsp.getSolrResponse().getResponse().get("facet_counts"); + } + catch(Exception ex) { + if(rb.req.getParams().getBool(ShardParams.SHARDS_TOLERANT, false)) { + continue; // looks like a shard did not return anything + } + throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to read facet info for shard: "+srsp.getShard(), ex); + } // handle facet queries NamedList facet_queries = (NamedList)facet_counts.get("facet_queries"); @@ -439,7 +450,7 @@ public class FacetComponent extends SearchComponent long maxCount = sfc.count; for (int shardNum=0; shardNum0) { + if(obs!=null && !obs.get(sfc.termNum) && dff.maxPossible(sfc,shardNum)>0) { dff.needRefinements = true; List lst = dff._toRefine[shardNum]; if (lst == null) { diff --git a/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java b/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java index 9871809..8450cac 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java +++ b/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.component; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java b/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java index 6a94897..69a3ce9 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java index 9daebb4..d88a683 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.component; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -161,7 +161,7 @@ public class HttpShardHandler extends ShardHandler { if (urls.size() <= 1) { String url = urls.get(0); srsp.setShardAddress(url); - SolrServer server = new HttpSolrServer(url, httpClient == null ? httpShardHandlerFactory.client : httpClient); + SolrServer server = new HttpSolrServer(url, httpClient); ssr.nl = server.request(req); } else { LBHttpSolrServer.Rsp rsp = httpShardHandlerFactory.loadbalancer.request(new LBHttpSolrServer.Req(req, urls)); diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java index d77bf80..79409fd 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java +++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.component; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,12 +21,10 @@ import java.util.Random; import java.util.concurrent.*; import org.apache.http.client.HttpClient; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; -import org.apache.http.params.CoreConnectionPNames; +import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.LBHttpSolrServer; import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.PluginInfo; import org.apache.solr.util.DefaultSolrThreadFactory; @@ -51,37 +49,25 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements Plug new DefaultSolrThreadFactory("httpShardExecutor") ); - HttpClient client; - Random r = new Random(); + private HttpClient defaultClient; LBHttpSolrServer loadbalancer; - int soTimeout = 0; //current default values - int connectionTimeout = 0; //current default values + //default values: + int soTimeout = 0; + int connectionTimeout = 0; int maxConnectionsPerHost = 20; int corePoolSize = 0; - int maximumPoolSize = 10; + int maximumPoolSize = Integer.MAX_VALUE; int keepAliveTime = 5; - int queueSize = 1; - boolean accessPolicy = true; + int queueSize = -1; + boolean accessPolicy = false; public String scheme = "http://"; //current default values - private ThreadSafeClientConnManager mgr; - // socket timeout measured in ms, closes a socket if read - // takes longer than x ms to complete. throws - // java.net.SocketTimeoutException: Read timed out exception - static final String INIT_SO_TIMEOUT = "socketTimeout"; - - // connection timeout measures in ms, closes a socket if connection - // cannot be established within x ms. with a - // java.net.SocketTimeoutException: Connection timed out - static final String INIT_CONNECTION_TIMEOUT = "connTimeout"; + final Random r = new Random(); // URL scheme to be used in distributed search. static final String INIT_URL_SCHEME = "urlScheme"; - // Maximum connections allowed per host - static final String INIT_MAX_CONNECTION_PER_HOST = "maxConnectionsPerHost"; - // The core size of the threadpool servicing requests static final String INIT_CORE_POOL_SIZE = "corePoolSize"; @@ -97,27 +83,32 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements Plug // Configure if the threadpool favours fairness over throughput static final String INIT_FAIRNESS_POLICY = "fairnessPolicy"; + /** + * Get {@link ShardHandler} that uses the default http client. + */ public ShardHandler getShardHandler() { - return getShardHandler(null); + return getShardHandler(defaultClient); } - public ShardHandler getShardHandler(DefaultHttpClient httpClient){ + /** + * Get {@link ShardHandler} that uses custom http client. + */ + public ShardHandler getShardHandler(final HttpClient httpClient){ return new HttpShardHandler(this, httpClient); } public void init(PluginInfo info) { NamedList args = info.initArgs; - this.soTimeout = getParameter(args, INIT_SO_TIMEOUT, 0); - + this.soTimeout = getParameter(args, HttpClientUtil.PROP_SO_TIMEOUT, soTimeout); this.scheme = getParameter(args, INIT_URL_SCHEME, "http://"); this.scheme = (this.scheme.endsWith("://")) ? this.scheme : this.scheme + "://"; - this.connectionTimeout = getParameter(args, INIT_CONNECTION_TIMEOUT, 0); - this.maxConnectionsPerHost = getParameter(args, INIT_MAX_CONNECTION_PER_HOST, 20); - this.corePoolSize = getParameter(args, INIT_CORE_POOL_SIZE, 0); - this.maximumPoolSize = getParameter(args, INIT_MAX_POOL_SIZE, Integer.MAX_VALUE); - this.keepAliveTime = getParameter(args, MAX_THREAD_IDLE_TIME, 5); - this.queueSize = getParameter(args, INIT_SIZE_OF_QUEUE, -1); - this.accessPolicy = getParameter(args, INIT_FAIRNESS_POLICY, false); + this.connectionTimeout = getParameter(args, HttpClientUtil.PROP_CONNECTION_TIMEOUT, connectionTimeout); + this.maxConnectionsPerHost = getParameter(args, HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, maxConnectionsPerHost); + this.corePoolSize = getParameter(args, INIT_CORE_POOL_SIZE, corePoolSize); + this.maximumPoolSize = getParameter(args, INIT_MAX_POOL_SIZE, maximumPoolSize); + this.keepAliveTime = getParameter(args, MAX_THREAD_IDLE_TIME, keepAliveTime); + this.queueSize = getParameter(args, INIT_SIZE_OF_QUEUE, queueSize); + this.accessPolicy = getParameter(args, INIT_FAIRNESS_POLICY, accessPolicy); BlockingQueue blockingQueue = (this.queueSize == -1) ? new SynchronousQueue(this.accessPolicy) : @@ -131,23 +122,16 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements Plug new DefaultSolrThreadFactory("httpShardExecutor") ); - mgr = new ThreadSafeClientConnManager(); - mgr.setDefaultMaxPerRoute(256); - mgr.setMaxTotal(10000); - DefaultHttpClient client = new DefaultHttpClient(mgr); - - client.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, connectionTimeout); - client.getParams().setIntParameter(CoreConnectionPNames.SO_TIMEOUT, soTimeout); - // mgr.getParams().setStaleCheckingEnabled(false); - - - // prevent retries (note: this didn't work when set on mgr.. needed to be set on client) - DefaultHttpRequestRetryHandler retryhandler = new DefaultHttpRequestRetryHandler(0, false); - client.setHttpRequestRetryHandler(retryhandler); - this.client = client; + ModifiableSolrParams clientParams = new ModifiableSolrParams(); + clientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, maxConnectionsPerHost); + clientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 10000); + clientParams.set(HttpClientUtil.PROP_SO_TIMEOUT, soTimeout); + clientParams.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, connectionTimeout); + clientParams.set(HttpClientUtil.PROP_USE_RETRY, false); + this.defaultClient = HttpClientUtil.createClient(clientParams); try { - loadbalancer = new LBHttpSolrServer(client); + loadbalancer = new LBHttpSolrServer(defaultClient); } catch (MalformedURLException e) { // should be impossible since we're not passing any URLs here throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); @@ -169,7 +153,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements Plug @Override public void close() { try { - mgr.shutdown(); + defaultClient.getConnectionManager().shutdown(); } catch (Throwable e) { SolrException.log(log, e); } diff --git a/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java b/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java index 395a6ae..42011b1 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java b/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java index 9143401..114e614 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java +++ b/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java index 518d73c..c8cd3cc 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,6 +21,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReaderContext; +import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.Term; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.search.*; @@ -29,7 +30,6 @@ import org.apache.lucene.search.grouping.SearchGroup; import org.apache.lucene.search.grouping.TopGroups; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CharsRef; -import org.apache.lucene.util.ReaderUtil; import org.apache.lucene.util.UnicodeUtil; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.common.SolrDocument; @@ -435,13 +435,13 @@ public class QueryComponent extends SearchComponent Sort sort = searcher.weightSort(rb.getSortSpec().getSort()); SortField[] sortFields = sort==null ? new SortField[]{SortField.FIELD_SCORE} : sort.getSort(); NamedList sortVals = new NamedList(); // order is important for the sort fields - Field field = new StringField("dummy", ""); // a dummy Field + Field field = new StringField("dummy", "", Field.Store.NO); // a dummy Field IndexReaderContext topReaderContext = searcher.getTopReaderContext(); - AtomicReaderContext[] leaves = topReaderContext.leaves(); + List leaves = topReaderContext.leaves(); AtomicReaderContext currentLeaf = null; - if (leaves.length==1) { + if (leaves.size()==1) { // if there is a single segment, use that subReader and avoid looking up each time - currentLeaf = leaves[0]; + currentLeaf = leaves.get(0); leaves=null; } @@ -478,7 +478,7 @@ public class QueryComponent extends SearchComponent if (leaves != null) { idx = ReaderUtil.subIndex(doc, leaves); - currentLeaf = leaves[idx]; + currentLeaf = leaves.get(idx); if (idx != lastIdx) { // we switched segments. invalidate comparator. comparator = null; @@ -652,7 +652,7 @@ public class QueryComponent extends SearchComponent GroupingSpecification groupSpec = rb.getGroupingSpec(); if (rb.mergedTopGroups.isEmpty()) { for (String field : groupSpec.getFields()) { - rb.mergedTopGroups.put(field, new TopGroups(null, null, 0, 0, new GroupDocs[]{})); + rb.mergedTopGroups.put(field, new TopGroups(null, null, 0, 0, new GroupDocs[]{}, Float.NaN)); } rb.resultIds = new HashMap(); } diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java index 1da847d..5d233f6 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -480,7 +480,7 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore private SentinelIntSet ordSet; //the key half of the map private BytesRef[] termValues;//the value half of the map - public ElevationComparatorSource(final QueryElevationComponent.ElevationObj elevations) throws IOException { + public ElevationComparatorSource(final QueryElevationComponent.ElevationObj elevations) { this.elevations = elevations; int size = elevations.ids.size(); ordSet = new SentinelIntSet(size, -1); @@ -506,7 +506,7 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore bottomVal = values[slot]; } - private int docVal(int doc) throws IOException { + private int docVal(int doc) { if (ordSet.size() > 0) { int slot = ordSet.find(doc); if (slot >= 0) { @@ -519,12 +519,12 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore } @Override - public int compareBottom(int doc) throws IOException { + public int compareBottom(int doc) { return bottomVal - docVal(doc); } @Override - public void copy(int slot, int doc) throws IOException { + public void copy(int slot, int doc) { values[slot] = docVal(doc); } @@ -562,7 +562,7 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore } @Override - public int compareDocToValue(int doc, Integer valueObj) throws IOException { + public int compareDocToValue(int doc, Integer valueObj) { final int value = valueObj.intValue(); final int docValue = docVal(doc); return docValue - value; // values will be small enough that there is no overflow concern diff --git a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java index 1f15c84..59c576a 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java b/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java index c7d35aa..378af85 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java index bf1f8e9..c9ba188 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,7 +17,6 @@ package org.apache.solr.handler.component; -import org.apache.lucene.queryparser.classic.ParseException; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; @@ -35,7 +34,10 @@ import org.apache.solr.util.plugin.SolrCoreAware; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + /** * @@ -156,7 +158,7 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware , @Override - public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception, ParseException, InstantiationException, IllegalAccessException + public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { // int sleep = req.getParams().getInt("sleep",0); // if (sleep > 0) {log.error("SLEEPING for " + sleep); Thread.sleep(sleep);} diff --git a/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java b/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java index c83200c..144a9c4 100755 --- a/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/component/ShardHandler.java b/solr/core/src/java/org/apache/solr/handler/component/ShardHandler.java index e9c912d..4a568b1 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ShardHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ShardHandler.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.component; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/component/ShardHandlerFactory.java b/solr/core/src/java/org/apache/solr/handler/component/ShardHandlerFactory.java index bba0171..07abdb6 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ShardHandlerFactory.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ShardHandlerFactory.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.component; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java b/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java index 0c3ebb1..5ca61d6 100755 --- a/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/component/ShardResponse.java b/solr/core/src/java/org/apache/solr/handler/component/ShardResponse.java index d9c5feb..07ef8ee 100755 --- a/solr/core/src/java/org/apache/solr/handler/component/ShardResponse.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ShardResponse.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.component; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java b/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java index aba69c8..c606679 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -124,6 +124,7 @@ public class SpellCheckComponent extends SearchComponent implements SolrCoreAwar String q = params.get(SPELLCHECK_Q); SolrSpellChecker spellChecker = getSpellChecker(params); Collection tokens = null; + if (q != null) { //we have a spell check param, tokenize it with the query analyzer applicable for this spellchecker tokens = getTokens(q, spellChecker.getQueryAnalyzer()); @@ -143,7 +144,10 @@ public class SpellCheckComponent extends SearchComponent implements SolrCoreAwar float accuracy = params.getFloat(SPELLCHECK_ACCURACY, Float.MIN_VALUE); Integer alternativeTermCount = params.getInt(SpellingParams.SPELLCHECK_ALTERNATIVE_TERM_COUNT); Integer maxResultsForSuggest = params.getInt(SpellingParams.SPELLCHECK_MAX_RESULTS_FOR_SUGGEST); - SolrParams customParams = getCustomParams(getDictionaryName(params), params); + ModifiableSolrParams customParams = new ModifiableSolrParams(); + for (String checkerName : getDictionaryNames(params)) { + customParams.add(getCustomParams(checkerName, params)); + } Integer hitsInteger = (Integer) rb.rsp.getToLog().get("hits"); long hits = 0; @@ -173,7 +177,7 @@ public class SpellCheckComponent extends SearchComponent implements SolrCoreAwar NamedList suggestions = toNamedList(shardRequest, spellingResult, q, extendedResults, collate, isCorrectlySpelled); if (collate) { - addCollationsToResponse(params, spellingResult, rb, q, suggestions); + addCollationsToResponse(params, spellingResult, rb, q, suggestions, spellChecker.isSuggestionsMayOverlap()); } NamedList response = new SimpleOrderedMap(); response.add("suggestions", suggestions); @@ -181,14 +185,14 @@ public class SpellCheckComponent extends SearchComponent implements SolrCoreAwar } else { throw new SolrException(SolrException.ErrorCode.NOT_FOUND, - "Specified dictionary does not exist: " + getDictionaryName(params)); + "Specified dictionaries do not exist: " + getDictionaryNameAsSingleString(getDictionaryNames(params))); } } } @SuppressWarnings("unchecked") protected void addCollationsToResponse(SolrParams params, SpellingResult spellingResult, ResponseBuilder rb, String q, - NamedList response) { + NamedList response, boolean suggestionsMayOverlap) { int maxCollations = params.getInt(SPELLCHECK_MAX_COLLATIONS, 1); int maxCollationTries = params.getInt(SPELLCHECK_MAX_COLLATION_TRIES, 0); int maxCollationEvaluations = params.getInt(SPELLCHECK_MAX_COLLATION_EVALUATIONS, 10000); @@ -196,8 +200,8 @@ public class SpellCheckComponent extends SearchComponent implements SolrCoreAwar boolean shard = params.getBool(ShardParams.IS_SHARD, false); SpellCheckCollator collator = new SpellCheckCollator(); - List collations = collator.collate(spellingResult, q, rb, maxCollations, maxCollationTries, maxCollationEvaluations); - //by sorting here we guarantee a non-distributed request returns all + List collations = collator.collate(spellingResult, q, rb, maxCollations, maxCollationTries, maxCollationEvaluations, suggestionsMayOverlap); + //by sorting here we guarantee a non-distributed request returns all //results in the same order as a distributed request would, //even in cases when the internal rank is the same. Collections.sort(collations); @@ -459,13 +463,38 @@ public class SpellCheckComponent extends SearchComponent implements SolrCoreAwar } protected SolrSpellChecker getSpellChecker(SolrParams params) { - return spellCheckers.get(getDictionaryName(params)); + String[] dictName = getDictionaryNames(params); + if (dictName.length == 1) { + return spellCheckers.get(dictName[0]); + } else { + String singleStr = getDictionaryNameAsSingleString(dictName); + SolrSpellChecker ssc = spellCheckers.get(singleStr); + if (ssc == null) { + ConjunctionSolrSpellChecker cssc = new ConjunctionSolrSpellChecker(); + for (String dn : dictName) { + cssc.addChecker(spellCheckers.get(dn)); + } + ssc = cssc; + } + return ssc; + } + } + + private String getDictionaryNameAsSingleString(String[] dictName) { + StringBuilder sb = new StringBuilder(); + for (String dn : dictName) { + if (sb.length() > 0) { + sb.append(" "); + } + sb.append(dn); + } + return sb.toString(); } - private String getDictionaryName(SolrParams params) { - String dictName = params.get(SPELLCHECK_DICT); + private String[] getDictionaryNames(SolrParams params) { + String[] dictName = params.getParams(SPELLCHECK_DICT); if (dictName == null) { - dictName = SolrSpellChecker.DEFAULT_DICTIONARY_NAME; + return new String[] {SolrSpellChecker.DEFAULT_DICTIONARY_NAME}; } return dictName; } @@ -650,7 +679,7 @@ public class SpellCheckComponent extends SearchComponent implements SolrCoreAwar if (buildOnCommit) { buildSpellIndex(newSearcher); } else if (buildOnOptimize) { - if (newSearcher.getIndexReader().getSequentialSubReaders().length == 1) { + if (newSearcher.getIndexReader().getSequentialSubReaders().size() == 1) { buildSpellIndex(newSearcher); } else { LOG.info("Index is not optimized therefore skipping building spell check index for: " + checker.getDictionaryName()); diff --git a/solr/core/src/java/org/apache/solr/handler/component/SpellCheckMergeData.java b/solr/core/src/java/org/apache/solr/handler/component/SpellCheckMergeData.java index e2de762..b41ab05 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SpellCheckMergeData.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SpellCheckMergeData.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.component; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java b/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java index e6ee7c9..c4f6882 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java b/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java index d07ed98..492ef01 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java index 7fb303d..1796da7 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -429,11 +429,11 @@ class StringStatsValues extends AbstractStatsValues { } /** - * Determines which of the given Strings is the maximum, as computed by {@link String#compareTo(Object)} + * Determines which of the given Strings is the maximum, as computed by {@link String#compareTo(String)} * * @param str1 String to compare against b * @param str2 String compared against a - * @return str1 if it is considered greater by {@link String#compareTo(Object)}, str2 otherwise + * @return str1 if it is considered greater by {@link String#compareTo(String)}, str2 otherwise */ private static String max(String str1, String str2) { if (str1 == null) { @@ -445,11 +445,11 @@ class StringStatsValues extends AbstractStatsValues { } /** - * Determines which of the given Strings is the minimum, as computed by {@link String#compareTo(Object)} + * Determines which of the given Strings is the minimum, as computed by {@link String#compareTo(String)} * * @param str1 String to compare against b * @param str2 String compared against a - * @return str1 if it is considered less by {@link String#compareTo(Object)}, str2 otherwise + * @return str1 if it is considered less by {@link String#compareTo(String)}, str2 otherwise */ private static String min(String str1, String str2) { if (str1 == null) { diff --git a/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java b/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java index 0099107..5b94c49 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java @@ -1,9 +1,13 @@ package org.apache.solr.handler.component; import java.io.IOException; +import java.util.Arrays; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.Set; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -27,13 +31,14 @@ import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.SolrCore; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; +import org.apache.solr.search.ReturnFields; import org.apache.solr.search.DocList; import org.apache.solr.search.DocListAndSet; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.util.SolrPluginUtils; import org.apache.solr.util.plugin.SolrCoreAware; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -81,6 +86,51 @@ public class TermVectorComponent extends SearchComponent implements SolrCoreAwar protected NamedList initParams; public static final String TERM_VECTORS = "termVectors"; + /** + * Helper method for determining the list of fields that we should + * try to find term vectors on. + *

      + * Does simple (non-glob-supporting) parsing on the + * {@link TermVectorParams#FIELDS} param if specified, otherwise it returns + * the concrete field values specified in {@link CommonParams#FL} -- + * ignoring functions, transformers, or literals. + *

      + *

      + * If "fl=*" is used, or neither param is specified, then null + * will be returned. If the empty set is returned, it means the "fl" + * specified consisted entirely of things that are not real fields + * (ie: functions, transformers, partial-globs, score, etc...) and not + * supported by this component. + *

      + */ + private Set getFields(ResponseBuilder rb) { + SolrParams params = rb.req.getParams(); + String[] fldLst = params.getParams(TermVectorParams.FIELDS); + if (null == fldLst || 0 == fldLst.length || + (1 == fldLst.length && 0 == fldLst[0].length())) { + + // no tv.fl, parse the main fl + ReturnFields rf = new ReturnFields + (params.getParams(CommonParams.FL), rb.req); + + if (rf.wantsAllFields()) { + return null; + } + + Set fieldNames = rf.getLuceneFieldNames(); + return (null != fieldNames) ? + fieldNames : + // return empty set indicating no fields should be used + Collections.emptySet(); + } + + // otherwise us the raw fldList as is, no special parsing or globs + Set fieldNames = new LinkedHashSet(); + for (String fl : fldLst) { + fieldNames.addAll(Arrays.asList(SolrPluginUtils.split(fl))); + } + return fieldNames; + } @Override public void process(ResponseBuilder rb) throws IOException { @@ -108,11 +158,6 @@ public class TermVectorComponent extends SearchComponent implements SolrCoreAwar allFields.tfIdf = true; } - String fldLst = params.get(TermVectorParams.FIELDS); - if (fldLst == null) { - fldLst = params.get(CommonParams.FL); - } - //use this to validate our fields IndexSchema schema = rb.req.getSchema(); //Build up our per field mapping @@ -122,10 +167,14 @@ public class TermVectorComponent extends SearchComponent implements SolrCoreAwar List noPos = new ArrayList(); List noOff = new ArrayList(); - //we have specific fields to retrieve - if (fldLst != null) { - String [] fields = SolrPluginUtils.split(fldLst); + Set fields = getFields(rb); + if ( null != fields ) { + //we have specific fields to retrieve, or no fields for (String field : fields) { + + // workarround SOLR-3523 + if (null == field || "score".equals(field)) continue; + SchemaField sf = schema.getFieldOrNull(field); if (sf != null) { if (sf.storeTermVector()) { @@ -203,22 +252,22 @@ public class TermVectorComponent extends SearchComponent implements SolrCoreAwar // once we find it... final StoredFieldVisitor getUniqValue = new StoredFieldVisitor() { @Override - public void stringField(FieldInfo fieldInfo, String value) throws IOException { + public void stringField(FieldInfo fieldInfo, String value) { uniqValues.add(value); } @Override - public void intField(FieldInfo fieldInfo, int value) throws IOException { + public void intField(FieldInfo fieldInfo, int value) { uniqValues.add(Integer.toString(value)); } @Override - public void longField(FieldInfo fieldInfo, long value) throws IOException { + public void longField(FieldInfo fieldInfo, long value) { uniqValues.add(Long.toString(value)); } @Override - public Status needsField(FieldInfo fieldInfo) throws IOException { + public Status needsField(FieldInfo fieldInfo) { return (fieldInfo.name.equals(finalUniqFieldName)) ? Status.YES : Status.NO; } }; @@ -240,7 +289,7 @@ public class TermVectorComponent extends SearchComponent implements SolrCoreAwar termVectors.add("uniqueKeyFieldName", uniqFieldName); } } - if (!fieldOptions.isEmpty()) { + if ( null != fields ) { for (Map.Entry entry : fieldOptions.entrySet()) { final String field = entry.getKey(); final Terms vector = reader.getTermVector(docId, field); diff --git a/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java b/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java index 2c1d664..4edffdd 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.component; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java index fb72231..d9ae10f 100644 --- a/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java +++ b/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java b/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java index eecf319..4cc2d66 100644 --- a/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java +++ b/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/loader/ContentStreamLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/ContentStreamLoader.java index e5a838a..a0feb2c 100644 --- a/solr/core/src/java/org/apache/solr/handler/loader/ContentStreamLoader.java +++ b/solr/core/src/java/org/apache/solr/handler/loader/ContentStreamLoader.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.loader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java index d4bf51b..ec28eb0 100644 --- a/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java +++ b/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -96,9 +96,6 @@ public class JavabinLoader extends ContentStreamLoader { update = new JavaBinUpdateRequestCodec().unmarshal(in, handler); } catch (EOFException e) { break; // this is expected - } catch (Exception e) { - log.error("Exception while processing update request", e); - break; } if (update.getDeleteById() != null || update.getDeleteQuery() != null) { delete(req, update, processor); diff --git a/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java index 7efca7f..95fead0 100644 --- a/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java +++ b/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.loader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,7 +23,6 @@ import java.util.*; import org.apache.commons.io.IOUtils; import org.apache.noggit.JSONParser; -import org.apache.noggit.JSONUtil; import org.apache.noggit.ObjectBuilder; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; @@ -138,7 +137,7 @@ public class JsonLoader extends ContentStreamLoader { processor.processCommit( cmd ); } else if( v.equals( UpdateRequestHandler.DELETE ) ) { - processor.processDelete( parseDelete() ); + handleDeleteCommand(); } else if( v.equals( UpdateRequestHandler.ROLLBACK ) ) { processor.processRollback( parseRollback() ); @@ -171,49 +170,116 @@ public class JsonLoader extends ContentStreamLoader { ev = parser.nextEvent(); } } - - DeleteUpdateCommand parseDelete() throws IOException { - assertNextEvent( JSONParser.OBJECT_START ); - + + // + // "delete":"id" + // "delete":["id1","id2"] + // "delete":{"id":"foo"} + // "delete":{"query":"myquery"} + // + void handleDeleteCommand() throws IOException { + int ev = parser.nextEvent(); + switch (ev) { + case JSONParser.ARRAY_START: + handleDeleteArray(ev); + break; + case JSONParser.OBJECT_START: + handleDeleteMap(ev); + break; + default: + handleSingleDelete(ev); + } + } + + // returns the string value for a primitive value, or null for the null value + String getString(int ev) throws IOException { + switch (ev) { + case JSONParser.STRING: + return parser.getString(); + case JSONParser.BIGNUMBER: + case JSONParser.NUMBER: + case JSONParser.LONG: + return parser.getNumberChars().toString(); + case JSONParser.BOOLEAN: + return Boolean.toString(parser.getBoolean()); + case JSONParser.NULL: + return null; + default: + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Expected primitive JSON value but got: "+JSONParser.getEventString( ev ) + +" at ["+parser.getPosition()+"]" ); + } + } + + + void handleSingleDelete(int ev) throws IOException { + if (ev == JSONParser.OBJECT_START) { + handleDeleteMap(ev); + } else { + DeleteUpdateCommand cmd = new DeleteUpdateCommand(req); + cmd.commitWithin = commitWithin; + String id = getString(ev); + cmd.setId(id); + processor.processDelete(cmd); + } + } + + void handleDeleteArray(int ev) throws IOException { + assert ev == JSONParser.ARRAY_START; + for (;;) { + ev = parser.nextEvent(); + if (ev == JSONParser.ARRAY_END) return; + handleSingleDelete(ev); + } + } + + void handleDeleteMap(int ev) throws IOException { + assert ev == JSONParser.OBJECT_START; + DeleteUpdateCommand cmd = new DeleteUpdateCommand(req); cmd.commitWithin = commitWithin; - + while( true ) { - int ev = parser.nextEvent(); + ev = parser.nextEvent(); if( ev == JSONParser.STRING ) { String key = parser.getString(); if( parser.wasKey() ) { if( "id".equals( key ) ) { - cmd.setId(parser.getString()); - } - else if( "query".equals(key) ) { + cmd.setId(getString(parser.nextEvent())); + } else if( "query".equals(key) ) { cmd.setQuery(parser.getString()); - } - else if( "commitWithin".equals(key) ) { - cmd.commitWithin = Integer.parseInt(parser.getString()); + } else if( "commitWithin".equals(key) ) { + cmd.commitWithin = (int)parser.getLong(); + } else if( "_version_".equals(key) ) { + cmd.setVersion(parser.getLong()); } else { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown key: "+key+" ["+parser.getPosition()+"]" ); } } else { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "invalid string: " + key - +" at ["+parser.getPosition()+"]" ); + "invalid string: " + key + +" at ["+parser.getPosition()+"]" ); } } else if( ev == JSONParser.OBJECT_END ) { if( cmd.getId() == null && cmd.getQuery() == null ) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Missing id or query for delete ["+parser.getPosition()+"]" ); } - return cmd; + + processor.processDelete(cmd); + return; } else { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Got: "+JSONParser.getEventString( ev ) - +" at ["+parser.getPosition()+"]" ); + +" at ["+parser.getPosition()+"]" ); } } } + + + RollbackUpdateCommand parseRollback() throws IOException { assertNextEvent( JSONParser.OBJECT_START ); diff --git a/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java index a3e67e1..0fe0165 100644 --- a/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java +++ b/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.loader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -48,7 +48,6 @@ import javax.xml.stream.FactoryConfigurationError; import javax.xml.stream.XMLStreamConstants; import javax.xml.stream.XMLInputFactory; import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerConfigurationException; import javax.xml.transform.TransformerException; import javax.xml.transform.dom.DOMResult; import javax.xml.transform.dom.DOMSource; @@ -190,9 +189,7 @@ public class XMLLoader extends ContentStreamLoader { * @since solr 1.2 */ void processUpdate(SolrQueryRequest req, UpdateRequestProcessor processor, XMLStreamReader parser) - throws XMLStreamException, IOException, FactoryConfigurationError, - InstantiationException, IllegalAccessException, - TransformerConfigurationException { + throws XMLStreamException, IOException, FactoryConfigurationError { AddUpdateCommand addCmd = null; SolrParams params = req.getParams(); while (true) { diff --git a/solr/core/src/java/org/apache/solr/highlight/BreakIteratorBoundaryScanner.java b/solr/core/src/java/org/apache/solr/highlight/BreakIteratorBoundaryScanner.java index c7ffbad..4db908c 100644 --- a/solr/core/src/java/org/apache/solr/highlight/BreakIteratorBoundaryScanner.java +++ b/solr/core/src/java/org/apache/solr/highlight/BreakIteratorBoundaryScanner.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/highlight/DefaultEncoder.java b/solr/core/src/java/org/apache/solr/highlight/DefaultEncoder.java index 33ad92f..28291f6 100644 --- a/solr/core/src/java/org/apache/solr/highlight/DefaultEncoder.java +++ b/solr/core/src/java/org/apache/solr/highlight/DefaultEncoder.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java b/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java index 403c4aa..7527f7b 100644 --- a/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java +++ b/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -202,9 +202,8 @@ public class DefaultSolrHighlighter extends SolrHighlighter implements PluginInf * @param tokenStream document text CachingTokenStream * @param fieldName The name of the field * @param request The SolrQueryRequest - * @throws IOException */ - private QueryScorer getSpanQueryScorer(Query query, String fieldName, TokenStream tokenStream, SolrQueryRequest request) throws IOException { + private QueryScorer getSpanQueryScorer(Query query, String fieldName, TokenStream tokenStream, SolrQueryRequest request) { boolean reqFieldMatch = request.getParams().getFieldBool(fieldName, HighlightParams.FIELD_MATCH, false); Boolean highlightMultiTerm = request.getParams().getBool(HighlightParams.HIGHLIGHT_MULTI_TERM, true); if(highlightMultiTerm == null) { diff --git a/solr/core/src/java/org/apache/solr/highlight/GapFragmenter.java b/solr/core/src/java/org/apache/solr/highlight/GapFragmenter.java index d2be75a..4b4bf89 100644 --- a/solr/core/src/java/org/apache/solr/highlight/GapFragmenter.java +++ b/solr/core/src/java/org/apache/solr/highlight/GapFragmenter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/highlight/HighlightingPluginBase.java b/solr/core/src/java/org/apache/solr/highlight/HighlightingPluginBase.java index 41d92f5..e900ef1 100644 --- a/solr/core/src/java/org/apache/solr/highlight/HighlightingPluginBase.java +++ b/solr/core/src/java/org/apache/solr/highlight/HighlightingPluginBase.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/highlight/HtmlEncoder.java b/solr/core/src/java/org/apache/solr/highlight/HtmlEncoder.java index 1557527..79d9d92 100644 --- a/solr/core/src/java/org/apache/solr/highlight/HtmlEncoder.java +++ b/solr/core/src/java/org/apache/solr/highlight/HtmlEncoder.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/highlight/HtmlFormatter.java b/solr/core/src/java/org/apache/solr/highlight/HtmlFormatter.java index f99fb00..1aeab29 100644 --- a/solr/core/src/java/org/apache/solr/highlight/HtmlFormatter.java +++ b/solr/core/src/java/org/apache/solr/highlight/HtmlFormatter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/highlight/RegexFragmenter.java b/solr/core/src/java/org/apache/solr/highlight/RegexFragmenter.java index bc272c3..a080854 100644 --- a/solr/core/src/java/org/apache/solr/highlight/RegexFragmenter.java +++ b/solr/core/src/java/org/apache/solr/highlight/RegexFragmenter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/highlight/ScoreOrderFragmentsBuilder.java b/solr/core/src/java/org/apache/solr/highlight/ScoreOrderFragmentsBuilder.java index ada968e..05375e2 100644 --- a/solr/core/src/java/org/apache/solr/highlight/ScoreOrderFragmentsBuilder.java +++ b/solr/core/src/java/org/apache/solr/highlight/ScoreOrderFragmentsBuilder.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/highlight/SimpleBoundaryScanner.java b/solr/core/src/java/org/apache/solr/highlight/SimpleBoundaryScanner.java index 61109b5..908c4f0 100644 --- a/solr/core/src/java/org/apache/solr/highlight/SimpleBoundaryScanner.java +++ b/solr/core/src/java/org/apache/solr/highlight/SimpleBoundaryScanner.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/highlight/SimpleFragListBuilder.java b/solr/core/src/java/org/apache/solr/highlight/SimpleFragListBuilder.java index 9859487..f1b842c 100644 --- a/solr/core/src/java/org/apache/solr/highlight/SimpleFragListBuilder.java +++ b/solr/core/src/java/org/apache/solr/highlight/SimpleFragListBuilder.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/highlight/SimpleFragmentsBuilder.java b/solr/core/src/java/org/apache/solr/highlight/SimpleFragmentsBuilder.java index 8c2210b..da5d552 100644 --- a/solr/core/src/java/org/apache/solr/highlight/SimpleFragmentsBuilder.java +++ b/solr/core/src/java/org/apache/solr/highlight/SimpleFragmentsBuilder.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/highlight/SingleFragListBuilder.java b/solr/core/src/java/org/apache/solr/highlight/SingleFragListBuilder.java index 273c8dc..d8571fc 100644 --- a/solr/core/src/java/org/apache/solr/highlight/SingleFragListBuilder.java +++ b/solr/core/src/java/org/apache/solr/highlight/SingleFragListBuilder.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/highlight/SolrBoundaryScanner.java b/solr/core/src/java/org/apache/solr/highlight/SolrBoundaryScanner.java index fd0aa01..038a8d0 100644 --- a/solr/core/src/java/org/apache/solr/highlight/SolrBoundaryScanner.java +++ b/solr/core/src/java/org/apache/solr/highlight/SolrBoundaryScanner.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/highlight/SolrEncoder.java b/solr/core/src/java/org/apache/solr/highlight/SolrEncoder.java index f37f737..14f991c 100644 --- a/solr/core/src/java/org/apache/solr/highlight/SolrEncoder.java +++ b/solr/core/src/java/org/apache/solr/highlight/SolrEncoder.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/highlight/SolrFormatter.java b/solr/core/src/java/org/apache/solr/highlight/SolrFormatter.java index 088a88b..ec14a44 100644 --- a/solr/core/src/java/org/apache/solr/highlight/SolrFormatter.java +++ b/solr/core/src/java/org/apache/solr/highlight/SolrFormatter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/highlight/SolrFragListBuilder.java b/solr/core/src/java/org/apache/solr/highlight/SolrFragListBuilder.java index e7e134d..bbf2b72 100644 --- a/solr/core/src/java/org/apache/solr/highlight/SolrFragListBuilder.java +++ b/solr/core/src/java/org/apache/solr/highlight/SolrFragListBuilder.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/highlight/SolrFragmenter.java b/solr/core/src/java/org/apache/solr/highlight/SolrFragmenter.java index 304fdf5..0678ec1 100644 --- a/solr/core/src/java/org/apache/solr/highlight/SolrFragmenter.java +++ b/solr/core/src/java/org/apache/solr/highlight/SolrFragmenter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/highlight/SolrFragmentsBuilder.java b/solr/core/src/java/org/apache/solr/highlight/SolrFragmentsBuilder.java index 2a22b00..c676488 100644 --- a/solr/core/src/java/org/apache/solr/highlight/SolrFragmentsBuilder.java +++ b/solr/core/src/java/org/apache/solr/highlight/SolrFragmentsBuilder.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java b/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java index 6f47f88..8bb6ecd 100644 --- a/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java +++ b/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java @@ -1,5 +1,5 @@ package org.apache.solr.highlight; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/highlight/WeightedFragListBuilder.java b/solr/core/src/java/org/apache/solr/highlight/WeightedFragListBuilder.java new file mode 100644 index 0000000..a63092e --- /dev/null +++ b/solr/core/src/java/org/apache/solr/highlight/WeightedFragListBuilder.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.highlight; + +import org.apache.lucene.search.vectorhighlight.FragListBuilder; +import org.apache.solr.common.params.SolrParams; + +public class WeightedFragListBuilder extends HighlightingPluginBase implements + SolrFragListBuilder { + + @Override + public FragListBuilder getFragListBuilder(SolrParams params) { + // NOTE: This class (currently) makes no use of params + // If that ever changes, it should wrap them with defaults... + // params = SolrParams.wrapDefaults(params, defaults) + + numRequests++; + + return new org.apache.lucene.search.vectorhighlight.WeightedFragListBuilder(); + } + + /////////////////////////////////////////////////////////////////////// + //////////////////////// SolrInfoMBeans methods /////////////////////// + /////////////////////////////////////////////////////////////////////// + + @Override + public String getDescription() { + return "WeightedFragListBuilder"; + } + + @Override + public String getSource() { + return "$URL: http://svn.apache.org/repos/asf/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/highlight/WeightedFragListBuilder.java $"; + } +} diff --git a/solr/core/src/java/org/apache/solr/logging/CircularList.java b/solr/core/src/java/org/apache/solr/logging/CircularList.java index 7095d72..fba31f2 100644 --- a/solr/core/src/java/org/apache/solr/logging/CircularList.java +++ b/solr/core/src/java/org/apache/solr/logging/CircularList.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/logging/ListenerConfig.java b/solr/core/src/java/org/apache/solr/logging/ListenerConfig.java index 5d1ef3a..486a6d4 100644 --- a/solr/core/src/java/org/apache/solr/logging/ListenerConfig.java +++ b/solr/core/src/java/org/apache/solr/logging/ListenerConfig.java @@ -1,7 +1,7 @@ package org.apache.solr.logging; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/logging/LogWatcher.java b/solr/core/src/java/org/apache/solr/logging/LogWatcher.java index 62f388c..de64f14 100644 --- a/solr/core/src/java/org/apache/solr/logging/LogWatcher.java +++ b/solr/core/src/java/org/apache/solr/logging/LogWatcher.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/logging/LoggerInfo.java b/solr/core/src/java/org/apache/solr/logging/LoggerInfo.java index 08987bb..f493761 100644 --- a/solr/core/src/java/org/apache/solr/logging/LoggerInfo.java +++ b/solr/core/src/java/org/apache/solr/logging/LoggerInfo.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/logging/jul/JulInfo.java b/solr/core/src/java/org/apache/solr/logging/jul/JulInfo.java index 7ec22c6..9c9d365 100644 --- a/solr/core/src/java/org/apache/solr/logging/jul/JulInfo.java +++ b/solr/core/src/java/org/apache/solr/logging/jul/JulInfo.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/logging/jul/JulWatcher.java b/solr/core/src/java/org/apache/solr/logging/jul/JulWatcher.java index de6adff..25e72a5 100644 --- a/solr/core/src/java/org/apache/solr/logging/jul/JulWatcher.java +++ b/solr/core/src/java/org/apache/solr/logging/jul/JulWatcher.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/logging/jul/RecordHandler.java b/solr/core/src/java/org/apache/solr/logging/jul/RecordHandler.java index 1067e20..6ed119e 100644 --- a/solr/core/src/java/org/apache/solr/logging/jul/RecordHandler.java +++ b/solr/core/src/java/org/apache/solr/logging/jul/RecordHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/request/LocalSolrQueryRequest.java b/solr/core/src/java/org/apache/solr/request/LocalSolrQueryRequest.java index 900cb86..fd601ef 100644 --- a/solr/core/src/java/org/apache/solr/request/LocalSolrQueryRequest.java +++ b/solr/core/src/java/org/apache/solr/request/LocalSolrQueryRequest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java b/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java index 3c94bec..60e6237 100755 --- a/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java +++ b/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -82,7 +82,7 @@ class PerSegmentSingleValuedFaceting { // reuse the translation logic to go from top level set to per-segment set baseSet = docs.getTopFilter(); - final AtomicReaderContext[] leaves = searcher.getTopReaderContext().leaves(); + final List leaves = searcher.getTopReaderContext().leaves(); // The list of pending tasks that aren't immediately submitted // TODO: Is there a completion service, or a delegating executor that can // limit the number of concurrent tasks submitted to a bigger executor? @@ -90,8 +90,8 @@ class PerSegmentSingleValuedFaceting { int threads = nThreads <= 0 ? Integer.MAX_VALUE : nThreads; - for (int i=0; i task = new Callable() { public SegFacet call() throws Exception { @@ -111,7 +111,7 @@ class PerSegmentSingleValuedFaceting { // now merge the per-segment results - PriorityQueue queue = new PriorityQueue(leaves.length) { + PriorityQueue queue = new PriorityQueue(leaves.size()) { @Override protected boolean lessThan(SegFacet a, SegFacet b) { return a.tempBR.compareTo(b.tempBR) < 0; @@ -121,7 +121,7 @@ class PerSegmentSingleValuedFaceting { boolean hasMissingCount=false; int missingCount=0; - for (int i=0; i getTermCounts(String field) throws IOException { int offset = params.getFieldInt(field, FacetParams.FACET_OFFSET, 0); @@ -1172,7 +1199,11 @@ public class SimpleFacets { protected int rangeCount(SchemaField sf, String low, String high, boolean iLow, boolean iHigh) throws IOException { Query rangeQ = sf.getType().getRangeQuery(null, sf,low,high,iLow,iHigh); - return searcher.numDocs(rangeQ ,base); + if (params.getBool(GroupParams.GROUP_FACET, false)) { + return getGroupedFacetQueryCount(rangeQ); + } else { + return searcher.numDocs(rangeQ ,base); + } } /** diff --git a/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java b/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java index 931acbc..80e5648 100644 --- a/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java +++ b/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java b/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java index 82d2872..881482e 100644 --- a/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java +++ b/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/request/SolrRequestHandler.java b/solr/core/src/java/org/apache/solr/request/SolrRequestHandler.java index 50adac0..9479f0e 100644 --- a/solr/core/src/java/org/apache/solr/request/SolrRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/request/SolrRequestHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -28,10 +28,10 @@ import org.apache.solr.response.SolrQueryResponse; * One way to register a SolrRequestHandler with the core is thorugh the solrconfig.xml file. *

      * Example solrconfig.xml entry to register a SolrRequestHandler implementation to - * handle all queries with a query type of "test": + * handle all queries with a Request Handler of "/test": *

      * - * <requestHandler name="test" class="solr.tst.TestRequestHandler" /> + * <requestHandler name="/test" class="solr.tst.TestRequestHandler" /> * *

      * A single instance of any registered SolrRequestHandler is created diff --git a/solr/core/src/java/org/apache/solr/request/SolrRequestInfo.java b/solr/core/src/java/org/apache/solr/request/SolrRequestInfo.java index 8b1233d..e371db6 100755 --- a/solr/core/src/java/org/apache/solr/request/SolrRequestInfo.java +++ b/solr/core/src/java/org/apache/solr/request/SolrRequestInfo.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/request/UnInvertedField.java b/solr/core/src/java/org/apache/solr/request/UnInvertedField.java index eeba3de..811b604 100755 --- a/solr/core/src/java/org/apache/solr/request/UnInvertedField.java +++ b/solr/core/src/java/org/apache/solr/request/UnInvertedField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/BinaryQueryResponseWriter.java b/solr/core/src/java/org/apache/solr/response/BinaryQueryResponseWriter.java index a641929..7df54e1 100755 --- a/solr/core/src/java/org/apache/solr/response/BinaryQueryResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/BinaryQueryResponseWriter.java @@ -1,5 +1,5 @@ package org.apache.solr.response; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java b/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java index 2fe6fba..253548a 100755 --- a/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java b/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java index 26fea81..eda4f5c 100755 --- a/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/JSONResponseWriter.java b/solr/core/src/java/org/apache/solr/response/JSONResponseWriter.java index 847c16d..ee9ebdb 100644 --- a/solr/core/src/java/org/apache/solr/response/JSONResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/JSONResponseWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -92,7 +92,7 @@ class JSONWriter extends TextResponseWriter { if(wrapperFunction!=null) { writer.write(')'); } - if (doIndent) writer.write('\n'); // ending with a newline looks much better from the command line + writer.write('\n'); // ending with a newline looks much better from the command line } protected void writeKey(String fname, boolean needsEscaping) throws IOException { diff --git a/solr/core/src/java/org/apache/solr/response/PHPResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PHPResponseWriter.java index f94c9df..f2a9e2f 100755 --- a/solr/core/src/java/org/apache/solr/response/PHPResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/PHPResponseWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java index 793b482..72cad7c 100755 --- a/solr/core/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/PythonResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PythonResponseWriter.java index 181a936..78c79b8 100644 --- a/solr/core/src/java/org/apache/solr/response/PythonResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/PythonResponseWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/QueryResponseWriter.java b/solr/core/src/java/org/apache/solr/response/QueryResponseWriter.java index 81e19b7..f6c3ea7 100644 --- a/solr/core/src/java/org/apache/solr/response/QueryResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/QueryResponseWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/RawResponseWriter.java b/solr/core/src/java/org/apache/solr/response/RawResponseWriter.java index dba1eac..0107ca1 100644 --- a/solr/core/src/java/org/apache/solr/response/RawResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/RawResponseWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/ResultContext.java b/solr/core/src/java/org/apache/solr/response/ResultContext.java index e218df5..12963ce 100644 --- a/solr/core/src/java/org/apache/solr/response/ResultContext.java +++ b/solr/core/src/java/org/apache/solr/response/ResultContext.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/RubyResponseWriter.java b/solr/core/src/java/org/apache/solr/response/RubyResponseWriter.java index 585b248..dcaf913 100644 --- a/solr/core/src/java/org/apache/solr/response/RubyResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/RubyResponseWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/SolrQueryResponse.java b/solr/core/src/java/org/apache/solr/response/SolrQueryResponse.java index 0afec1e..6918bdf 100644 --- a/solr/core/src/java/org/apache/solr/response/SolrQueryResponse.java +++ b/solr/core/src/java/org/apache/solr/response/SolrQueryResponse.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java b/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java index 38fcd54..bb64aad 100644 --- a/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/XMLResponseWriter.java b/solr/core/src/java/org/apache/solr/response/XMLResponseWriter.java index 95d4654..642a575 100644 --- a/solr/core/src/java/org/apache/solr/response/XMLResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/XMLResponseWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/XMLWriter.java b/solr/core/src/java/org/apache/solr/response/XMLWriter.java index 8bc74a9..f453646 100644 --- a/solr/core/src/java/org/apache/solr/response/XMLWriter.java +++ b/solr/core/src/java/org/apache/solr/response/XMLWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/XSLTResponseWriter.java b/solr/core/src/java/org/apache/solr/response/XSLTResponseWriter.java index f4c421f..3879437 100644 --- a/solr/core/src/java/org/apache/solr/response/XSLTResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/XSLTResponseWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/transform/DocIdAugmenterFactory.java b/solr/core/src/java/org/apache/solr/response/transform/DocIdAugmenterFactory.java index 8c23d31..5b857e1 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/DocIdAugmenterFactory.java +++ b/solr/core/src/java/org/apache/solr/response/transform/DocIdAugmenterFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java b/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java index f1cd3c8..85d775f 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java +++ b/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/transform/DocTransformers.java b/solr/core/src/java/org/apache/solr/response/transform/DocTransformers.java index 3be17c4..950b1cc 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/DocTransformers.java +++ b/solr/core/src/java/org/apache/solr/response/transform/DocTransformers.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/transform/ExplainAugmenterFactory.java b/solr/core/src/java/org/apache/solr/response/transform/ExplainAugmenterFactory.java index 3b0d429..c305b5b 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/ExplainAugmenterFactory.java +++ b/solr/core/src/java/org/apache/solr/response/transform/ExplainAugmenterFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/transform/RenameFieldTransformer.java b/solr/core/src/java/org/apache/solr/response/transform/RenameFieldTransformer.java index 2e1a20b..a054f0b 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/RenameFieldTransformer.java +++ b/solr/core/src/java/org/apache/solr/response/transform/RenameFieldTransformer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/transform/ScoreAugmenter.java b/solr/core/src/java/org/apache/solr/response/transform/ScoreAugmenter.java index afb56c9..83f5de1 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/ScoreAugmenter.java +++ b/solr/core/src/java/org/apache/solr/response/transform/ScoreAugmenter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/transform/ShardAugmenterFactory.java b/solr/core/src/java/org/apache/solr/response/transform/ShardAugmenterFactory.java index 42979b6..e65bb93 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/ShardAugmenterFactory.java +++ b/solr/core/src/java/org/apache/solr/response/transform/ShardAugmenterFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/transform/TransformContext.java b/solr/core/src/java/org/apache/solr/response/transform/TransformContext.java index 66eed65..bf5dd9c 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/TransformContext.java +++ b/solr/core/src/java/org/apache/solr/response/transform/TransformContext.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java b/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java index 9b316d6..06daa99 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java +++ b/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/transform/TransformerWithContext.java b/solr/core/src/java/org/apache/solr/response/transform/TransformerWithContext.java index 060bfd5..cedb9ea 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/TransformerWithContext.java +++ b/solr/core/src/java/org/apache/solr/response/transform/TransformerWithContext.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/transform/ValueAugmenterFactory.java b/solr/core/src/java/org/apache/solr/response/transform/ValueAugmenterFactory.java index f0eeafb..9ca571d 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/ValueAugmenterFactory.java +++ b/solr/core/src/java/org/apache/solr/response/transform/ValueAugmenterFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java b/solr/core/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java index cd77bc0..15b099b 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java +++ b/solr/core/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,16 +17,16 @@ package org.apache.solr.response.transform; import java.io.IOException; +import java.util.List; import java.util.Map; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; -import org.apache.lucene.util.ReaderUtil; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrException; -import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.search.QParser; import org.apache.solr.search.SolrIndexSearcher; @@ -64,7 +64,7 @@ public class ValueSourceAugmenter extends DocTransformer try { IndexReader reader = qparser.getReq().getSearcher().getIndexReader(); readerContexts = reader.getTopReaderContext().leaves(); - docValuesArr = new FunctionValues[readerContexts.length]; + docValuesArr = new FunctionValues[readerContexts.size()]; searcher = qparser.getReq().getSearcher(); fcontext = ValueSource.newContext(searcher); @@ -77,7 +77,7 @@ public class ValueSourceAugmenter extends DocTransformer Map fcontext; SolrIndexSearcher searcher; - AtomicReaderContext[] readerContexts; + List readerContexts; FunctionValues docValuesArr[]; @@ -89,7 +89,7 @@ public class ValueSourceAugmenter extends DocTransformer // TODO: calculate this stuff just once across diff functions int idx = ReaderUtil.subIndex(docid, readerContexts); - AtomicReaderContext rcontext = readerContexts[idx]; + AtomicReaderContext rcontext = readerContexts.get(idx); FunctionValues values = docValuesArr[idx]; if (values == null) { docValuesArr[idx] = values = valueSource.getValues(fcontext, rcontext); diff --git a/solr/core/src/java/org/apache/solr/schema/AbstractSubTypeFieldType.java b/solr/core/src/java/org/apache/solr/schema/AbstractSubTypeFieldType.java index 197acac..bd38bfe 100644 --- a/solr/core/src/java/org/apache/solr/schema/AbstractSubTypeFieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/AbstractSubTypeFieldType.java @@ -1,5 +1,5 @@ package org.apache.solr.schema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/BCDIntField.java b/solr/core/src/java/org/apache/solr/schema/BCDIntField.java index d127123..ba49a4d 100644 --- a/solr/core/src/java/org/apache/solr/schema/BCDIntField.java +++ b/solr/core/src/java/org/apache/solr/schema/BCDIntField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/BCDLongField.java b/solr/core/src/java/org/apache/solr/schema/BCDLongField.java index 8335975..b8f62b4 100644 --- a/solr/core/src/java/org/apache/solr/schema/BCDLongField.java +++ b/solr/core/src/java/org/apache/solr/schema/BCDLongField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/BCDStrField.java b/solr/core/src/java/org/apache/solr/schema/BCDStrField.java index 345f789..460ba14 100644 --- a/solr/core/src/java/org/apache/solr/schema/BCDStrField.java +++ b/solr/core/src/java/org/apache/solr/schema/BCDStrField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/BinaryField.java b/solr/core/src/java/org/apache/solr/schema/BinaryField.java index 77e1a1c..b02ccd1 100644 --- a/solr/core/src/java/org/apache/solr/schema/BinaryField.java +++ b/solr/core/src/java/org/apache/solr/schema/BinaryField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/BoolField.java b/solr/core/src/java/org/apache/solr/schema/BoolField.java index 0081e5f..8b29a8c 100644 --- a/solr/core/src/java/org/apache/solr/schema/BoolField.java +++ b/solr/core/src/java/org/apache/solr/schema/BoolField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/ByteField.java b/solr/core/src/java/org/apache/solr/schema/ByteField.java index 650a9aa..bbcc23b 100644 --- a/solr/core/src/java/org/apache/solr/schema/ByteField.java +++ b/solr/core/src/java/org/apache/solr/schema/ByteField.java @@ -1,5 +1,5 @@ package org.apache.solr.schema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/CollationField.java b/solr/core/src/java/org/apache/solr/schema/CollationField.java index 5327123..42d7d09 100644 --- a/solr/core/src/java/org/apache/solr/schema/CollationField.java +++ b/solr/core/src/java/org/apache/solr/schema/CollationField.java @@ -1,6 +1,6 @@ package org.apache.solr.schema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/CoordinateFieldType.java b/solr/core/src/java/org/apache/solr/schema/CoordinateFieldType.java index d986ddd..b81a281 100644 --- a/solr/core/src/java/org/apache/solr/schema/CoordinateFieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/CoordinateFieldType.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/CopyField.java b/solr/core/src/java/org/apache/solr/schema/CopyField.java index 372cb4f..d152192 100644 --- a/solr/core/src/java/org/apache/solr/schema/CopyField.java +++ b/solr/core/src/java/org/apache/solr/schema/CopyField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/CurrencyField.java b/solr/core/src/java/org/apache/solr/schema/CurrencyField.java index e1ee2f4..4fc7ac2 100644 --- a/solr/core/src/java/org/apache/solr/schema/CurrencyField.java +++ b/solr/core/src/java/org/apache/solr/schema/CurrencyField.java @@ -1,5 +1,5 @@ package org.apache.solr.schema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/DateField.java b/solr/core/src/java/org/apache/solr/schema/DateField.java index c333fab..58d22fd 100644 --- a/solr/core/src/java/org/apache/solr/schema/DateField.java +++ b/solr/core/src/java/org/apache/solr/schema/DateField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/DoubleField.java b/solr/core/src/java/org/apache/solr/schema/DoubleField.java index 0a7b740..06ecd3c 100644 --- a/solr/core/src/java/org/apache/solr/schema/DoubleField.java +++ b/solr/core/src/java/org/apache/solr/schema/DoubleField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/ExchangeRateProvider.java b/solr/core/src/java/org/apache/solr/schema/ExchangeRateProvider.java index 6c6ac78..cc558f3 100644 --- a/solr/core/src/java/org/apache/solr/schema/ExchangeRateProvider.java +++ b/solr/core/src/java/org/apache/solr/schema/ExchangeRateProvider.java @@ -1,5 +1,5 @@ package org.apache.solr.schema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/ExternalFileField.java b/solr/core/src/java/org/apache/solr/schema/ExternalFileField.java index c4dbf19..e7f4c08 100755 --- a/solr/core/src/java/org/apache/solr/schema/ExternalFileField.java +++ b/solr/core/src/java/org/apache/solr/schema/ExternalFileField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/FieldProperties.java b/solr/core/src/java/org/apache/solr/schema/FieldProperties.java index 91e26f4..571afa5 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldProperties.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldProperties.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java index 02f11aa..962bfa0 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java b/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java index 75bbd83..0adac8e 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -135,10 +135,10 @@ public final class FieldTypePluginLoader return fieldTypes.put( name, plugin ); } - // The point here is that, if no multitermanalyzer was specified in the schema file, do one of several things: + // The point here is that, if no multiterm analyzer was specified in the schema file, do one of several things: // 1> If legacyMultiTerm == false, assemble a new analyzer composed of all of the charfilters, // lowercase filters and asciifoldingfilter. - // 2> If letacyMultiTerm == true just construct the analyzer from a KeywordTokenizer. That should mimic current behavior. + // 2> If legacyMultiTerm == true just construct the analyzer from a KeywordTokenizer. That should mimic current behavior. // Do the same if they've specified that the old behavior is required (legacyMultiTerm="true") private Analyzer constructMultiTermAnalyzer(Analyzer queryAnalyzer) { diff --git a/solr/core/src/java/org/apache/solr/schema/FloatField.java b/solr/core/src/java/org/apache/solr/schema/FloatField.java index 45c546f..4ac8b21 100644 --- a/solr/core/src/java/org/apache/solr/schema/FloatField.java +++ b/solr/core/src/java/org/apache/solr/schema/FloatField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/GeoHashField.java b/solr/core/src/java/org/apache/solr/schema/GeoHashField.java index 2912678..48540d8 100644 --- a/solr/core/src/java/org/apache/solr/schema/GeoHashField.java +++ b/solr/core/src/java/org/apache/solr/schema/GeoHashField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java index 6b48187..75edf3c 100644 --- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java +++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,14 +20,12 @@ package org.apache.solr.schema; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.AnalyzerWrapper; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.Version; import org.apache.lucene.analysis.util.ResourceLoader; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.SolrParams; import org.apache.solr.util.DOMUtil; -import org.apache.solr.common.util.NamedList; import org.apache.solr.util.SystemIdResolver; import org.apache.solr.core.SolrConfig; import org.apache.solr.core.Config; @@ -39,11 +37,18 @@ import org.xml.sax.InputSource; import javax.xml.xpath.XPath; import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpressionException; -import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + /** * IndexSchema contains information about the valid fields in an index * and the types of those fields. @@ -211,7 +216,9 @@ public final class IndexSchema { /** - * Name of the default search field specified in the schema file + * Name of the default search field specified in the schema file. + *
      Note:Avoid calling this, try to use this method so that the 'df' param is consulted as an override: + * {@link org.apache.solr.search.QueryParsing#getDefaultField(IndexSchema, String)} */ public String getDefaultSearchFieldName() { return defaultSearchFieldName; @@ -433,12 +440,22 @@ public final class IndexSchema { } if (simFactory instanceof SchemaAware) { ((SchemaAware)simFactory).inform(this); + } else { + // if the sim facotry isn't schema aware, then we are responsible for + // erroring if a field type is trying to specify a sim. + for (FieldType ft : fieldTypes.values()) { + if (null != ft.getSimilarity()) { + String msg = "FieldType '" + ft.getTypeName() + "' is configured with a similarity, but the global similarity does not support it: " + simFactory.getClass(); + log.error(msg); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg); + } + } } similarity = simFactory.getSimilarity(); node = (Node) xpath.evaluate("/schema/defaultSearchField/text()", document, XPathConstants.NODE); if (node==null) { - log.warn("no default search field specified in schema."); + log.debug("no default search field specified in schema."); } else { defaultSearchFieldName=node.getNodeValue().trim(); // throw exception if specified, but not found or not indexed @@ -449,7 +466,7 @@ public final class IndexSchema { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, msg ); } } - log.info("default search field is "+defaultSearchFieldName); + log.info("default search field in schema is "+defaultSearchFieldName); } node = (Node) xpath.evaluate("/schema/solrQueryParser/@defaultOperator", document, XPathConstants.NODE); @@ -465,6 +482,14 @@ public final class IndexSchema { log.warn("no uniqueKey specified in schema."); } else { uniqueKeyField=getIndexedField(node.getNodeValue().trim()); + if (null != uniqueKeyField.getDefaultValue()) { + String msg = "uniqueKey field ("+uniqueKeyFieldName+ + ") can not be configured with a default value ("+ + uniqueKeyField.getDefaultValue()+")"; + log.error(msg); + throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, msg ); + } + if (!uniqueKeyField.stored()) { log.error("uniqueKey is not stored - distributed search will not work"); } @@ -507,6 +532,14 @@ public final class IndexSchema { } } + if (dest.equals(uniqueKeyFieldName)) { + String msg = "uniqueKey field ("+uniqueKeyFieldName+ + ") can not be the dest of a copyField (src="+source+")"; + log.error(msg); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg); + + } + registerCopyField(source, dest, maxCharsInt); } @@ -517,6 +550,8 @@ public final class IndexSchema { entry.getValue()+")"); } } + + //Run the callbacks on SchemaAware now that everything else is done for (SchemaAware aware : schemaAware) { aware.inform(this); @@ -658,7 +693,7 @@ public final class IndexSchema { return newArr; } - static SimilarityFactory readSimilarity(ResourceLoader loader, Node node) throws XPathExpressionException { + static SimilarityFactory readSimilarity(ResourceLoader loader, Node node) { if (node==null) { return null; } else { diff --git a/solr/core/src/java/org/apache/solr/schema/IntField.java b/solr/core/src/java/org/apache/solr/schema/IntField.java index 286c24e..20680b2 100644 --- a/solr/core/src/java/org/apache/solr/schema/IntField.java +++ b/solr/core/src/java/org/apache/solr/schema/IntField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/JsonPreAnalyzedParser.java b/solr/core/src/java/org/apache/solr/schema/JsonPreAnalyzedParser.java index 62b6405..e3cb5f6 100644 --- a/solr/core/src/java/org/apache/solr/schema/JsonPreAnalyzedParser.java +++ b/solr/core/src/java/org/apache/solr/schema/JsonPreAnalyzedParser.java @@ -2,7 +2,7 @@ package org.apache.solr.schema; import java.io.IOException; import java.io.Reader; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -19,7 +19,6 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; import org.apache.lucene.analysis.tokenattributes.TypeAttribute; import org.apache.lucene.document.Field; -import org.apache.lucene.index.Payload; import org.apache.lucene.util.Attribute; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.AttributeSource.State; @@ -171,7 +170,7 @@ public class JsonPreAnalyzedParser implements PreAnalyzedParser { byte[] data = Base64.base64ToByteArray(str); PayloadAttribute p = parent.addAttribute(PayloadAttribute.class); if (data != null && data.length > 0) { - p.setPayload(new Payload(data)); + p.setPayload(new BytesRef(data)); } } } else if (key.equals(FLAGS_KEY)) { @@ -209,7 +208,7 @@ public class JsonPreAnalyzedParser implements PreAnalyzedParser { @Override public String toFormattedString(Field f) throws IOException { - Map map = new HashMap(); + Map map = new LinkedHashMap(); map.put(VERSION_KEY, VERSION); if (f.fieldType().stored()) { String stringValue = f.stringValue(); @@ -248,9 +247,9 @@ public class JsonPreAnalyzedParser implements PreAnalyzedParser { tok.put(OFFSET_START_KEY, ((OffsetAttribute)att).startOffset()); tok.put(OFFSET_END_KEY, ((OffsetAttribute)att).endOffset()); } else if (cl.isAssignableFrom(PayloadAttribute.class)) { - Payload p = ((PayloadAttribute)att).getPayload(); - if (p != null && p.length() > 0) { - tok.put(PAYLOAD_KEY, Base64.byteArrayToBase64(p.getData(), p.getOffset(), p.length())); + BytesRef p = ((PayloadAttribute)att).getPayload(); + if (p != null && p.length > 0) { + tok.put(PAYLOAD_KEY, Base64.byteArrayToBase64(p.bytes, p.offset, p.length)); } } else if (cl.isAssignableFrom(PositionIncrementAttribute.class)) { tok.put(POSINCR_KEY, ((PositionIncrementAttribute)att).getPositionIncrement()); diff --git a/solr/core/src/java/org/apache/solr/schema/LatLonType.java b/solr/core/src/java/org/apache/solr/schema/LatLonType.java index 4177dfa..5852364 100644 --- a/solr/core/src/java/org/apache/solr/schema/LatLonType.java +++ b/solr/core/src/java/org/apache/solr/schema/LatLonType.java @@ -1,5 +1,5 @@ package org.apache.solr.schema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -602,6 +602,7 @@ class SpatialDistanceQuery extends ExtendedQueryBase implements PostFilter { // don't bother making the hash expensive - the center latitude + min longitude will be very uinque long hash = Double.doubleToLongBits(latCenter); hash = hash * 31 + Double.doubleToLongBits(lonMin); + hash = hash * 31 + (long)super.hashCode(); return (int)(hash >> 32 + hash); } diff --git a/solr/core/src/java/org/apache/solr/schema/LongField.java b/solr/core/src/java/org/apache/solr/schema/LongField.java index d20040d..d9e5dad 100644 --- a/solr/core/src/java/org/apache/solr/schema/LongField.java +++ b/solr/core/src/java/org/apache/solr/schema/LongField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java b/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java index ca9f304..c69c99f 100644 --- a/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java +++ b/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java @@ -1,5 +1,5 @@ package org.apache.solr.schema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/PointType.java b/solr/core/src/java/org/apache/solr/schema/PointType.java index 7b7a8ba..927032b 100644 --- a/solr/core/src/java/org/apache/solr/schema/PointType.java +++ b/solr/core/src/java/org/apache/solr/schema/PointType.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java b/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java index 91c736c..be93b63 100644 --- a/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java +++ b/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java @@ -1,5 +1,5 @@ package org.apache.solr.schema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -214,7 +214,7 @@ public class PreAnalyzedField extends FieldType { return binaryValue; } - public final boolean incrementToken() throws IOException { + public final boolean incrementToken() { // lazy init the iterator if (it == null) { it = cachedStates.iterator(); diff --git a/solr/core/src/java/org/apache/solr/schema/PrimitiveFieldType.java b/solr/core/src/java/org/apache/solr/schema/PrimitiveFieldType.java index 983b5f3..7fe4c05 100644 --- a/solr/core/src/java/org/apache/solr/schema/PrimitiveFieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/PrimitiveFieldType.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/RandomSortField.java b/solr/core/src/java/org/apache/solr/schema/RandomSortField.java index 9640f94..e766609 100644 --- a/solr/core/src/java/org/apache/solr/schema/RandomSortField.java +++ b/solr/core/src/java/org/apache/solr/schema/RandomSortField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,11 +23,11 @@ import java.util.Map; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.docvalues.IntDocValues; import org.apache.lucene.search.*; -import org.apache.lucene.util.ReaderUtil; import org.apache.solr.response.TextResponseWriter; import org.apache.solr.search.QParser; @@ -102,7 +102,7 @@ public class RandomSortField extends FieldType { private static FieldComparatorSource randomComparatorSource = new FieldComparatorSource() { @Override - public FieldComparator newComparator(final String fieldname, final int numHits, int sortPos, boolean reversed) throws IOException { + public FieldComparator newComparator(final String fieldname, final int numHits, int sortPos, boolean reversed) { return new FieldComparator() { int seed; private final int[] values = new int[numHits]; @@ -119,17 +119,17 @@ public class RandomSortField extends FieldType { } @Override - public int compareBottom(int doc) throws IOException { + public int compareBottom(int doc) { return bottomVal - hash(doc+seed); } @Override - public void copy(int slot, int doc) throws IOException { + public void copy(int slot, int doc) { values[slot] = hash(doc+seed); } @Override - public FieldComparator setNextReader(AtomicReaderContext context) throws IOException { + public FieldComparator setNextReader(AtomicReaderContext context) { seed = getSeed(fieldname, context); return this; } diff --git a/solr/core/src/java/org/apache/solr/schema/SchemaAware.java b/solr/core/src/java/org/apache/solr/schema/SchemaAware.java index 34ee8e6..ad074a9 100644 --- a/solr/core/src/java/org/apache/solr/schema/SchemaAware.java +++ b/solr/core/src/java/org/apache/solr/schema/SchemaAware.java @@ -1,5 +1,5 @@ package org.apache.solr.schema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/SchemaField.java b/solr/core/src/java/org/apache/solr/schema/SchemaField.java index aaf5c06..c7bd130 100644 --- a/solr/core/src/java/org/apache/solr/schema/SchemaField.java +++ b/solr/core/src/java/org/apache/solr/schema/SchemaField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/ShortField.java b/solr/core/src/java/org/apache/solr/schema/ShortField.java index c0b6b20..52dd913 100644 --- a/solr/core/src/java/org/apache/solr/schema/ShortField.java +++ b/solr/core/src/java/org/apache/solr/schema/ShortField.java @@ -1,5 +1,5 @@ package org.apache.solr.schema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/SimilarityFactory.java b/solr/core/src/java/org/apache/solr/schema/SimilarityFactory.java index e7a73dc..bc7cf63 100644 --- a/solr/core/src/java/org/apache/solr/schema/SimilarityFactory.java +++ b/solr/core/src/java/org/apache/solr/schema/SimilarityFactory.java @@ -1,5 +1,5 @@ package org.apache.solr.schema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,8 +17,26 @@ package org.apache.solr.schema; */ import org.apache.lucene.search.similarities.Similarity; +import org.apache.solr.schema.SchemaAware; // javadocs +import org.apache.solr.schema.FieldType; // javadocs import org.apache.solr.common.params.SolrParams; + +/** + * A factory interface for configuring a {@link Similarity} in the Solr + * schema.xml. + * + *

      + * Subclasses of SimilarityFactory which are {@link SchemaAware} + * must take responsibility for either consulting the similarities configured + * on individual field types, or generating appropriate error/warning messages + * if field type specific similarities exist but are being ignored. The + * IndexSchema will provide such error checking if a + * non-SchemaAware instance of SimilarityFactory + * is used. + * + * @see FieldType#getSimilarity + */ public abstract class SimilarityFactory { protected SolrParams params; diff --git a/solr/core/src/java/org/apache/solr/schema/SimplePreAnalyzedParser.java b/solr/core/src/java/org/apache/solr/schema/SimplePreAnalyzedParser.java index a5fd6db..faa540e 100644 --- a/solr/core/src/java/org/apache/solr/schema/SimplePreAnalyzedParser.java +++ b/solr/core/src/java/org/apache/solr/schema/SimplePreAnalyzedParser.java @@ -33,10 +33,10 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; import org.apache.lucene.analysis.tokenattributes.TypeAttribute; import org.apache.lucene.document.Field; -import org.apache.lucene.index.Payload; import org.apache.lucene.util.Attribute; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.AttributeSource.State; +import org.apache.lucene.util.BytesRef; import org.apache.solr.schema.PreAnalyzedField.ParseResult; import org.apache.solr.schema.PreAnalyzedField.PreAnalyzedParser; @@ -437,7 +437,7 @@ public final class SimplePreAnalyzedParser implements PreAnalyzedParser { PayloadAttribute p = a.addAttribute(PayloadAttribute.class); byte[] data = hexToBytes(e.getValue()); if (data != null && data.length > 0) { - p.setPayload(new Payload(data)); + p.setPayload(new BytesRef(data)); } } else { // unknown attribute @@ -498,9 +498,9 @@ public final class SimplePreAnalyzedParser implements PreAnalyzedParser { } else if (cl.isAssignableFrom(OffsetAttribute.class)) { tok.append("s=" + ((OffsetAttribute)att).startOffset() + ",e=" + ((OffsetAttribute)att).endOffset()); } else if (cl.isAssignableFrom(PayloadAttribute.class)) { - Payload p = ((PayloadAttribute)att).getPayload(); - if (p != null && p.length() > 0) { - tok.append("p=" + bytesToHex(p.getData(), p.getOffset(), p.length())); + BytesRef p = ((PayloadAttribute)att).getPayload(); + if (p != null && p.length > 0) { + tok.append("p=" + bytesToHex(p.bytes, p.offset, p.length)); } else if (tok.length() > 0) { tok.setLength(tok.length() - 1); // remove the last comma } diff --git a/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java b/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java index 79a8db1..3d34125 100644 --- a/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java +++ b/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java b/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java index 0cb9acf..f99b2b5 100644 --- a/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java +++ b/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/SortableIntField.java b/solr/core/src/java/org/apache/solr/schema/SortableIntField.java index 54be301..7c45394 100644 --- a/solr/core/src/java/org/apache/solr/schema/SortableIntField.java +++ b/solr/core/src/java/org/apache/solr/schema/SortableIntField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/SortableLongField.java b/solr/core/src/java/org/apache/solr/schema/SortableLongField.java index 3d54fc6..39ce59f 100644 --- a/solr/core/src/java/org/apache/solr/schema/SortableLongField.java +++ b/solr/core/src/java/org/apache/solr/schema/SortableLongField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/SpatialQueryable.java b/solr/core/src/java/org/apache/solr/schema/SpatialQueryable.java index e60c111..c014570 100644 --- a/solr/core/src/java/org/apache/solr/schema/SpatialQueryable.java +++ b/solr/core/src/java/org/apache/solr/schema/SpatialQueryable.java @@ -1,5 +1,5 @@ package org.apache.solr.schema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/StrField.java b/solr/core/src/java/org/apache/solr/schema/StrField.java index 6d50b66..2590737 100644 --- a/solr/core/src/java/org/apache/solr/schema/StrField.java +++ b/solr/core/src/java/org/apache/solr/schema/StrField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/StrFieldSource.java b/solr/core/src/java/org/apache/solr/schema/StrFieldSource.java index c7fe148..33f7e03 100755 --- a/solr/core/src/java/org/apache/solr/schema/StrFieldSource.java +++ b/solr/core/src/java/org/apache/solr/schema/StrFieldSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/TextField.java b/solr/core/src/java/org/apache/solr/schema/TextField.java index 8af29f0..b294988 100644 --- a/solr/core/src/java/org/apache/solr/schema/TextField.java +++ b/solr/core/src/java/org/apache/solr/schema/TextField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -168,7 +168,6 @@ public class TextField extends FieldType { static Query parseFieldQuery(QParser parser, Analyzer analyzer, String field, String queryText) { int phraseSlop = 0; - boolean enablePositionIncrements = true; // most of the following code is taken from the Lucene QueryParser @@ -187,11 +186,7 @@ public class TextField extends FieldType { PositionIncrementAttribute posIncrAtt = null; int numTokens = 0; - try { - buffer.reset(); - } catch (IOException e) { - throw new RuntimeException("Unable to initialize TokenStream to analyze query text", e); - } + buffer.reset(); if (buffer.hasAttribute(CharTermAttribute.class)) { termAtt = buffer.getAttribute(CharTermAttribute.class); @@ -289,21 +284,13 @@ public class TextField extends FieldType { } if (positionIncrement > 0 && multiTerms.size() > 0) { - if (enablePositionIncrements) { - mpq.add((Term[])multiTerms.toArray(new Term[0]),position); - } else { - mpq.add((Term[])multiTerms.toArray(new Term[0])); - } + mpq.add((Term[])multiTerms.toArray(new Term[multiTerms.size()]),position); multiTerms.clear(); } position += positionIncrement; multiTerms.add(new Term(field, term)); } - if (enablePositionIncrements) { - mpq.add((Term[])multiTerms.toArray(new Term[0]),position); - } else { - mpq.add((Term[])multiTerms.toArray(new Term[0])); - } + mpq.add((Term[])multiTerms.toArray(new Term[multiTerms.size()]),position); return mpq; } } @@ -329,12 +316,8 @@ public class TextField extends FieldType { // safe to ignore, because we know the number of tokens } - if (enablePositionIncrements) { - position += positionIncrement; - pq.add(new Term(field, term),position); - } else { - pq.add(new Term(field, term)); - } + position += positionIncrement; + pq.add(new Term(field, term),position); } return pq; } diff --git a/solr/core/src/java/org/apache/solr/schema/TrieDateField.java b/solr/core/src/java/org/apache/solr/schema/TrieDateField.java index 8e9a358..fe797e5 100755 --- a/solr/core/src/java/org/apache/solr/schema/TrieDateField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieDateField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/TrieDoubleField.java b/solr/core/src/java/org/apache/solr/schema/TrieDoubleField.java index d5dba1a..a8884b4 100755 --- a/solr/core/src/java/org/apache/solr/schema/TrieDoubleField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieDoubleField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/TrieField.java b/solr/core/src/java/org/apache/solr/schema/TrieField.java index 9844bc7..892ce0c 100644 --- a/solr/core/src/java/org/apache/solr/schema/TrieField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/TrieFloatField.java b/solr/core/src/java/org/apache/solr/schema/TrieFloatField.java index 3c83269..2ea1d14 100755 --- a/solr/core/src/java/org/apache/solr/schema/TrieFloatField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieFloatField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/TrieIntField.java b/solr/core/src/java/org/apache/solr/schema/TrieIntField.java index 12a26da..4cc2999 100755 --- a/solr/core/src/java/org/apache/solr/schema/TrieIntField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieIntField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/TrieLongField.java b/solr/core/src/java/org/apache/solr/schema/TrieLongField.java index e0c0f2c..c20f252 100755 --- a/solr/core/src/java/org/apache/solr/schema/TrieLongField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieLongField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/schema/UUIDField.java b/solr/core/src/java/org/apache/solr/schema/UUIDField.java index 53abd04..e65ec3d 100644 --- a/solr/core/src/java/org/apache/solr/schema/UUIDField.java +++ b/solr/core/src/java/org/apache/solr/schema/UUIDField.java @@ -1,6 +1,6 @@ package org.apache.solr.schema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/BitDocSet.java b/solr/core/src/java/org/apache/solr/search/BitDocSet.java index 6339791..2a8aecf 100644 --- a/solr/core/src/java/org/apache/solr/search/BitDocSet.java +++ b/solr/core/src/java/org/apache/solr/search/BitDocSet.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,7 +18,6 @@ package org.apache.solr.search; import org.apache.lucene.index.AtomicReader; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.search.BitsFilteredDocIdSet; import org.apache.lucene.search.DocIdSet; @@ -28,8 +27,6 @@ import org.apache.lucene.util.OpenBitSet; import org.apache.lucene.util.OpenBitSetIterator; import org.apache.lucene.search.DocIdSetIterator; -import java.io.IOException; - /** * BitDocSet represents an unordered set of Lucene Document Ids * using a BitSet. A set bit represents inclusion in the set for that document. @@ -249,7 +246,7 @@ public class BitDocSet extends DocSetBase { return new Filter() { @Override - public DocIdSet getDocIdSet(final AtomicReaderContext context, final Bits acceptDocs) throws IOException { + public DocIdSet getDocIdSet(final AtomicReaderContext context, final Bits acceptDocs) { AtomicReader reader = context.reader(); // all Solr DocSets that are used as filters only include live docs final Bits acceptDocs2 = acceptDocs == null ? null : (reader.getLiveDocs() == acceptDocs ? null : acceptDocs); @@ -264,7 +261,7 @@ public class BitDocSet extends DocSetBase { return BitsFilteredDocIdSet.wrap(new DocIdSet() { @Override - public DocIdSetIterator iterator() throws IOException { + public DocIdSetIterator iterator() { return new DocIdSetIterator() { int pos=base-1; int adjustedDoc=-1; @@ -275,13 +272,13 @@ public class BitDocSet extends DocSetBase { } @Override - public int nextDoc() throws IOException { + public int nextDoc() { pos = bs.nextSetBit(pos+1); return adjustedDoc = (pos>=0 && pos=0 && pos parseQueryFields(final IndexSchema indexSchema, final SolrParams solrParams) + throws ParseException { + Map queryFields = SolrPluginUtils.parseFieldBoosts(solrParams.getParams(DisMaxParams.QF)); + if (queryFields.isEmpty()) { + String df = QueryParsing.getDefaultField(indexSchema, solrParams.get(CommonParams.DF)); + if (df == null) { + throw new ParseException("Neither "+DisMaxParams.QF+", "+CommonParams.DF +", nor the default search field are present."); + } + queryFields.put(df, 1.0f); + } + return queryFields; + } + public DisMaxQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { super(qstr, localParams, params, req); } @@ -81,10 +99,7 @@ public class DisMaxQParser extends QParser { public Query parse() throws ParseException { SolrParams solrParams = SolrParams.wrapDefaults(localParams, params); - queryFields = SolrPluginUtils.parseFieldBoosts(solrParams.getParams(DisMaxParams.QF)); - if (0 == queryFields.size()) { - queryFields.put(req.getSchema().getDefaultSearchFieldName(), 1.0f); - } + queryFields = parseQueryFields(req.getSchema(), solrParams); /* the main query we will execute. we disable the coord because * this query is an artificial construct diff --git a/solr/core/src/java/org/apache/solr/search/DisMaxQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/DisMaxQParserPlugin.java index 717dce7..e8a2a65 100755 --- a/solr/core/src/java/org/apache/solr/search/DisMaxQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/DisMaxQParserPlugin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/DocIterator.java b/solr/core/src/java/org/apache/solr/search/DocIterator.java index 299a626..9c85867 100644 --- a/solr/core/src/java/org/apache/solr/search/DocIterator.java +++ b/solr/core/src/java/org/apache/solr/search/DocIterator.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/DocList.java b/solr/core/src/java/org/apache/solr/search/DocList.java index b344e5f..3fcac21 100644 --- a/solr/core/src/java/org/apache/solr/search/DocList.java +++ b/solr/core/src/java/org/apache/solr/search/DocList.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/DocListAndSet.java b/solr/core/src/java/org/apache/solr/search/DocListAndSet.java index c304c77..6d55ea9 100644 --- a/solr/core/src/java/org/apache/solr/search/DocListAndSet.java +++ b/solr/core/src/java/org/apache/solr/search/DocListAndSet.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/DocSet.java b/solr/core/src/java/org/apache/solr/search/DocSet.java index bc47b38..82b30db 100644 --- a/solr/core/src/java/org/apache/solr/search/DocSet.java +++ b/solr/core/src/java/org/apache/solr/search/DocSet.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/DocSetBase.java b/solr/core/src/java/org/apache/solr/search/DocSetBase.java index 08709a9..c5b7518 100644 --- a/solr/core/src/java/org/apache/solr/search/DocSetBase.java +++ b/solr/core/src/java/org/apache/solr/search/DocSetBase.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -27,8 +27,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.BitsFilteredDocIdSet; import org.apache.lucene.index.AtomicReaderContext; -import java.io.IOException; - /** A base class that may be usefull for implementing DocSets */ abstract class DocSetBase implements DocSet { @@ -140,7 +138,7 @@ abstract class DocSetBase implements DocSet { return new Filter() { @Override - public DocIdSet getDocIdSet(final AtomicReaderContext context, Bits acceptDocs) throws IOException { + public DocIdSet getDocIdSet(final AtomicReaderContext context, Bits acceptDocs) { AtomicReader reader = context.reader(); // all Solr DocSets that are used as filters only include live docs final Bits acceptDocs2 = acceptDocs == null ? null : (reader.getLiveDocs() == acceptDocs ? null : acceptDocs); @@ -155,7 +153,7 @@ abstract class DocSetBase implements DocSet { return BitsFilteredDocIdSet.wrap(new DocIdSet() { @Override - public DocIdSetIterator iterator() throws IOException { + public DocIdSetIterator iterator() { return new DocIdSetIterator() { int pos=base-1; int adjustedDoc=-1; @@ -166,13 +164,13 @@ abstract class DocSetBase implements DocSet { } @Override - public int nextDoc() throws IOException { + public int nextDoc() { pos = bs.nextSetBit(pos+1); return adjustedDoc = (pos>=0 && pos=0 && pos phraseFields = - U.parseFieldBoostsAndSlop(solrParams.getParams(DMP.PF),0); + U.parseFieldBoostsAndSlop(solrParams.getParams(DMP.PF),0,pslop[0]); // Boosted Bi-Term Shingles from the query string List phraseFields2 = - U.parseFieldBoostsAndSlop(solrParams.getParams("pf2"),2); + U.parseFieldBoostsAndSlop(solrParams.getParams(DMP.PF2),2,pslop[2]); // Boosted Tri-Term Shingles from the query string List phraseFields3 = - U.parseFieldBoostsAndSlop(solrParams.getParams("pf3"),3); - + U.parseFieldBoostsAndSlop(solrParams.getParams(DMP.PF3),3,pslop[3]); float tiebreaker = solrParams.getFloat(DisMaxParams.TIE, 0.0f); - int pslop = solrParams.getInt(DisMaxParams.PS, 0); int qslop = solrParams.getInt(DisMaxParams.QS, 0); // remove stopwords from mandatory "matching" component? @@ -229,7 +227,7 @@ class ExtendedDismaxQParser extends QParser { Clause clause = clauses.get(i); String s = clause.raw; // and and or won't be operators at the start or end - if (i>0 && i+10 && i+1 pf = new HashMap(1); pf.put(phraseField.getField(),phraseField.getBoost()); addShingledPhraseQueries(query, normalClauses, pf, - phraseField.getWordGrams(),tiebreaker, slop); + phraseField.getWordGrams(),tiebreaker, phraseField.getSlop()); } } @@ -423,9 +420,8 @@ class ExtendedDismaxQParser extends QParser { * Extracts all the alised fields from the requests and adds them to up * @param up * @param tiebreaker - * @throws ParseException */ - private void addAliasesFromRequest(ExtendedSolrQueryParser up, float tiebreaker) throws ParseException { + private void addAliasesFromRequest(ExtendedSolrQueryParser up, float tiebreaker) { Iterator it = solrParams.getParameterNamesIterator(); while(it.hasNext()) { String param = it.next(); @@ -544,81 +540,81 @@ class ExtendedDismaxQParser extends QParser { } - - public static CharSequence partialEscape(CharSequence s) { - StringBuilder sb = new StringBuilder(); - - int len = s.length(); - for (int i = 0; i < len; i++) { - char c = s.charAt(i); - if (c == ':') { - // look forward to make sure it's something that won't - // cause a parse exception (something that won't be escaped... like - // +,-,:, whitespace - if (i+10) { - char ch = s.charAt(i+1); - if (!(Character.isWhitespace(ch) || ch=='+' || ch=='-' || ch==':')) { - // OK, at this point the chars after the ':' will be fine. - // now look back and try to determine if this is a fieldname - // [+,-]? [letter,_] [letter digit,_,-,.]* - // This won't cover *all* possible lucene fieldnames, but we should - // only pick nice names to begin with - int start, pos; - for (start=i-1; start>=0; start--) { - ch = s.charAt(start); - if (Character.isWhitespace(ch)) break; - } - - // skip whitespace - pos = start+1; - - // skip leading + or - - ch = s.charAt(pos); - if (ch=='+' || ch=='-') { - pos++; - } - - // we don't need to explicitly check for end of string - // since ':' will act as our sentinal - - // first char can't be '-' or '.' - ch = s.charAt(pos++); - if (Character.isJavaIdentifierPart(ch)) { - - for(;;) { - ch = s.charAt(pos++); - if (!(Character.isJavaIdentifierPart(ch) || ch=='-' || ch=='.')) { - break; - } - } - - if (pos<=i) { - // OK, we got to the ':' and everything looked like a valid fieldname, so - // don't escape the ':' - sb.append(':'); - continue; // jump back to start of outer-most loop - } - - } - - - } - } - - // we fell through to here, so we should escape this like other reserved chars. - sb.append('\\'); - } - else if (c == '\\' || c == '!' || c == '(' || c == ')' || - c == '^' || c == '[' || c == ']' || - c == '{' || c == '}' || c == '~' || c == '*' || c == '?' - ) - { - sb.append('\\'); - } - sb.append(c); - } - return sb; - } +// FIXME: Not in use +// public static CharSequence partialEscape(CharSequence s) { +// StringBuilder sb = new StringBuilder(); +// +// int len = s.length(); +// for (int i = 0; i < len; i++) { +// char c = s.charAt(i); +// if (c == ':') { +// // look forward to make sure it's something that won't +// // cause a parse exception (something that won't be escaped... like +// // +,-,:, whitespace +// if (i+10) { +// char ch = s.charAt(i+1); +// if (!(Character.isWhitespace(ch) || ch=='+' || ch=='-' || ch==':')) { +// // OK, at this point the chars after the ':' will be fine. +// // now look back and try to determine if this is a fieldname +// // [+,-]? [letter,_] [letter digit,_,-,.]* +// // This won't cover *all* possible lucene fieldnames, but we should +// // only pick nice names to begin with +// int start, pos; +// for (start=i-1; start>=0; start--) { +// ch = s.charAt(start); +// if (Character.isWhitespace(ch)) break; +// } +// +// // skip whitespace +// pos = start+1; +// +// // skip leading + or - +// ch = s.charAt(pos); +// if (ch=='+' || ch=='-') { +// pos++; +// } +// +// // we don't need to explicitly check for end of string +// // since ':' will act as our sentinal +// +// // first char can't be '-' or '.' +// ch = s.charAt(pos++); +// if (Character.isJavaIdentifierPart(ch)) { +// +// for(;;) { +// ch = s.charAt(pos++); +// if (!(Character.isJavaIdentifierPart(ch) || ch=='-' || ch=='.')) { +// break; +// } +// } +// +// if (pos<=i) { +// // OK, we got to the ':' and everything looked like a valid fieldname, so +// // don't escape the ':' +// sb.append(':'); +// continue; // jump back to start of outer-most loop +// } +// +// } +// +// +// } +// } +// +// // we fell through to here, so we should escape this like other reserved chars. +// sb.append('\\'); +// } +// else if (c == '\\' || c == '!' || c == '(' || c == ')' || +// c == '^' || c == '[' || c == ']' || +// c == '{' || c == '}' || c == '~' || c == '*' || c == '?' +// ) +// { +// sb.append('\\'); +// } +// sb.append(c); +// } +// return sb; +// } static class Clause { @@ -729,6 +725,10 @@ class ExtendedDismaxQParser extends QParser { case '"': case '+': case '-': + case '\\': + case '|': + case '&': + case '/': clause.hasSpecialSyntax = true; sb.append('\\'); } @@ -1154,7 +1154,7 @@ class ExtendedDismaxQParser extends QParser { return lst; } - private Query getQuery() throws ParseException { + private Query getQuery() { try { switch (type) { diff --git a/solr/core/src/java/org/apache/solr/search/ExtendedQuery.java b/solr/core/src/java/org/apache/solr/search/ExtendedQuery.java index 92a217c..654d036 100644 --- a/solr/core/src/java/org/apache/solr/search/ExtendedQuery.java +++ b/solr/core/src/java/org/apache/solr/search/ExtendedQuery.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/ExtendedQueryBase.java b/solr/core/src/java/org/apache/solr/search/ExtendedQueryBase.java index 48e31ee..08b81c8 100644 --- a/solr/core/src/java/org/apache/solr/search/ExtendedQueryBase.java +++ b/solr/core/src/java/org/apache/solr/search/ExtendedQueryBase.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/FastLRUCache.java b/solr/core/src/java/org/apache/solr/search/FastLRUCache.java index 5e0e783..c4732f6 100644 --- a/solr/core/src/java/org/apache/solr/search/FastLRUCache.java +++ b/solr/core/src/java/org/apache/solr/search/FastLRUCache.java @@ -1,5 +1,5 @@ package org.apache.solr.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,9 +20,7 @@ import org.apache.solr.common.SolrException; import org.apache.solr.util.ConcurrentLRUCache; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; -import org.apache.solr.core.SolrCore; -import java.io.IOException; import java.io.Serializable; import java.util.List; import java.util.Map; @@ -136,7 +134,7 @@ public class FastLRUCache extends SolrCacheBase implements SolrCache { cache.setAlive(state == State.LIVE); } - public void warm(SolrIndexSearcher searcher, SolrCache old) throws IOException { + public void warm(SolrIndexSearcher searcher, SolrCache old) { if (regenerator == null) return; long warmingStartTime = System.currentTimeMillis(); FastLRUCache other = (FastLRUCache) old; diff --git a/solr/core/src/java/org/apache/solr/search/FieldParams.java b/solr/core/src/java/org/apache/solr/search/FieldParams.java index aaf9c29..728369c 100644 --- a/solr/core/src/java/org/apache/solr/search/FieldParams.java +++ b/solr/core/src/java/org/apache/solr/search/FieldParams.java @@ -1,5 +1,5 @@ package org.apache.solr.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,7 +21,7 @@ package org.apache.solr.search; **/ public class FieldParams { private final int wordGrams; // make bigrams if 2, trigrams if 3, or all if 0 - private final int slop; // null defaults to ps parameter + private final int slop; private final float boost; private final String field; public FieldParams(String field, int wordGrams, int slop, float boost) { diff --git a/solr/core/src/java/org/apache/solr/search/FieldQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/FieldQParserPlugin.java index 4f01081..0271680 100644 --- a/solr/core/src/java/org/apache/solr/search/FieldQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/FieldQParserPlugin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -16,8 +16,7 @@ */ package org.apache.solr.search; -import org.apache.lucene.queryparser.classic.ParseException; -import org.apache.lucene.search.*; +import org.apache.lucene.search.Query; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.request.SolrQueryRequest; @@ -42,7 +41,7 @@ public class FieldQParserPlugin extends QParserPlugin { public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new QParser(qstr, localParams, params, req) { @Override - public Query parse() throws ParseException { + public Query parse() { String field = localParams.get(QueryParsing.F); String queryText = localParams.get(QueryParsing.V); SchemaField sf = req.getSchema().getField(field); diff --git a/solr/core/src/java/org/apache/solr/search/FunctionQParser.java b/solr/core/src/java/org/apache/solr/search/FunctionQParser.java index 58a766b..1c68e3f 100755 --- a/solr/core/src/java/org/apache/solr/search/FunctionQParser.java +++ b/solr/core/src/java/org/apache/solr/search/FunctionQParser.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/FunctionQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/FunctionQParserPlugin.java index 80cd846..f88b409 100644 --- a/solr/core/src/java/org/apache/solr/search/FunctionQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/FunctionQParserPlugin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java index a509592..0e89772 100755 --- a/solr/core/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/FunctionRangeQuery.java b/solr/core/src/java/org/apache/solr/search/FunctionRangeQuery.java index 44d0356..4e913dd 100644 --- a/solr/core/src/java/org/apache/solr/search/FunctionRangeQuery.java +++ b/solr/core/src/java/org/apache/solr/search/FunctionRangeQuery.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/Grouping.java b/solr/core/src/java/org/apache/solr/search/Grouping.java index 54ec934..0af470e 100755 --- a/solr/core/src/java/org/apache/solr/search/Grouping.java +++ b/solr/core/src/java/org/apache/solr/search/Grouping.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -858,7 +858,7 @@ public class Grouping { protected void finish() throws IOException { TopDocsCollector topDocsCollector = (TopDocsCollector) collector.getDelegate(); TopDocs topDocs = topDocsCollector.topDocs(); - GroupDocs groupDocs = new GroupDocs(topDocs.getMaxScore(), topDocs.totalHits, topDocs.scoreDocs, query.toString(), null); + GroupDocs groupDocs = new GroupDocs(Float.NaN, topDocs.getMaxScore(), topDocs.totalHits, topDocs.scoreDocs, query.toString(), null); if (main) { mainResult = getDocList(groupDocs); } else { diff --git a/solr/core/src/java/org/apache/solr/search/HashDocSet.java b/solr/core/src/java/org/apache/solr/search/HashDocSet.java index 691211c..21eb417 100644 --- a/solr/core/src/java/org/apache/solr/search/HashDocSet.java +++ b/solr/core/src/java/org/apache/solr/search/HashDocSet.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java index b7d6ea4..b594f03 100644 --- a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -136,7 +136,7 @@ class JoinQuery extends Query { private float queryWeight; ResponseBuilder rb; - public JoinQueryWeight(SolrIndexSearcher searcher) throws IOException { + public JoinQueryWeight(SolrIndexSearcher searcher) { this.fromSearcher = searcher; SolrRequestInfo info = SolrRequestInfo.getRequestInfo(); if (info != null) { @@ -176,7 +176,7 @@ class JoinQuery extends Query { final RefCounted ref = fromRef; info.addCloseHook(new Closeable() { @Override - public void close() throws IOException { + public void close() { ref.decref(); } }); @@ -184,7 +184,7 @@ class JoinQuery extends Query { info.addCloseHook(new Closeable() { @Override - public void close() throws IOException { + public void close() { fromCore.close(); } }); @@ -559,7 +559,7 @@ class JoinQuery extends Query { @Override public boolean equals(Object o) { - if (getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; JoinQuery other = (JoinQuery)o; return this.fromField.equals(other.fromField) && this.toField.equals(other.toField) @@ -572,7 +572,9 @@ class JoinQuery extends Query { @Override public int hashCode() { - int h = q.hashCode() + (int)fromCoreOpenTime; + int h = super.hashCode(); + h = h * 31 + q.hashCode(); + h = h * 31 + (int)fromCoreOpenTime; h = h * 31 + fromField.hashCode(); h = h * 31 + toField.hashCode(); return h; diff --git a/solr/core/src/java/org/apache/solr/search/LFUCache.java b/solr/core/src/java/org/apache/solr/search/LFUCache.java index 63754a6..521115b 100644 --- a/solr/core/src/java/org/apache/solr/search/LFUCache.java +++ b/solr/core/src/java/org/apache/solr/search/LFUCache.java @@ -1,5 +1,5 @@ package org.apache.solr.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,7 +22,6 @@ import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.SolrCore; import org.apache.solr.util.ConcurrentLFUCache; -import java.io.IOException; import java.io.Serializable; import java.net.URL; import java.util.List; @@ -155,7 +154,7 @@ public class LFUCache implements SolrCache { return state; } - public void warm(SolrIndexSearcher searcher, SolrCache old) throws IOException { + public void warm(SolrIndexSearcher searcher, SolrCache old) { if (regenerator == null) return; long warmingStartTime = System.currentTimeMillis(); LFUCache other = (LFUCache) old; diff --git a/solr/core/src/java/org/apache/solr/search/LRUCache.java b/solr/core/src/java/org/apache/solr/search/LRUCache.java index bb3d6d4..8c631c0 100644 --- a/solr/core/src/java/org/apache/solr/search/LRUCache.java +++ b/solr/core/src/java/org/apache/solr/search/LRUCache.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,7 +17,6 @@ package org.apache.solr.search; -import java.io.IOException; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; @@ -144,7 +143,7 @@ public class LRUCache extends SolrCacheBase implements SolrCache { } } - public void warm(SolrIndexSearcher searcher, SolrCache old) throws IOException { + public void warm(SolrIndexSearcher searcher, SolrCache old) { if (regenerator==null) return; long warmingStartTime = System.currentTimeMillis(); LRUCache other = (LRUCache)old; diff --git a/solr/core/src/java/org/apache/solr/search/LuceneQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/LuceneQParserPlugin.java index 3d6b500..6912c4d 100755 --- a/solr/core/src/java/org/apache/solr/search/LuceneQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/LuceneQParserPlugin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/LuceneQueryOptimizer.java b/solr/core/src/java/org/apache/solr/search/LuceneQueryOptimizer.java index be43794..2e48ed9 100644 --- a/solr/core/src/java/org/apache/solr/search/LuceneQueryOptimizer.java +++ b/solr/core/src/java/org/apache/solr/search/LuceneQueryOptimizer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java b/solr/core/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java index 65b7ad5..565c937 100644 --- a/solr/core/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java +++ b/solr/core/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/NestedQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/NestedQParserPlugin.java index ad3bf5c..d7fdff8 100755 --- a/solr/core/src/java/org/apache/solr/search/NestedQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/NestedQParserPlugin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/OldLuceneQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/OldLuceneQParserPlugin.java index 1a5e4eb..fb14aad 100755 --- a/solr/core/src/java/org/apache/solr/search/OldLuceneQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/OldLuceneQParserPlugin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/PostFilter.java b/solr/core/src/java/org/apache/solr/search/PostFilter.java index a2d31c9..8fc28a6 100644 --- a/solr/core/src/java/org/apache/solr/search/PostFilter.java +++ b/solr/core/src/java/org/apache/solr/search/PostFilter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/PrefixQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/PrefixQParserPlugin.java index 6258209..7745b4e 100755 --- a/solr/core/src/java/org/apache/solr/search/PrefixQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/PrefixQParserPlugin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,7 +17,6 @@ package org.apache.solr.search; import org.apache.lucene.index.Term; -import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.solr.common.params.SolrParams; @@ -40,7 +39,7 @@ public class PrefixQParserPlugin extends QParserPlugin { public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new QParser(qstr, localParams, params, req) { @Override - public Query parse() throws ParseException { + public Query parse() { return new PrefixQuery(new Term(localParams.get(QueryParsing.F), localParams.get(QueryParsing.V))); } }; diff --git a/solr/core/src/java/org/apache/solr/search/QParser.java b/solr/core/src/java/org/apache/solr/search/QParser.java index f9c419b..b5cf1ec 100755 --- a/solr/core/src/java/org/apache/solr/search/QParser.java +++ b/solr/core/src/java/org/apache/solr/search/QParser.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -300,15 +300,15 @@ public abstract class QParser { } /** Create a QParser to parse qstr, - * assuming that the default query type is defaultType. - * The query type may be overridden by local parameters in the query - * string itself. For example if defaultType="dismax" + * assuming that the default query parser is defaultParser. + * The query parser may be overridden by local parameters in the query + * string itself. For example if defaultParser="dismax" * and qstr=foo, then the dismax query parser will be used * to parse and construct the query object. However * if qstr={!prefix f=myfield}foo * then the prefix query parser will be used. */ - public static QParser getParser(String qstr, String defaultType, SolrQueryRequest req) throws ParseException { + public static QParser getParser(String qstr, String defaultParser, SolrQueryRequest req) throws ParseException { // SolrParams localParams = QueryParsing.getLocalParams(qstr, req.getParams()); String stringIncludingLocalParams = qstr; @@ -335,18 +335,18 @@ public abstract class QParser { } - String type; + String parserName; if (localParams == null) { - type = defaultType; + parserName = defaultParser; } else { - type = localParams.get(QueryParsing.TYPE,defaultType); + parserName = localParams.get(QueryParsing.TYPE,defaultParser); qstr = localParams.get("v"); } - type = type==null ? QParserPlugin.DEFAULT_QTYPE : type; + parserName = parserName==null ? QParserPlugin.DEFAULT_QTYPE : parserName; - QParserPlugin qplug = req.getCore().getQueryPlugin(type); + QParserPlugin qplug = req.getCore().getQueryPlugin(parserName); QParser parser = qplug.createParser(qstr, localParams, req.getParams(), req); parser.stringIncludingLocalParams = stringIncludingLocalParams; diff --git a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java index 6032fe1..574d04c 100755 --- a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/QueryParsing.java b/solr/core/src/java/org/apache/solr/search/QueryParsing.java index 7cafe79..2d149ca 100644 --- a/solr/core/src/java/org/apache/solr/search/QueryParsing.java +++ b/solr/core/src/java/org/apache/solr/search/QueryParsing.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -39,8 +39,6 @@ import org.apache.lucene.util.CharsRef; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.SolrParams; -import org.apache.solr.core.SolrCore; -import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.schema.FieldType; import org.apache.solr.schema.IndexSchema; @@ -60,8 +58,8 @@ public class QueryParsing { public static final String OP = "q.op"; // the SolrParam used to override the QueryParser "default operator" public static final String V = "v"; // value of this parameter public static final String F = "f"; // field that a query or command pertains to - public static final String TYPE = "type";// type of this query or command - public static final String DEFTYPE = "defType"; // default type for any direct subqueries + public static final String TYPE = "type";// parser for this query or command + public static final String DEFTYPE = "defType"; // default parser for any direct subqueries public static final String LOCALPARAM_START = "{!"; public static final char LOCALPARAM_END = '}'; public static final String DOCID = "_docid_"; @@ -72,7 +70,7 @@ public class QueryParsing { /** - * Returns the "prefered" default operator for use by Query Parsers, + * Returns the "preferred" default operator for use by Query Parsers, * based on the settings in the IndexSchema which may be overridden using * an optional String override value. * @@ -86,6 +84,15 @@ public class QueryParsing { return "AND".equals(val) ? QueryParser.Operator.AND : QueryParser.Operator.OR; } + /** + * Returns the effective default field based on the 'df' param or + * hardcoded schema default. May be null if either exists specified. + * @see org.apache.solr.common.params.CommonParams#DF + * @see org.apache.solr.schema.IndexSchema#getDefaultSearchFieldName + */ + public static String getDefaultField(final IndexSchema s, final String df) { + return df != null ? df : s.getDefaultSearchFieldName(); + } // note to self: something needs to detect infinite recursion when parsing queries public static int parseLocalParams(String txt, int start, Map target, SolrParams params) throws ParseException { @@ -630,7 +637,7 @@ public class QueryParsing { } } - float getFloat() throws ParseException { + float getFloat() { eatws(); char[] arr = new char[end - pos]; int i; @@ -650,7 +657,7 @@ public class QueryParsing { return Float.parseFloat(new String(arr, 0, i)); } - Number getNumber() throws ParseException { + Number getNumber() { eatws(); int start = pos; boolean flt = false; @@ -675,7 +682,7 @@ public class QueryParsing { } } - double getDouble() throws ParseException { + double getDouble() { eatws(); char[] arr = new char[end - pos]; int i; @@ -695,7 +702,7 @@ public class QueryParsing { return Double.parseDouble(new String(arr, 0, i)); } - int getInt() throws ParseException { + int getInt() { eatws(); char[] arr = new char[end - pos]; int i; diff --git a/solr/core/src/java/org/apache/solr/search/QueryResultKey.java b/solr/core/src/java/org/apache/solr/search/QueryResultKey.java index 3aa743a..003aae4 100644 --- a/solr/core/src/java/org/apache/solr/search/QueryResultKey.java +++ b/solr/core/src/java/org/apache/solr/search/QueryResultKey.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,7 +20,6 @@ package org.apache.solr.search; import org.apache.lucene.search.Query; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; -import java.io.IOException; import java.util.List; /** A hash key encapsulating a query, a list of filters, and a sort @@ -38,7 +37,7 @@ public final class QueryResultKey { private static SortField[] defaultSort = new SortField[0]; - public QueryResultKey(Query query, List filters, Sort sort, int nc_flags) throws IOException { + public QueryResultKey(Query query, List filters, Sort sort, int nc_flags) { this.query = query; this.sort = sort; this.filters = filters; diff --git a/solr/core/src/java/org/apache/solr/search/QueryUtils.java b/solr/core/src/java/org/apache/solr/search/QueryUtils.java index 2eb6011..2df8f1f 100755 --- a/solr/core/src/java/org/apache/solr/search/QueryUtils.java +++ b/solr/core/src/java/org/apache/solr/search/QueryUtils.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/RawQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/RawQParserPlugin.java index 7c7d894..f4289a7 100644 --- a/solr/core/src/java/org/apache/solr/search/RawQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/RawQParserPlugin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,7 +17,6 @@ package org.apache.solr.search; import org.apache.lucene.index.Term; -import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.solr.common.params.SolrParams; @@ -44,7 +43,7 @@ public class RawQParserPlugin extends QParserPlugin { public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new QParser(qstr, localParams, params, req) { @Override - public Query parse() throws ParseException { + public Query parse() { return new TermQuery(new Term(localParams.get(QueryParsing.F), localParams.get(QueryParsing.V))); } }; diff --git a/solr/core/src/java/org/apache/solr/search/ReturnFields.java b/solr/core/src/java/org/apache/solr/search/ReturnFields.java index 923f5cc..dcc4420 100644 --- a/solr/core/src/java/org/apache/solr/search/ReturnFields.java +++ b/solr/core/src/java/org/apache/solr/search/ReturnFields.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -155,7 +155,7 @@ public class ReturnFields // like getId, but also accepts dashes for legacy fields - String getFieldName(QueryParsing.StrParser sp) throws ParseException { + String getFieldName(QueryParsing.StrParser sp) { sp.eatws(); int id_start = sp.pos; char ch; diff --git a/solr/core/src/java/org/apache/solr/search/SolrCache.java b/solr/core/src/java/org/apache/solr/search/SolrCache.java index 2a87043..caf636d 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrCache.java +++ b/solr/core/src/java/org/apache/solr/search/SolrCache.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,7 +22,6 @@ import org.apache.solr.core.SolrInfoMBean; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; /** @@ -125,7 +124,7 @@ public interface SolrCache extends SolrInfoMBean { * Warm this cache associated with searcher using the old * cache object. this and old will have the same concrete type. */ - void warm(SolrIndexSearcher searcher, SolrCache old) throws IOException; + void warm(SolrIndexSearcher searcher, SolrCache old); // Q: an alternative to passing the searcher here would be to pass it in // init and have the cache implementation save it. diff --git a/solr/core/src/java/org/apache/solr/search/SolrCacheBase.java b/solr/core/src/java/org/apache/solr/search/SolrCacheBase.java index 5eb1d96..8870f9b 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrCacheBase.java +++ b/solr/core/src/java/org/apache/solr/search/SolrCacheBase.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java b/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java index 9eea652..edc4617 100755 --- a/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java +++ b/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java @@ -12,7 +12,7 @@ import java.io.IOException; import java.util.Set; import java.util.Map; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java b/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java index 56ecee3..7c2fe86 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java +++ b/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/SolrFilter.java b/solr/core/src/java/org/apache/solr/search/SolrFilter.java index b26c0b9..989af96 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrFilter.java +++ b/solr/core/src/java/org/apache/solr/search/SolrFilter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index 7a151e6..109eb26 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -34,14 +34,12 @@ import org.apache.lucene.document.LongField; import org.apache.lucene.document.StoredField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.*; -import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.search.*; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.OpenBitSet; -import org.apache.lucene.util.ReaderUtil; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; @@ -461,7 +459,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn @Override public void intField(FieldInfo fieldInfo, int value) { - FieldType ft = new FieldType(IntField.TYPE); + FieldType ft = new FieldType(IntField.TYPE_NOT_STORED); ft.setStored(true); ft.setIndexed(fieldInfo.isIndexed()); doc.add(new IntField(fieldInfo.name, value, ft)); @@ -469,7 +467,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn @Override public void longField(FieldInfo fieldInfo, long value) { - FieldType ft = new FieldType(LongField.TYPE); + FieldType ft = new FieldType(LongField.TYPE_NOT_STORED); ft.setStored(true); ft.setIndexed(fieldInfo.isIndexed()); doc.add(new LongField(fieldInfo.name, value, ft)); @@ -477,7 +475,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn @Override public void floatField(FieldInfo fieldInfo, float value) { - FieldType ft = new FieldType(FloatField.TYPE); + FieldType ft = new FieldType(FloatField.TYPE_NOT_STORED); ft.setStored(true); ft.setIndexed(fieldInfo.isIndexed()); doc.add(new FloatField(fieldInfo.name, value, ft)); @@ -485,7 +483,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn @Override public void doubleField(FieldInfo fieldInfo, double value) { - FieldType ft = new FieldType(DoubleField.TYPE); + FieldType ft = new FieldType(DoubleField.TYPE_NOT_STORED); ft.setStored(true); ft.setIndexed(fieldInfo.isIndexed()); doc.add(new DoubleField(fieldInfo.name, value, ft)); @@ -604,11 +602,9 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn */ public long lookupId(BytesRef idBytes) throws IOException { String field = schema.getUniqueKeyField().getName(); - final AtomicReaderContext[] leaves = leafContexts; - - for (int i=0; i topscore[0]) topscore[0]=score; } @Override - public void setNextReader(AtomicReaderContext context) throws IOException { + public void setNextReader(AtomicReaderContext context) { } @Override public boolean acceptsDocsOutOfOrder() { @@ -1452,7 +1441,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn collector = setCollector = new DocSetDelegateCollector(smallSetSize, maxDoc, new Collector() { Scorer scorer; @Override - public void setScorer(Scorer scorer) throws IOException { + public void setScorer(Scorer scorer) { this.scorer = scorer; } @Override @@ -1461,7 +1450,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn if (score > topscore[0]) topscore[0]=score; } @Override - public void setNextReader(AtomicReaderContext context) throws IOException { + public void setNextReader(AtomicReaderContext context) { } @Override public boolean acceptsDocsOutOfOrder() { @@ -1801,7 +1790,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn while (iter.hasNext()) { int doc = iter.nextDoc(); while (doc>=end) { - AtomicReaderContext leaf = leafContexts[readerIndex++]; + AtomicReaderContext leaf = leafContexts.get(readerIndex++); base = leaf.docBase; end = base + leaf.reader().maxDoc(); topCollector.setNextReader(leaf); diff --git a/solr/core/src/java/org/apache/solr/search/SolrQueryParser.java b/solr/core/src/java/org/apache/solr/search/SolrQueryParser.java index e30579e..eb32a3a 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrQueryParser.java +++ b/solr/core/src/java/org/apache/solr/search/SolrQueryParser.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -135,7 +135,7 @@ public class SolrQueryParser extends QueryParser { if (field == null && defaultField == null) { throw new SolrException (SolrException.ErrorCode.BAD_REQUEST, - "no field name specified in query and no defaultSearchField defined in schema.xml"); + "no field name specified in query and no default specified via 'df' param"); } } @@ -194,6 +194,7 @@ public class SolrQueryParser extends QueryParser { } @Override protected Query getWildcardQuery(String field, String termStr) throws ParseException { + checkNullField(field); // *:* -> MatchAllDocsQuery if ("*".equals(field) && "*".equals(termStr)) { return newMatchAllDocsQuery(); diff --git a/solr/core/src/java/org/apache/solr/search/SolrSortField.java b/solr/core/src/java/org/apache/solr/search/SolrSortField.java index 8b21e43..0e58c69 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrSortField.java +++ b/solr/core/src/java/org/apache/solr/search/SolrSortField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/SortSpec.java b/solr/core/src/java/org/apache/solr/search/SortSpec.java index f2c8004..4573bf6 100644 --- a/solr/core/src/java/org/apache/solr/search/SortSpec.java +++ b/solr/core/src/java/org/apache/solr/search/SortSpec.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java b/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java index bac0b8d..754fd14 100755 --- a/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java +++ b/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -24,11 +24,8 @@ import org.apache.lucene.search.BitsFilteredDocIdSet; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Filter; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.AtomicReaderContext; -import java.io.IOException; - /** * SortedIntDocSet represents a sorted set of Lucene Document Ids. */ @@ -658,7 +655,7 @@ public class SortedIntDocSet extends DocSetBase { int lastEndIdx = 0; @Override - public DocIdSet getDocIdSet(final AtomicReaderContext context, final Bits acceptDocs) throws IOException { + public DocIdSet getDocIdSet(final AtomicReaderContext context, final Bits acceptDocs) { AtomicReader reader = context.reader(); // all Solr DocSets that are used as filters only include live docs final Bits acceptDocs2 = acceptDocs == null ? null : (reader.getLiveDocs() == acceptDocs ? null : acceptDocs); @@ -695,7 +692,7 @@ public class SortedIntDocSet extends DocSetBase { return BitsFilteredDocIdSet.wrap(new DocIdSet() { @Override - public DocIdSetIterator iterator() throws IOException { + public DocIdSetIterator iterator() { return new DocIdSetIterator() { int idx = startIdx; int adjustedDoc = -1; @@ -706,12 +703,12 @@ public class SortedIntDocSet extends DocSetBase { } @Override - public int nextDoc() throws IOException { + public int nextDoc() { return adjustedDoc = (idx > endIdx) ? NO_MORE_DOCS : (docs[idx++] - base); } @Override - public int advance(int target) throws IOException { + public int advance(int target) { if (idx > endIdx || target==NO_MORE_DOCS) return adjustedDoc=NO_MORE_DOCS; target += base; @@ -757,7 +754,7 @@ public class SortedIntDocSet extends DocSetBase { } @Override - public Bits bits() throws IOException { + public Bits bits() { // random access is expensive for this set return null; } diff --git a/solr/core/src/java/org/apache/solr/search/Sorting.java b/solr/core/src/java/org/apache/solr/search/Sorting.java index 9b9a97c..535309b 100644 --- a/solr/core/src/java/org/apache/solr/search/Sorting.java +++ b/solr/core/src/java/org/apache/solr/search/Sorting.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/SpatialBoxQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/SpatialBoxQParserPlugin.java index 37065c5..5638bba 100755 --- a/solr/core/src/java/org/apache/solr/search/SpatialBoxQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/SpatialBoxQParserPlugin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/SpatialFilterQParser.java b/solr/core/src/java/org/apache/solr/search/SpatialFilterQParser.java index eabacac..dea818c 100644 --- a/solr/core/src/java/org/apache/solr/search/SpatialFilterQParser.java +++ b/solr/core/src/java/org/apache/solr/search/SpatialFilterQParser.java @@ -1,5 +1,5 @@ package org.apache.solr.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/SpatialFilterQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/SpatialFilterQParserPlugin.java index 064a4be..7ce1382 100644 --- a/solr/core/src/java/org/apache/solr/search/SpatialFilterQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/SpatialFilterQParserPlugin.java @@ -1,5 +1,5 @@ package org.apache.solr.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/SpatialOptions.java b/solr/core/src/java/org/apache/solr/search/SpatialOptions.java index 21fa225..0b1d058 100644 --- a/solr/core/src/java/org/apache/solr/search/SpatialOptions.java +++ b/solr/core/src/java/org/apache/solr/search/SpatialOptions.java @@ -1,5 +1,5 @@ package org.apache.solr.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/SurroundQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/SurroundQParserPlugin.java index af3417e..d7ba2a6 100644 --- a/solr/core/src/java/org/apache/solr/search/SurroundQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/SurroundQParserPlugin.java @@ -1,6 +1,6 @@ package org.apache.solr.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -103,12 +103,8 @@ class SurroundQParser extends QParser { // so what do we do with the SrndQuery ?? // processing based on example in LIA Ch 9 - String defaultField = getParam(CommonParams.DF); - if (defaultField == null) { - defaultField = getReq().getSchema().getDefaultSearchFieldName(); - } - BasicQueryFactory bqFactory = new BasicQueryFactory(this.maxBasicQueries); + String defaultField = QueryParsing.getDefaultField(getReq().getSchema(),getParam(CommonParams.DF)); Query lquery = sq.makeLuceneQueryField(defaultField, bqFactory); return lquery; } diff --git a/solr/core/src/java/org/apache/solr/search/TermQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/TermQParserPlugin.java index 1b9e021..381e2e2 100644 --- a/solr/core/src/java/org/apache/solr/search/TermQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/TermQParserPlugin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,7 +17,6 @@ package org.apache.solr.search; import org.apache.lucene.index.Term; -import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; @@ -51,7 +50,7 @@ public class TermQParserPlugin extends QParserPlugin { public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new QParser(qstr, localParams, params, req) { @Override - public Query parse() throws ParseException { + public Query parse() { String fname = localParams.get(QueryParsing.F); FieldType ft = req.getSchema().getFieldTypeNoEx(fname); String val = localParams.get(QueryParsing.V); diff --git a/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java b/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java index 9b35f28..74ee847 100755 --- a/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java +++ b/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -104,7 +104,7 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin { addParser("literal", new ValueSourceParser() { @Override public ValueSource parse(FunctionQParser fp) throws ParseException { - return new LiteralValueSource(fp.getString()); + return new LiteralValueSource(fp.parseArg()); } }); addParser("rord", new ValueSourceParser() { @@ -529,13 +529,13 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin { addParser("pi", new ValueSourceParser() { @Override - public ValueSource parse(FunctionQParser fp) throws ParseException { + public ValueSource parse(FunctionQParser fp) { return new DoubleConstValueSource(Math.PI); } }); addParser("e", new ValueSourceParser() { @Override - public ValueSource parse(FunctionQParser fp) throws ParseException { + public ValueSource parse(FunctionQParser fp) { return new DoubleConstValueSource(Math.E); } }); @@ -601,28 +601,28 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin { addParser("maxdoc", new ValueSourceParser() { @Override - public ValueSource parse(FunctionQParser fp) throws ParseException { + public ValueSource parse(FunctionQParser fp) { return new MaxDocValueSource(); } }); addParser("numdocs", new ValueSourceParser() { @Override - public ValueSource parse(FunctionQParser fp) throws ParseException { + public ValueSource parse(FunctionQParser fp) { return new NumDocsValueSource(); } }); addParser("true", new ValueSourceParser() { @Override - public ValueSource parse(FunctionQParser fp) throws ParseException { + public ValueSource parse(FunctionQParser fp) { return new BoolConstValueSource(true); } }); addParser("false", new ValueSourceParser() { @Override - public ValueSource parse(FunctionQParser fp) throws ParseException { + public ValueSource parse(FunctionQParser fp) { return new BoolConstValueSource(false); } }); diff --git a/solr/core/src/java/org/apache/solr/search/WrappedQuery.java b/solr/core/src/java/org/apache/solr/search/WrappedQuery.java index ca186d1..462b258 100644 --- a/solr/core/src/java/org/apache/solr/search/WrappedQuery.java +++ b/solr/core/src/java/org/apache/solr/search/WrappedQuery.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java b/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java index fba2ca3..b5801a3 100755 --- a/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java +++ b/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -31,6 +31,7 @@ import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.IndexReaderContext; +import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.queries.function.FunctionValues; @@ -38,7 +39,6 @@ import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.docvalues.FloatDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.ReaderUtil; import org.apache.solr.core.SolrCore; import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.handler.RequestHandlerUtils; diff --git a/solr/core/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java b/solr/core/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java index e4b58bd..e33d260 100755 --- a/solr/core/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java +++ b/solr/core/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -81,7 +81,7 @@ public class ValueSourceRangeFilter extends SolrFilter { return valueSource.getValues(context, readerContext).getRangeScorer(readerContext.reader(), lowerVal, upperVal, includeLower, includeUpper); } @Override - public Bits bits() throws IOException { + public Bits bits() { return null; // don't use random access } }, acceptDocs); diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java index 9357294..a7e1c3f 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java @@ -1,5 +1,5 @@ package org.apache.solr.search.function.distance; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java index 32a1bb9..148360c 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java @@ -1,5 +1,5 @@ package org.apache.solr.search.function.distance; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java index 5e22eb3..a9cd8ba 100755 --- a/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java @@ -1,5 +1,5 @@ package org.apache.solr.search.function.distance; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java index 2cc8d06..3f99e16 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java @@ -1,5 +1,5 @@ package org.apache.solr.search.function.distance; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/SquaredEuclideanFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/SquaredEuclideanFunction.java index c82b05a..43a870a 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/SquaredEuclideanFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/SquaredEuclideanFunction.java @@ -1,5 +1,5 @@ package org.apache.solr.search.function.distance; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/StringDistanceFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/StringDistanceFunction.java index f6168df..ef5dc40 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/StringDistanceFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/StringDistanceFunction.java @@ -1,6 +1,6 @@ package org.apache.solr.search.function.distance; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/VectorDistanceFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/VectorDistanceFunction.java index 70cf81f..9fc5203 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/VectorDistanceFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/VectorDistanceFunction.java @@ -1,5 +1,5 @@ package org.apache.solr.search.function.distance; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/grouping/collector/FilterCollector.java b/solr/core/src/java/org/apache/solr/search/grouping/collector/FilterCollector.java index 6114b81..a200fe6 100644 --- a/solr/core/src/java/org/apache/solr/search/grouping/collector/FilterCollector.java +++ b/solr/core/src/java/org/apache/solr/search/grouping/collector/FilterCollector.java @@ -36,7 +36,7 @@ public class FilterCollector extends Collector { private int docBase; private int matches; - public FilterCollector(DocSet filter, Collector delegate) throws IOException { + public FilterCollector(DocSet filter, Collector delegate) { this.filter = filter; this.delegate = delegate; } diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/TopGroupsFieldCommand.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/TopGroupsFieldCommand.java index 66dcabb..61736e5 100644 --- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/TopGroupsFieldCommand.java +++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/TopGroupsFieldCommand.java @@ -135,7 +135,7 @@ public class TopGroupsFieldCommand implements Command> { @SuppressWarnings("unchecked") public TopGroups result() { if (firstPhaseGroups.isEmpty()) { - return new TopGroups(groupSort.getSort(), sortWithinGroup.getSort(), 0, 0, new GroupDocs[0]); + return new TopGroups(groupSort.getSort(), sortWithinGroup.getSort(), 0, 0, new GroupDocs[0], Float.NaN); } return secondPassCollector.getTopGroups(0); diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java index 1b845b4..cc35b62 100644 --- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java +++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java @@ -97,7 +97,7 @@ public class TopGroupsShardResponseProcessor implements ShardResponseProcessor { } TopGroups[] topGroupsArr = new TopGroups[topGroups.size()]; - rb.mergedTopGroups.put(groupField, TopGroups.merge(topGroups.toArray(topGroupsArr), groupSort, sortWithinGroup, groupOffsetDefault, docsPerGroupDefault)); + rb.mergedTopGroups.put(groupField, TopGroups.merge(topGroups.toArray(topGroupsArr), groupSort, sortWithinGroup, groupOffsetDefault, docsPerGroupDefault, TopGroups.ScoreMergeMode.None)); } for (String query : commandTopDocs.keySet()) { diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/SearchGroupsResultTransformer.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/SearchGroupsResultTransformer.java index 09bffec..d1e1da2 100644 --- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/SearchGroupsResultTransformer.java +++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/SearchGroupsResultTransformer.java @@ -74,7 +74,7 @@ public class SearchGroupsResultTransformer implements ShardResultTransformer>>> transformToNative(NamedList shardResponse, Sort groupSort, Sort sortWithinGroup, String shard) throws IOException { + public Map>>> transformToNative(NamedList shardResponse, Sort groupSort, Sort sortWithinGroup, String shard) { Map>>> result = new HashMap>>>(); for (Map.Entry command : shardResponse) { List> searchGroups = new ArrayList>(); diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/ShardResultTransformer.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/ShardResultTransformer.java index eb4d5c4..79998b3 100644 --- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/ShardResultTransformer.java +++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/ShardResultTransformer.java @@ -48,8 +48,7 @@ public interface ShardResultTransformer { * @param sortWithinGroup The sort inside a group * @param shard The shard address where the response originated from * @return native structure of the data - * @throws IOException If I/O related errors occur during transforming */ - R transformToNative(NamedList shardResponse, Sort groupSort, Sort sortWithinGroup, String shard) throws IOException; + R transformToNative(NamedList shardResponse, Sort groupSort, Sort sortWithinGroup, String shard); } diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java index 29de7dd..141f152 100644 --- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java +++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java @@ -142,13 +142,13 @@ public class TopGroupsResultTransformer implements ShardResultTransformer(maxScore, totalGroupHits, scoreDocs, groupValueRef, null)); + groupDocs.add(new GroupDocs(Float.NaN, maxScore, totalGroupHits, scoreDocs, groupValueRef, null)); } @SuppressWarnings("unchecked") GroupDocs[] groupDocsArr = groupDocs.toArray(new GroupDocs[groupDocs.size()]); TopGroups topGroups = new TopGroups( - groupSort.getSort(), sortWithinGroup.getSort(), totalHitCount, totalGroupedHitCount, groupDocsArr + groupSort.getSort(), sortWithinGroup.getSort(), totalHitCount, totalGroupedHitCount, groupDocsArr, Float.NaN ); result.put(key, topGroups); diff --git a/solr/core/src/java/org/apache/solr/search/similarities/BM25SimilarityFactory.java b/solr/core/src/java/org/apache/solr/search/similarities/BM25SimilarityFactory.java index 3b18ef0..b09c7c9 100644 --- a/solr/core/src/java/org/apache/solr/search/similarities/BM25SimilarityFactory.java +++ b/solr/core/src/java/org/apache/solr/search/similarities/BM25SimilarityFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/similarities/DFRSimilarityFactory.java b/solr/core/src/java/org/apache/solr/search/similarities/DFRSimilarityFactory.java index 59daf07..62fc00a 100644 --- a/solr/core/src/java/org/apache/solr/search/similarities/DFRSimilarityFactory.java +++ b/solr/core/src/java/org/apache/solr/search/similarities/DFRSimilarityFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/similarities/DefaultSimilarityFactory.java b/solr/core/src/java/org/apache/solr/search/similarities/DefaultSimilarityFactory.java index 137a9d1..61d150c 100644 --- a/solr/core/src/java/org/apache/solr/search/similarities/DefaultSimilarityFactory.java +++ b/solr/core/src/java/org/apache/solr/search/similarities/DefaultSimilarityFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/similarities/IBSimilarityFactory.java b/solr/core/src/java/org/apache/solr/search/similarities/IBSimilarityFactory.java index 012272c..0ad9507 100644 --- a/solr/core/src/java/org/apache/solr/search/similarities/IBSimilarityFactory.java +++ b/solr/core/src/java/org/apache/solr/search/similarities/IBSimilarityFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/similarities/LMDirichletSimilarityFactory.java b/solr/core/src/java/org/apache/solr/search/similarities/LMDirichletSimilarityFactory.java index b118df7..b294416 100644 --- a/solr/core/src/java/org/apache/solr/search/similarities/LMDirichletSimilarityFactory.java +++ b/solr/core/src/java/org/apache/solr/search/similarities/LMDirichletSimilarityFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/similarities/LMJelinekMercerSimilarityFactory.java b/solr/core/src/java/org/apache/solr/search/similarities/LMJelinekMercerSimilarityFactory.java index 9a8f743..d3bff7b 100644 --- a/solr/core/src/java/org/apache/solr/search/similarities/LMJelinekMercerSimilarityFactory.java +++ b/solr/core/src/java/org/apache/solr/search/similarities/LMJelinekMercerSimilarityFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/search/similarities/SchemaSimilarityFactory.java b/solr/core/src/java/org/apache/solr/search/similarities/SchemaSimilarityFactory.java index 355c6eb..efdb644 100644 --- a/solr/core/src/java/org/apache/solr/search/similarities/SchemaSimilarityFactory.java +++ b/solr/core/src/java/org/apache/solr/search/similarities/SchemaSimilarityFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -26,9 +26,21 @@ import org.apache.solr.schema.SchemaAware; import org.apache.solr.schema.SimilarityFactory; /** - * SimilarityFactory that returns a PerFieldSimilarityWrapper - * that delegates to the fieldtype, if its configured, otherwise + * SimilarityFactory that returns a {@link PerFieldSimilarityWrapper} + * that delegates to the field type, if its configured, otherwise * {@link DefaultSimilarity}. + * + *

      + * NOTE: Users should be aware that in addition to supporting + * Similarity configurations specified on individual + * field types, this factory also differs in behavior from + * {@link DefaultSimilarityFactory} because of other differences in the + * implementations of PerFieldSimilarityWrapper and + * DefaultSimilarity - notably in methods such as + * {@link Similarity#coord} and {@link Similarity#queryNorm}. + *

      + * + * @see FieldType#getSimilarity */ public class SchemaSimilarityFactory extends SimilarityFactory implements SchemaAware { private Similarity similarity; diff --git a/solr/core/src/java/org/apache/solr/servlet/DirectSolrConnection.java b/solr/core/src/java/org/apache/solr/servlet/DirectSolrConnection.java index b781bef..c97b91d 100644 --- a/solr/core/src/java/org/apache/solr/servlet/DirectSolrConnection.java +++ b/solr/core/src/java/org/apache/solr/servlet/DirectSolrConnection.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java b/solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java index 08b1815..9dffd95 100644 --- a/solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java +++ b/solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,7 +21,6 @@ import java.io.InputStream; import java.io.IOException; import java.io.PrintWriter; -import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -42,7 +41,7 @@ public final class LoadAdminUiServlet extends HttpServlet { @Override public void doGet(HttpServletRequest request, HttpServletResponse response) - throws IOException, ServletException { + throws IOException { response.setCharacterEncoding("UTF-8"); response.setContentType("text/html"); @@ -76,7 +75,7 @@ public final class LoadAdminUiServlet extends HttpServlet { @Override public void doPost(HttpServletRequest request, HttpServletResponse response) - throws IOException, ServletException { + throws IOException { doGet(request, response); } } diff --git a/solr/core/src/java/org/apache/solr/servlet/RedirectServlet.java b/solr/core/src/java/org/apache/solr/servlet/RedirectServlet.java index abbfa85..d3b8c53 100644 --- a/solr/core/src/java/org/apache/solr/servlet/RedirectServlet.java +++ b/solr/core/src/java/org/apache/solr/servlet/RedirectServlet.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java index 9c28fd7..2407084 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,23 +17,18 @@ package org.apache.solr.servlet; +import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.Writer; +import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.io.StringWriter; -import java.io.OutputStreamWriter; -import java.io.ByteArrayInputStream; +import java.io.Writer; import java.nio.charset.Charset; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.WeakHashMap; -import org.apache.solr.handler.ContentStreamHandlerBase; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.xml.sax.InputSource; - import javax.servlet.Filter; import javax.servlet.FilterChain; import javax.servlet.FilterConfig; @@ -44,23 +39,33 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.solr.common.SolrException; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.cloud.CloudState; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CommonParams; -import org.apache.solr.util.FastWriter; import org.apache.solr.common.util.ContentStreamBase; -import org.apache.solr.core.*; -import org.apache.solr.handler.component.SearchHandler; -import org.apache.solr.request.*; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.core.Config; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.SolrConfig; +import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.ContentStreamHandlerBase; +import org.apache.solr.request.ServletSolrParams; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrQueryRequestBase; +import org.apache.solr.request.SolrRequestHandler; +import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.response.BinaryQueryResponseWriter; import org.apache.solr.response.QueryResponseWriter; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.servlet.cache.HttpCacheHeaderUtil; import org.apache.solr.servlet.cache.Method; +import org.apache.solr.util.FastWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.xml.sax.InputSource; /** * This filter looks at the incoming URL maps them to handlers defined in solrconfig.xml @@ -175,6 +180,13 @@ public class SolrDispatchFilter implements Filter handleAdminRequest(req, response, handler, solrReq); return; } + // Check for the core admin collections url + if( path.equals( "/admin/collections" ) ) { + handler = cores.getCollectionsHandler(); + solrReq = adminRequestParser.parse(null,path, req); + handleAdminRequest(req, response, handler, solrReq); + return; + } else { //otherwise, we should find a core from the path idx = path.indexOf( "/", 1 ); @@ -234,7 +246,7 @@ public class SolrDispatchFilter implements Filter if( qt != null && qt.startsWith("/") && (handler instanceof ContentStreamHandlerBase)) { //For security reasons it's a bad idea to allow a leading '/', ex: /select?qt=/update see SOLR-3161 //There was no restriction from Solr 1.4 thru 3.5 and it's not supported for update handlers. - throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Invalid query type. Do not use /select to access: "+qt); + throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Invalid Request Handler ('qt'). Do not use /select to access: "+qt); } } } diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java index a046633..fce3ae8 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -246,7 +246,7 @@ class HttpRequestContentStream extends ContentStreamBase { private final HttpServletRequest req; - public HttpRequestContentStream( HttpServletRequest req ) throws IOException { + public HttpRequestContentStream( HttpServletRequest req ) { this.req = req; contentType = req.getContentType(); diff --git a/solr/core/src/java/org/apache/solr/servlet/ZookeeperInfoServlet.java b/solr/core/src/java/org/apache/solr/servlet/ZookeeperInfoServlet.java index 366acdf..3cbca19 100644 --- a/solr/core/src/java/org/apache/solr/servlet/ZookeeperInfoServlet.java +++ b/solr/core/src/java/org/apache/solr/servlet/ZookeeperInfoServlet.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -28,7 +28,6 @@ import org.apache.zookeeper.data.Stat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -49,13 +48,13 @@ public final class ZookeeperInfoServlet extends HttpServlet { static final Logger log = LoggerFactory.getLogger(ZookeeperInfoServlet.class); @Override - public void init() throws ServletException { + public void init() { } @Override public void doGet(HttpServletRequest request, HttpServletResponse response) - throws IOException, ServletException { + throws IOException { response.setCharacterEncoding("UTF-8"); response.setContentType("application/json"); @@ -92,7 +91,7 @@ public final class ZookeeperInfoServlet extends HttpServlet { @Override public void doPost(HttpServletRequest request, HttpServletResponse response) - throws IOException, ServletException { + throws IOException { doGet(request, response); } @@ -363,7 +362,7 @@ public final class ZookeeperInfoServlet extends HttpServlet { json.write(v); } - boolean printZnode(JSONWriter json, String path) throws IOException { + boolean printZnode(JSONWriter json, String path) { try { Stat stat = new Stat(); // Trickily, the call to zkClient.getData fills in the stat variable diff --git a/solr/core/src/java/org/apache/solr/servlet/cache/HttpCacheHeaderUtil.java b/solr/core/src/java/org/apache/solr/servlet/cache/HttpCacheHeaderUtil.java index ece1e6b..5d09973 100644 --- a/solr/core/src/java/org/apache/solr/servlet/cache/HttpCacheHeaderUtil.java +++ b/solr/core/src/java/org/apache/solr/servlet/cache/HttpCacheHeaderUtil.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -27,8 +27,6 @@ import java.util.List; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.lucene.index.IndexReader; - import org.apache.solr.core.IndexDeletionPolicyWrapper; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrConfig; @@ -43,13 +41,11 @@ import org.apache.commons.codec.binary.Base64; public final class HttpCacheHeaderUtil { - public static void sendNotModified(HttpServletResponse res) - throws IOException { + public static void sendNotModified(HttpServletResponse res) { res.setStatus(HttpServletResponse.SC_NOT_MODIFIED); } - public static void sendPreconditionFailed(HttpServletResponse res) - throws IOException { + public static void sendPreconditionFailed(HttpServletResponse res) { res.setStatus(HttpServletResponse.SC_PRECONDITION_FAILED); } @@ -215,8 +211,7 @@ public final class HttpCacheHeaderUtil { public static boolean doCacheHeaderValidation(final SolrQueryRequest solrReq, final HttpServletRequest req, final Method reqMethod, - final HttpServletResponse resp) - throws IOException { + final HttpServletResponse resp) { if (Method.POST==reqMethod || Method.OTHER==reqMethod) { return false; @@ -244,14 +239,12 @@ public final class HttpCacheHeaderUtil { * Check for etag related conditional headers and set status * * @return true if no request processing is necessary and HTTP response status has been set, false otherwise. - * @throws IOException */ @SuppressWarnings("unchecked") public static boolean checkETagValidators(final HttpServletRequest req, final HttpServletResponse resp, final Method reqMethod, - final String etag) - throws IOException { + final String etag) { // First check If-None-Match because this is the common used header // element by HTTP clients @@ -281,12 +274,10 @@ public final class HttpCacheHeaderUtil { * Check for modify time related conditional headers and set status * * @return true if no request processing is necessary and HTTP response status has been set, false otherwise. - * @throws IOException */ public static boolean checkLastModValidators(final HttpServletRequest req, final HttpServletResponse resp, - final long lastMod) - throws IOException { + final long lastMod) { try { // First check for If-Modified-Since because this is the common diff --git a/solr/core/src/java/org/apache/solr/servlet/cache/Method.java b/solr/core/src/java/org/apache/solr/servlet/cache/Method.java index 5477f6a..4fe393e 100644 --- a/solr/core/src/java/org/apache/solr/servlet/cache/Method.java +++ b/solr/core/src/java/org/apache/solr/servlet/cache/Method.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java b/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java index 4e6f7c7..91322d9 100644 --- a/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java +++ b/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java @@ -1,7 +1,7 @@ package org.apache.solr.spelling; import org.apache.lucene.search.spell.StringDistance; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/spelling/ConjunctionSolrSpellChecker.java b/solr/core/src/java/org/apache/solr/spelling/ConjunctionSolrSpellChecker.java new file mode 100644 index 0000000..574f508 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/spelling/ConjunctionSolrSpellChecker.java @@ -0,0 +1,224 @@ +package org.apache.solr.spelling; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.Token; +import org.apache.lucene.search.spell.StringDistance; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.component.SpellCheckMergeData; +import org.apache.solr.search.SolrIndexSearcher; + +/** + *

      This class lets a query be run through multiple spell checkers. + * The initial use-case is to use {@link WordBreakSolrSpellChecker} + * in conjunction with a "standard" spell checker + * (such as {@link DirectSolrSpellChecker} + *

      + */ +public class ConjunctionSolrSpellChecker extends SolrSpellChecker { + private StringDistance stringDistance = null; + private Float accuracy = null; + private String dictionaryName = null; + private Analyzer queryAnalyzer = null; + private List checkers = new ArrayList(); + private boolean initalized = false; + + public void addChecker(SolrSpellChecker checker) { + if (initalized) { + throw new IllegalStateException( + "Need to add checkers before calling init()"); + } + try { + if (stringDistance == null) { + stringDistance = checker.getStringDistance(); + } else if (stringDistance != checker.getStringDistance()) { + throw new IllegalArgumentException( + "All checkers need to use the same StringDistance."); + } + } catch (UnsupportedOperationException uoe) { + // ignore + } + try { + if (accuracy == null) { + accuracy = checker.getAccuracy(); + } else if (accuracy != checker.getAccuracy()) { + throw new IllegalArgumentException( + "All checkers need to use the same Accuracy."); + } + } catch (UnsupportedOperationException uoe) { + // ignore + } + if (queryAnalyzer == null) { + queryAnalyzer = checker.getQueryAnalyzer(); + } else if (queryAnalyzer != checker.getQueryAnalyzer()) { + throw new IllegalArgumentException( + "All checkers need to use the same Analyzer."); + } + checkers.add(checker); + } + + @SuppressWarnings("unchecked") + @Override + public String init(NamedList config, SolrCore core) { + for (int i = 0; i < checkers.size(); i++) { + SolrSpellChecker c = checkers.get(i); + String dn = c.init(config, core); + + //TODO: in the future, we could develop this further to allow + // multiple spellcheckers with per-field dictionaries... + if (dictionaryName != null && !dictionaryName.equals(dn)) { + throw new IllegalArgumentException( + "Cannot have more than one dictionary. (" + dn + " , " + + dictionaryName + ")"); + } + dictionaryName = dn; + } + if (dictionaryName == null) { + dictionaryName = DEFAULT_DICTIONARY_NAME; + } + initalized = true; + return dictionaryName; + } + + @Override + public void build(SolrCore core, SolrIndexSearcher searcher) { + for (SolrSpellChecker c : checkers) { + c.build(core, searcher); + } + } + + @Override + public SpellingResult getSuggestions(SpellingOptions options) + throws IOException { + SpellingResult[] results = new SpellingResult[checkers.size()]; + for (int i = 0; i < checkers.size(); i++) { + results[i] = checkers.get(i).getSuggestions(options); + } + return mergeCheckers(results, options.count); + } + + @Override + public SpellingResult mergeSuggestions(SpellCheckMergeData mergeData, + int numSug, int count, boolean extendedResults) { + SpellingResult[] results = new SpellingResult[checkers.size()]; + for (int i = 0; i < checkers.size(); i++) { + results[i] = checkers.get(i).mergeSuggestions(mergeData, numSug, count, + extendedResults); + } + return mergeCheckers(results, numSug); + } + + //TODO: This just interleaves the results. In the future, we might want to let users give each checker its + // own weight and use that in combination to score & frequency to sort the results ? + private SpellingResult mergeCheckers(SpellingResult[] results, int numSug) { + Map combinedTokenFrequency = new HashMap(); + Map>> allSuggestions = new LinkedHashMap>>(); + for(SpellingResult result : results) { + if(result.getTokenFrequency()!=null) { + combinedTokenFrequency.putAll(result.getTokenFrequency()); + } + for(Map.Entry> entry : result.getSuggestions().entrySet()) { + List> allForThisToken = allSuggestions.get(entry.getKey()); + if(allForThisToken==null) { + allForThisToken = new ArrayList>(); + allSuggestions.put(entry.getKey(), allForThisToken); + } + allForThisToken.add(entry.getValue()); + } + } + SpellingResult combinedResult = new SpellingResult(); + for(Map.Entry>> entry : allSuggestions.entrySet()) { + Token original = entry.getKey(); + List>> corrIters = new ArrayList>>(entry.getValue().size()); + for(LinkedHashMap corrections : entry.getValue()) { + corrIters.add(corrections.entrySet().iterator()); + } + int numberAdded = 0; + while(numberAdded < numSug) { + boolean anyData = false; + for(Iterator> iter : corrIters) { + if(iter.hasNext()) { + anyData = true; + Map.Entry corr = iter.next(); + combinedResult.add(original, corr.getKey(), corr.getValue()); + Integer tokenFrequency = combinedTokenFrequency.get(original); + if(tokenFrequency!=null) { + combinedResult.addFrequency(original, tokenFrequency); + } + if(++numberAdded==numSug) { + break; + } + } + } + if(!anyData) { + break; + } + } + } + return combinedResult; + } + + @Override + public void reload(SolrCore core, SolrIndexSearcher searcher) + throws IOException { + for (SolrSpellChecker c : checkers) { + c.reload(core, searcher); + } + } + + @Override + public Analyzer getQueryAnalyzer() { + return queryAnalyzer; + } + + @Override + public String getDictionaryName() { + return dictionaryName; + } + + @Override + protected float getAccuracy() { + if (accuracy == null) { + return super.getAccuracy(); + } + return accuracy; + } + + @Override + protected StringDistance getStringDistance() { + if (stringDistance == null) { + return super.getStringDistance(); + } + return stringDistance; + } + + @Override + public boolean isSuggestionsMayOverlap() { + return true; + } +} diff --git a/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java b/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java index 0e3916b..830968a 100644 --- a/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java +++ b/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java @@ -1,6 +1,6 @@ package org.apache.solr.spelling; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java b/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java index 45702e7..11436f7 100644 --- a/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java +++ b/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.InputStreamReader; import java.util.List; +import org.apache.lucene.document.Field; import org.apache.lucene.index.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -105,13 +106,13 @@ public class FileBasedSpellChecker extends AbstractLuceneSpellChecker { for (String s : lines) { Document d = new Document(); - d.add(new TextField(WORD_FIELD_NAME, s)); + d.add(new TextField(WORD_FIELD_NAME, s, Field.Store.NO)); writer.addDocument(d); } writer.forceMerge(1); writer.close(); - dictionary = new HighFrequencyDictionary(IndexReader.open(ramDir), + dictionary = new HighFrequencyDictionary(DirectoryReader.open(ramDir), WORD_FIELD_NAME, 0.0f); } else { // check if character encoding is defined diff --git a/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java b/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java index d482e2e..4956db1 100644 --- a/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java +++ b/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java @@ -1,5 +1,5 @@ package org.apache.solr.spelling; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -16,6 +16,7 @@ package org.apache.solr.spelling; * limitations under the License. */ +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.store.FSDirectory; @@ -64,7 +65,7 @@ public class IndexBasedSpellChecker extends AbstractLuceneSpellChecker { if (sourceLocation != null) { try { FSDirectory luceneIndexDir = FSDirectory.open(new File(sourceLocation)); - this.reader = IndexReader.open(luceneIndexDir); + this.reader = DirectoryReader.open(luceneIndexDir); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.java b/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.java index 8d35039..b5435c2 100644 --- a/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.java +++ b/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.java @@ -1,5 +1,6 @@ package org.apache.solr.spelling; -/** + +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,12 +19,16 @@ package org.apache.solr.spelling; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.PriorityQueue; +import java.util.Set; import org.apache.lucene.analysis.Token; @@ -37,161 +42,388 @@ import org.apache.lucene.analysis.Token; *

      * */ -public class PossibilityIterator implements Iterator { - private List> possibilityList = new ArrayList>(); - private Iterator rankedPossibilityIterator = null; - private int correctionIndex[]; - private boolean done = false; - - @SuppressWarnings("unused") - private PossibilityIterator() { - throw new AssertionError("You shan't go here."); - } - - /** - *

      - * We assume here that the passed-in inner LinkedHashMaps are already sorted - * in order of "Best Possible Correction". - *

      - * - * @param suggestions - */ - public PossibilityIterator(Map> suggestions, int maximumRequiredSuggestions, int maxEvaluations) { - for (Map.Entry> entry : suggestions.entrySet()) { - Token token = entry.getKey(); - if(entry.getValue().size()==0) { - continue; - } - List possibleCorrections = new ArrayList(); - for (Map.Entry entry1 : entry.getValue().entrySet()) { - SpellCheckCorrection correction = new SpellCheckCorrection(); - correction.setOriginal(token); - correction.setCorrection(entry1.getKey()); - correction.setNumberOfOccurences(entry1.getValue()); - possibleCorrections.add(correction); - } - possibilityList.add(possibleCorrections); - } - - int wrapSize = possibilityList.size(); - if (wrapSize == 0) { - done = true; - } else { - correctionIndex = new int[wrapSize]; - for (int i = 0; i < wrapSize; i++) { - int suggestSize = possibilityList.get(i).size(); - if (suggestSize == 0) { - done = true; - break; - } - correctionIndex[i] = 0; - } - } - - long count = 0; - PriorityQueue rankedPossibilities = new PriorityQueue(); - while (count < maxEvaluations && internalHasNext()) { - RankedSpellPossibility rsp = internalNext(); - count++; - - if(rankedPossibilities.size() >= maximumRequiredSuggestions && rsp.getRank() >= rankedPossibilities.peek().getRank()) { - continue; - } +public class PossibilityIterator implements + Iterator { + private List> possibilityList = new ArrayList>(); + private Iterator rankedPossibilityIterator = null; + private int correctionIndex[]; + private boolean done = false; + private Iterator> nextOnes = null; + private int nextOnesRank = 0; + private int nextOnesIndex = 0; + private boolean suggestionsMayOverlap = false; + + @SuppressWarnings("unused") + private PossibilityIterator() { + throw new AssertionError("You shan't go here."); + } + + /** + *

      + * We assume here that the passed-in inner LinkedHashMaps are already sorted + * in order of "Best Possible Correction". + *

      + * + * @param suggestions + */ + public PossibilityIterator( + Map> suggestions, + int maximumRequiredSuggestions, int maxEvaluations, boolean overlap) { + this.suggestionsMayOverlap = overlap; + for (Map.Entry> entry : suggestions + .entrySet()) { + Token token = entry.getKey(); + if (entry.getValue().size() == 0) { + continue; + } + List possibleCorrections = new ArrayList(); + for (Map.Entry entry1 : entry.getValue().entrySet()) { + SpellCheckCorrection correction = new SpellCheckCorrection(); + correction.setOriginal(token); + correction.setCorrection(entry1.getKey()); + correction.setNumberOfOccurences(entry1.getValue()); + possibleCorrections.add(correction); + } + possibilityList.add(possibleCorrections); + } + + int wrapSize = possibilityList.size(); + if (wrapSize == 0) { + done = true; + } else { + correctionIndex = new int[wrapSize]; + for (int i = 0; i < wrapSize; i++) { + int suggestSize = possibilityList.get(i).size(); + if (suggestSize == 0) { + done = true; + break; + } + correctionIndex[i] = 0; + } + } + PriorityQueue rankedPossibilities = new PriorityQueue( + 11, new RankComparator()); + Set removeDuplicates = null; + if (suggestionsMayOverlap) { + removeDuplicates = new HashSet(); + } + long numEvaluations = 0; + while (numEvaluations < maxEvaluations && internalHasNext()) { + RankedSpellPossibility rsp = internalNext(); + numEvaluations++; + if (rankedPossibilities.size() >= maximumRequiredSuggestions + && rsp.rank >= rankedPossibilities.peek().rank) { + continue; + } if (!isSuggestionForReal(rsp)) { continue; } - rankedPossibilities.offer(rsp); - if(rankedPossibilities.size() > maximumRequiredSuggestions) { - rankedPossibilities.poll(); - } - } - - RankedSpellPossibility[] rpArr = new RankedSpellPossibility[rankedPossibilities.size()]; - for(int i=rankedPossibilities.size() - 1 ; i>=0 ; i--) { - rpArr[i] = rankedPossibilities.remove(); - } - rankedPossibilityIterator = Arrays.asList(rpArr).iterator(); - } - + if (removeDuplicates == null) { + rankedPossibilities.offer(rsp); + } else { + // Needs to be in token-offset order so that the match-and-replace + // option for collations can work. + Collections.sort(rsp.corrections, new StartOffsetComparator()); + if (removeDuplicates.add(rsp)) { + rankedPossibilities.offer(rsp); + } + } + if (rankedPossibilities.size() > maximumRequiredSuggestions) { + RankedSpellPossibility removed = rankedPossibilities.poll(); + if (removeDuplicates != null) { + removeDuplicates.remove(removed); + } + } + } + + RankedSpellPossibility[] rpArr = new RankedSpellPossibility[rankedPossibilities + .size()]; + for (int i = rankedPossibilities.size() - 1; i >= 0; i--) { + rpArr[i] = rankedPossibilities.remove(); + } + rankedPossibilityIterator = Arrays.asList(rpArr).iterator(); + } + private boolean isSuggestionForReal(RankedSpellPossibility rsp) { - for (SpellCheckCorrection corr : rsp.getCorrections()) { + for (SpellCheckCorrection corr : rsp.corrections) { if (!corr.getOriginalAsString().equals(corr.getCorrection())) { return true; } } return false; } - - private boolean internalHasNext() { - return !done; - } - - /** - *

      - * This method is converting the independent LinkHashMaps containing various - * (silo'ed) suggestions for each mis-spelled word into individual - * "holistic query corrections", aka. "Spell Check Possibility" - *

      - *

      - * Rank here is the sum of each selected term's position in its respective - * LinkedHashMap. - *

      - * - * @return - */ - private RankedSpellPossibility internalNext() { - if (done) { - throw new NoSuchElementException(); - } - - List possibleCorrection = new ArrayList(); - int rank = 0; - for (int i = 0; i < correctionIndex.length; i++) { - List singleWordPossibilities = possibilityList.get(i); - SpellCheckCorrection singleWordPossibility = singleWordPossibilities.get(correctionIndex[i]); - rank += correctionIndex[i]; - - if (i == correctionIndex.length - 1) { - correctionIndex[i]++; - if (correctionIndex[i] == singleWordPossibilities.size()) { - correctionIndex[i] = 0; - if (correctionIndex.length == 1) { - done = true; - } - for (int ii = i - 1; ii >= 0; ii--) { - correctionIndex[ii]++; - if (correctionIndex[ii] >= possibilityList.get(ii).size() && ii > 0) { - correctionIndex[ii] = 0; - } else { - break; - } - } - } - } - possibleCorrection.add(singleWordPossibility); - } - - if(correctionIndex[0] == possibilityList.get(0).size()) - { - done = true; - } - - RankedSpellPossibility rsl = new RankedSpellPossibility(); - rsl.setCorrections(possibleCorrection); - rsl.setRank(rank); - return rsl; - } - - public boolean hasNext() { - return rankedPossibilityIterator.hasNext(); - } - - public RankedSpellPossibility next() { - return rankedPossibilityIterator.next(); - } - - public void remove() { - throw new UnsupportedOperationException(); - } - + + private boolean internalHasNext() { + if (nextOnes != null && nextOnes.hasNext()) { + return true; + } + if (done) { + return false; + } + internalNextAdvance(); + if (nextOnes != null && nextOnes.hasNext()) { + return true; + } + return false; + } + + /** + *

      + * This method is converting the independent LinkHashMaps containing various + * (silo'ed) suggestions for each mis-spelled word into individual + * "holistic query corrections", aka. "Spell Check Possibility" + *

      + *

      + * Rank here is the sum of each selected term's position in its respective + * LinkedHashMap. + *

      + * + * @return + */ + private RankedSpellPossibility internalNext() { + if (nextOnes != null && nextOnes.hasNext()) { + RankedSpellPossibility rsl = new RankedSpellPossibility(); + rsl.corrections = nextOnes.next(); + rsl.rank = nextOnesRank; + rsl.index = nextOnesIndex++; + return rsl; + } + if (done) { + throw new NoSuchElementException(); + } + internalNextAdvance(); + if (nextOnes != null && nextOnes.hasNext()) { + RankedSpellPossibility rsl = new RankedSpellPossibility(); + rsl.corrections = nextOnes.next(); + rsl.rank = nextOnesRank; + rsl.index = nextOnesIndex++; + return rsl; + } + throw new NoSuchElementException(); + } + + private void internalNextAdvance() { + List possibleCorrection = null; + if (nextOnes != null && nextOnes.hasNext()) { + possibleCorrection = nextOnes.next(); + } else { + if (done) { + throw new NoSuchElementException(); + } + possibleCorrection = new ArrayList(); + List> possibleCorrections = null; + int rank = 0; + while (!done + && (possibleCorrections == null || possibleCorrections.size() == 0)) { + rank = 0; + for (int i = 0; i < correctionIndex.length; i++) { + List singleWordPossibilities = possibilityList + .get(i); + SpellCheckCorrection singleWordPossibility = singleWordPossibilities + .get(correctionIndex[i]); + rank += correctionIndex[i]; + if (i == correctionIndex.length - 1) { + correctionIndex[i]++; + if (correctionIndex[i] == singleWordPossibilities.size()) { + correctionIndex[i] = 0; + if (correctionIndex.length == 1) { + done = true; + } + for (int ii = i - 1; ii >= 0; ii--) { + correctionIndex[ii]++; + if (correctionIndex[ii] >= possibilityList.get(ii).size() + && ii > 0) { + correctionIndex[ii] = 0; + } else { + break; + } + } + } + } + possibleCorrection.add(singleWordPossibility); + } + if (correctionIndex[0] == possibilityList.get(0).size()) { + done = true; + } + if (suggestionsMayOverlap) { + possibleCorrections = separateOverlappingTokens(possibleCorrection); + } else { + possibleCorrections = new ArrayList>(1); + possibleCorrections.add(possibleCorrection); + } + } + nextOnes = possibleCorrections.iterator(); + nextOnesRank = rank; + nextOnesIndex = 0; + } + } + + private List> separateOverlappingTokens( + List possibleCorrection) { + List> ret = null; + if (possibleCorrection.size() == 1) { + ret = new ArrayList>(1); + ret.add(possibleCorrection); + return ret; + } + ret = new ArrayList>(); + for (int i = 0; i < possibleCorrection.size(); i++) { + List c = compatible(possibleCorrection, i); + ret.add(c); + } + return ret; + } + + private List compatible(List all, + int pos) { + List priorPassCompatibles = null; + { + List firstPassCompatibles = new ArrayList( + all.size()); + SpellCheckCorrection sacred = all.get(pos); + firstPassCompatibles.add(sacred); + int index = pos; + boolean gotOne = false; + for (int i = 0; i < all.size() - 1; i++) { + index++; + if (index == all.size()) { + index = 0; + } + SpellCheckCorrection disposable = all.get(index); + if (!conflicts(sacred, disposable)) { + firstPassCompatibles.add(disposable); + gotOne = true; + } + } + if (!gotOne) { + return firstPassCompatibles; + } + priorPassCompatibles = firstPassCompatibles; + } + + { + pos = 1; + while (true) { + if (pos == priorPassCompatibles.size() - 1) { + return priorPassCompatibles; + } + List subsequentPassCompatibles = new ArrayList( + priorPassCompatibles.size()); + SpellCheckCorrection sacred = null; + for (int i = 0; i <= pos; i++) { + sacred = priorPassCompatibles.get(i); + subsequentPassCompatibles.add(sacred); + } + int index = pos; + boolean gotOne = false; + for (int i = 0; i < priorPassCompatibles.size() - 1; i++) { + index++; + if (index == priorPassCompatibles.size()) { + break; + } + SpellCheckCorrection disposable = priorPassCompatibles.get(index); + if (!conflicts(sacred, disposable)) { + subsequentPassCompatibles.add(disposable); + gotOne = true; + } + } + if (!gotOne || pos == priorPassCompatibles.size() - 1) { + return subsequentPassCompatibles; + } + priorPassCompatibles = subsequentPassCompatibles; + pos++; + } + } + } + + private boolean conflicts(SpellCheckCorrection c1, SpellCheckCorrection c2) { + int s1 = c1.getOriginal().startOffset(); + int e1 = c1.getOriginal().endOffset(); + int s2 = c2.getOriginal().startOffset(); + int e2 = c2.getOriginal().endOffset(); + if (s2 >= s1 && s2 <= e1) { + return true; + } + if (s1 >= s2 && s1 <= e2) { + return true; + } + return false; + } + + public boolean hasNext() { + return rankedPossibilityIterator.hasNext(); + } + + public PossibilityIterator.RankedSpellPossibility next() { + return rankedPossibilityIterator.next(); + } + + public void remove() { + throw new UnsupportedOperationException(); + } + + public class RankedSpellPossibility { + public List corrections; + public int rank; + public int index; + + @Override + // hashCode() and equals() only consider the actual correction, not the rank + // or index. + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((corrections == null) ? 0 : corrections.hashCode()); + return result; + } + + @Override + // hashCode() and equals() only consider the actual correction, not the rank + // or index. + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + RankedSpellPossibility other = (RankedSpellPossibility) obj; + if (corrections == null) { + if (other.corrections != null) return false; + } else if (!corrections.equals(other.corrections)) return false; + return true; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("rank=").append(rank).append(" (").append(index).append(")"); + if (corrections != null) { + for (SpellCheckCorrection corr : corrections) { + sb.append(" "); + sb.append(corr.getOriginal()).append(">") + .append(corr.getCorrection()).append(" (").append( + corr.getNumberOfOccurences()).append(")"); + } + } + return sb.toString(); + } + } + + private class StartOffsetComparator implements + Comparator { + @Override + public int compare(SpellCheckCorrection o1, SpellCheckCorrection o2) { + return o1.getOriginal().startOffset() - o2.getOriginal().startOffset(); + } + } + + private class RankComparator implements Comparator { + // Rank poorer suggestions ahead of better ones for use with a PriorityQueue + public int compare(RankedSpellPossibility r1, RankedSpellPossibility r2) { + int retval = r2.rank - r1.rank; + if (retval == 0) { + retval = r2.index - r1.index; + } + return retval; + } + } + } diff --git a/solr/core/src/java/org/apache/solr/spelling/QueryConverter.java b/solr/core/src/java/org/apache/solr/spelling/QueryConverter.java index 4e276af..03ca029 100644 --- a/solr/core/src/java/org/apache/solr/spelling/QueryConverter.java +++ b/solr/core/src/java/org/apache/solr/spelling/QueryConverter.java @@ -1,5 +1,5 @@ package org.apache.solr.spelling; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -50,7 +50,32 @@ public abstract class QueryConverter implements NamedListInitializedPlugin { private NamedList args; protected Analyzer analyzer; - + + /** + *

      This term is marked prohibited in the query with the minus sign.

      + * + */ + public static final int PROHIBITED_TERM_FLAG = 16384; + /** + *

      This term is marked required in the query with the plus sign.

      + */ + public static final int REQUIRED_TERM_FLAG = 32768; + /** + *

      + * This term is directly followed by a boolean operator (AND/OR/NOT) + * and this operator differs from the prior boolean operator + * in the query (this signifies this term is likely part of a different + * query clause than the next term in the query) + *

      + */ + public static final int TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG = 65536; + /** + *

      + * This term exists in a query that contains boolean operators + * (AND/OR/NOT) + *

      + */ + public static final int TERM_IN_BOOLEAN_QUERY_FLAG = 131072; public void init(NamedList args) { this.args = args; } diff --git a/solr/core/src/java/org/apache/solr/spelling/RankedSpellPossibility.java b/solr/core/src/java/org/apache/solr/spelling/RankedSpellPossibility.java deleted file mode 100644 index 61093fc..0000000 --- a/solr/core/src/java/org/apache/solr/spelling/RankedSpellPossibility.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.apache.solr.spelling; -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.List; - -public class RankedSpellPossibility implements Comparable { - private List corrections; - private int rank; - - //Rank poorer suggestions ahead of better ones for use with a PriorityQueue - public int compareTo(RankedSpellPossibility rcl) { - return new Integer(rcl.rank).compareTo(rank); - } - - public List getCorrections() { - return corrections; - } - - public void setCorrections(List corrections) { - this.corrections = corrections; - } - - public int getRank() { - return rank; - } - - public void setRank(int rank) { - this.rank = rank; - } - - - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("rank=").append(rank); - if(corrections != null) { - for(SpellCheckCorrection corr : corrections) { - sb.append(" "); - sb.append(corr.getOriginal()).append(">").append(corr.getCorrection()).append(" (").append(corr.getNumberOfOccurences()).append(")"); - } - } - return sb.toString(); - } -} diff --git a/solr/core/src/java/org/apache/solr/spelling/ResultEntry.java b/solr/core/src/java/org/apache/solr/spelling/ResultEntry.java new file mode 100644 index 0000000..e6f64b5 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/spelling/ResultEntry.java @@ -0,0 +1,57 @@ +package org.apache.solr.spelling; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.analysis.Token; + +public class ResultEntry { + public Token token; + public String suggestion; + public int freq; + ResultEntry(Token t, String s, int f) { + token = t; + suggestion = s; + freq = f; + } + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + freq; + result = prime * result + + ((suggestion == null) ? 0 : suggestion.hashCode()); + result = prime * result + ((token == null) ? 0 : token.hashCode()); + return result; + } + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + ResultEntry other = (ResultEntry) obj; + if (freq != other.freq) return false; + if (suggestion == null) { + if (other.suggestion != null) return false; + } else if (!suggestion.equals(other.suggestion)) return false; + if (token == null) { + if (other.token != null) return false; + } else if (!token.equals(other.token)) return false; + return true; + } + +} diff --git a/solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.java b/solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.java index 6d00999..fe15e0e 100644 --- a/solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.java +++ b/solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.java @@ -1,5 +1,5 @@ package org.apache.solr.spelling; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -92,7 +92,12 @@ public abstract class SolrSpellChecker { //just use .5 as a default } - StringDistance sd = getStringDistance() == null ? new LevensteinDistance() : getStringDistance(); + StringDistance sd = null; + try { + sd = getStringDistance() == null ? new LevensteinDistance() : getStringDistance(); + } catch(UnsupportedOperationException uoe) { + sd = new LevensteinDistance(); + } SpellingResult result = new SpellingResult(); for (Map.Entry> entry : mergeData.origVsSuggested.entrySet()) { @@ -190,4 +195,8 @@ public abstract class SolrSpellChecker { * @throws IOException if there is an error producing suggestions */ public abstract SpellingResult getSuggestions(SpellingOptions options) throws IOException; + + public boolean isSuggestionsMayOverlap() { + return false; + } } diff --git a/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollation.java b/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollation.java index 2ce4c9c..18e98f3 100644 --- a/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollation.java +++ b/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollation.java @@ -1,6 +1,6 @@ package org.apache.solr.spelling; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java b/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java index 939fbc9..b859fd0 100644 --- a/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java +++ b/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java @@ -1,5 +1,5 @@ package org.apache.solr.spelling; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,12 +18,15 @@ package org.apache.solr.spelling; import java.util.ArrayList; import java.util.Arrays; +import java.util.Iterator; import java.util.List; import org.apache.lucene.analysis.Token; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.GroupParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.params.SpellingParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.handler.component.QueryComponent; import org.apache.solr.handler.component.ResponseBuilder; @@ -37,7 +40,7 @@ public class SpellCheckCollator { private static final Logger LOG = LoggerFactory.getLogger(SpellCheckCollator.class); public List collate(SpellingResult result, String originalQuery, ResponseBuilder ultimateResponse, - int maxCollations, int maxTries, int maxEvaluations) { + int maxCollations, int maxTries, int maxEvaluations, boolean suggestionsMayOverlap) { List collations = new ArrayList(); QueryComponent queryComponent = null; @@ -51,8 +54,10 @@ public class SpellCheckCollator { } boolean verifyCandidateWithQuery = true; + int maxNumberToIterate = maxTries; if (maxTries < 1) { maxTries = 1; + maxNumberToIterate = maxCollations; verifyCandidateWithQuery = false; } if (queryComponent == null && verifyCandidateWithQuery) { @@ -63,17 +68,32 @@ public class SpellCheckCollator { int tryNo = 0; int collNo = 0; - PossibilityIterator possibilityIter = new PossibilityIterator(result.getSuggestions(), maxTries, maxEvaluations); + PossibilityIterator possibilityIter = new PossibilityIterator(result.getSuggestions(), maxNumberToIterate, maxEvaluations, suggestionsMayOverlap); while (tryNo < maxTries && collNo < maxCollations && possibilityIter.hasNext()) { - RankedSpellPossibility possibility = possibilityIter.next(); - String collationQueryStr = getCollation(originalQuery, possibility.getCorrections()); + PossibilityIterator.RankedSpellPossibility possibility = possibilityIter.next(); + String collationQueryStr = getCollation(originalQuery, possibility.corrections); int hits = 0; if (verifyCandidateWithQuery) { tryNo++; - - ModifiableSolrParams params = new ModifiableSolrParams(ultimateResponse.req.getParams()); + SolrParams origParams = ultimateResponse.req.getParams(); + ModifiableSolrParams params = new ModifiableSolrParams(origParams); + Iterator origParamIterator = origParams.getParameterNamesIterator(); + int pl = SpellingParams.SPELLCHECK_COLLATE_PARAM_OVERRIDE.length(); + while (origParamIterator.hasNext()) { + String origParamName = origParamIterator.next(); + if (origParamName + .startsWith(SpellingParams.SPELLCHECK_COLLATE_PARAM_OVERRIDE) + && origParamName.length() > pl) { + String[] val = origParams.getParams(origParamName); + if (val.length == 1 && val[0].length() == 0) { + params.set(origParamName.substring(pl), (String[]) null); + } else { + params.set(origParamName.substring(pl), val); + } + } + } params.set(CommonParams.Q, collationQueryStr); params.remove(CommonParams.START); params.set(CommonParams.FL, "id"); @@ -102,10 +122,10 @@ public class SpellCheckCollator { SpellCheckCollation collation = new SpellCheckCollation(); collation.setCollationQuery(collationQueryStr); collation.setHits(hits); - collation.setInternalRank(possibility.getRank()); + collation.setInternalRank(suggestionsMayOverlap ? ((possibility.rank * 1000) + possibility.index) : possibility.rank); NamedList misspellingsAndCorrections = new NamedList(); - for (SpellCheckCorrection corr : possibility.getCorrections()) { + for (SpellCheckCorrection corr : possibility.corrections) { misspellingsAndCorrections.add(corr.getOriginal().toString(), corr.getCorrection()); } collation.setMisspellingsAndCorrections(misspellingsAndCorrections); @@ -122,16 +142,53 @@ public class SpellCheckCollator { List corrections) { StringBuilder collation = new StringBuilder(origQuery); int offset = 0; - for (SpellCheckCorrection correction : corrections) { + String corr = ""; + for(int i=0 ; i-1 && indexOfSpace0 ? collation.charAt(tok.startOffset()-1) : ' '; + if(previousChar=='-' || previousChar=='+') { + corrSb.insert(indexOfSpace + bump, previousChar); + if(requiredOrProhibited==null) { + requiredOrProhibited = previousChar; + } + bump++; + } else if ((tok.getFlags() & QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG) == QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG) { + corrSb.insert(indexOfSpace + bump, "AND "); + bump += 4; + } + indexOfSpace = correction.getCorrection().indexOf(' ', indexOfSpace + bump); + } + + int oneForReqOrProhib = 0; + if(addParenthesis) { + if(requiredOrProhibited!=null) { + corrSb.insert(0, requiredOrProhibited); + oneForReqOrProhib++; + } + corrSb.insert(0, '('); + corrSb.append(')'); + } + corr = corrSb.toString(); + int startIndex = tok.startOffset() + offset - oneForReqOrProhib; + int endIndex = tok.endOffset() + offset; + collation.replace(startIndex, endIndex, corr); + offset += corr.length() - oneForReqOrProhib - (tok.endOffset() - tok.startOffset()); } return collation.toString(); } diff --git a/solr/core/src/java/org/apache/solr/spelling/SpellCheckCorrection.java b/solr/core/src/java/org/apache/solr/spelling/SpellCheckCorrection.java index 4525fe4..a7edda9 100644 --- a/solr/core/src/java/org/apache/solr/spelling/SpellCheckCorrection.java +++ b/solr/core/src/java/org/apache/solr/spelling/SpellCheckCorrection.java @@ -1,5 +1,5 @@ package org.apache.solr.spelling; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/spelling/SpellingOptions.java b/solr/core/src/java/org/apache/solr/spelling/SpellingOptions.java index 8c7bc5d..cf12f4c 100644 --- a/solr/core/src/java/org/apache/solr/spelling/SpellingOptions.java +++ b/solr/core/src/java/org/apache/solr/spelling/SpellingOptions.java @@ -7,7 +7,7 @@ import org.apache.solr.common.params.SolrParams; import java.util.Collection; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java b/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java index 68f5835..bb7c10c 100644 --- a/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java +++ b/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -38,8 +38,18 @@ import org.apache.lucene.analysis.tokenattributes.TypeAttribute; /** * Converts the query string to a Collection of Lucene tokens using a regular expression. - * Boolean operators AND and OR are skipped. - * + * Boolean operators AND, OR, NOT are skipped. + * + * Each term is checked to determine if it is optional, required or prohibited. Required + * terms output a {@link Token} with the {@link QueryConverter#REQUIRED_TERM_FLAG} set. + * Prohibited terms output a {@link Token} with the {@link QueryConverter#PROHIBITED_TERM_FLAG} + * set. If the query uses the plus (+) and minus (-) to denote required and prohibited, this + * determination will be accurate. In the case boolean AND/OR/NOTs are used, this + * converter makes an uninformed guess as to whether the term would likely behave as if it + * is Required or Prohibited and sets the flags accordingly. These flags are used downstream + * to generate collations for {@link WordBreakSolrSpellChecker}, in cases where an original + * term is split up into multiple Tokens. + * * @since solr 1.3 **/ public class SpellingQueryConverter extends QueryConverter { @@ -86,8 +96,7 @@ public class SpellingQueryConverter extends QueryConverter { final static String PATTERN = "(?:(?!(" + NMTOKEN + ":|\\d+)))[\\p{L}_\\-0-9]+"; // previous version: Pattern.compile("(?:(?!(\\w+:|\\d+)))\\w+"); protected Pattern QUERY_REGEX = Pattern.compile(PATTERN); - - + /** * Converts the original query string to a collection of Lucene Tokens. * @param original the original query string @@ -99,37 +108,87 @@ public class SpellingQueryConverter extends QueryConverter { return Collections.emptyList(); } Collection result = new ArrayList(); - //TODO: Extract the words using a simple regex, but not query stuff, and then analyze them to produce the token stream Matcher matcher = QUERY_REGEX.matcher(original); - while (matcher.find()) { - String word = matcher.group(0); - if (word.equals("AND") == false && word.equals("OR") == false) { - try { - analyze(result, new StringReader(word), matcher.start()); - } catch (IOException e) { - // TODO: shouldn't we log something? - } + String nextWord = null; + int nextStartIndex = 0; + String lastBooleanOp = null; + while (nextWord!=null || matcher.find()) { + String word = null; + int startIndex = 0; + if(nextWord != null) { + word = nextWord; + startIndex = nextStartIndex; + nextWord = null; + } else { + word = matcher.group(0); + startIndex = matcher.start(); + } + if(matcher.find()) { + nextWord = matcher.group(0); + nextStartIndex = matcher.start(); + } + if("AND".equals(word) || "OR".equals(word) || "NOT".equals(word)) { + lastBooleanOp = word; + continue; + } + // treat "AND NOT" as "NOT"... + if ("AND".equals(nextWord) + && original.length() > nextStartIndex + 7 + && original.substring(nextStartIndex, nextStartIndex + 7).equals( + "AND NOT")) { + nextWord = "NOT"; + } + + int flagValue = 0; + if (word.charAt(0) == '-' + || (startIndex > 0 && original.charAt(startIndex - 1) == '-')) { + flagValue = PROHIBITED_TERM_FLAG; + } else if (word.charAt(0) == '+' + || (startIndex > 0 && original.charAt(startIndex - 1) == '+')) { + flagValue = REQUIRED_TERM_FLAG; + //we don't know the default operator so just assume the first operator isn't new. + } else if (nextWord != null + && lastBooleanOp != null + && !nextWord.equals(lastBooleanOp) + && ("AND".equals(nextWord) || "OR".equals(nextWord) || "NOT".equals(nextWord))) { + flagValue = TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG; + //...unless the 1st boolean operator is a NOT, because only AND/OR can be default. + } else if (nextWord != null + && lastBooleanOp == null + && !nextWord.equals(lastBooleanOp) + && ("NOT".equals(nextWord))) { + flagValue = TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG; + } + try { + analyze(result, new StringReader(word), startIndex, flagValue); + } catch (IOException e) { + // TODO: shouldn't we log something? + } + } + if(lastBooleanOp != null) { + for(Token t : result) { + int f = t.getFlags(); + t.setFlags(f |= QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG); } } return result; } - protected void analyze(Collection result, Reader text, int offset) throws IOException { + protected void analyze(Collection result, Reader text, int offset, int flagsAttValue) throws IOException { TokenStream stream = analyzer.tokenStream("", text); // TODO: support custom attributes CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class); - FlagsAttribute flagsAtt = stream.addAttribute(FlagsAttribute.class); TypeAttribute typeAtt = stream.addAttribute(TypeAttribute.class); PayloadAttribute payloadAtt = stream.addAttribute(PayloadAttribute.class); PositionIncrementAttribute posIncAtt = stream.addAttribute(PositionIncrementAttribute.class); OffsetAttribute offsetAtt = stream.addAttribute(OffsetAttribute.class); stream.reset(); - while (stream.incrementToken()) { + while (stream.incrementToken()) { Token token = new Token(); token.copyBuffer(termAtt.buffer(), 0, termAtt.length()); - token.setStartOffset(offset + offsetAtt.startOffset()); - token.setEndOffset(offset + offsetAtt.endOffset()); - token.setFlags(flagsAtt.getFlags()); + token.setOffset(offset + offsetAtt.startOffset(), + offset + offsetAtt.endOffset()); + token.setFlags(flagsAttValue); //overwriting any flags already set... token.setType(typeAtt.type()); token.setPayload(payloadAtt.getPayload()); token.setPositionIncrement(posIncAtt.getPositionIncrement()); diff --git a/solr/core/src/java/org/apache/solr/spelling/SpellingResult.java b/solr/core/src/java/org/apache/solr/spelling/SpellingResult.java index 7eb1c66..06816ba 100644 --- a/solr/core/src/java/org/apache/solr/spelling/SpellingResult.java +++ b/solr/core/src/java/org/apache/solr/spelling/SpellingResult.java @@ -1,5 +1,5 @@ package org.apache.solr.spelling; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/spelling/SuggestQueryConverter.java b/solr/core/src/java/org/apache/solr/spelling/SuggestQueryConverter.java index 85d4718..58a2e3f 100644 --- a/solr/core/src/java/org/apache/solr/spelling/SuggestQueryConverter.java +++ b/solr/core/src/java/org/apache/solr/spelling/SuggestQueryConverter.java @@ -1,6 +1,6 @@ package org.apache.solr.spelling; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -38,7 +38,7 @@ public class SuggestQueryConverter extends SpellingQueryConverter { Collection result = new ArrayList(); try { - analyze(result, new StringReader(original), 0); + analyze(result, new StringReader(original), 0, 0); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/solr/core/src/java/org/apache/solr/spelling/WordBreakSolrSpellChecker.java b/solr/core/src/java/org/apache/solr/spelling/WordBreakSolrSpellChecker.java new file mode 100644 index 0000000..00705d4 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/spelling/WordBreakSolrSpellChecker.java @@ -0,0 +1,353 @@ +package org.apache.solr.spelling; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.regex.Pattern; + +import org.apache.lucene.analysis.Token; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.spell.CombineSuggestion; +import org.apache.lucene.search.spell.SuggestWord; +import org.apache.lucene.search.spell.WordBreakSpellChecker; +import org.apache.lucene.search.spell.WordBreakSpellChecker.BreakSuggestionSortMethod; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.SolrCore; +import org.apache.solr.search.SolrIndexSearcher; + +/** + *

      + * A spellchecker that breaks and combines words. + *

      + *

      + * This will not combine adjacent tokens that do not have + * the same required status (prohibited, required, optional). + * However, this feature depends on incoming term flags + * being properly set. ({@link QueryConverter#PROHIBITED_TERM_FLAG}, + * {@link QueryConverter#REQUIRED_TERM_FLAG}, + * {@link QueryConverter#TERM_IN_BOOLEAN_QUERY_FLAG}, and + * {@link QueryConverter#TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG} ) + * This feature breaks completely if the upstream analyzer or query + * converter sets flags with the same values but different meanings. + * The default query converter (if not using "spellcheck.q") + * is {@link SpellingQueryConverter}, which properly sets these flags. + *

      + */ +public class WordBreakSolrSpellChecker extends SolrSpellChecker { + /** + *

      + * Try to combine multiple words into one? [true|false] + *

      + */ + public static final String PARAM_COMBINE_WORDS = "combineWords"; + /** + *

      + * Try to break words into multiples? [true|false] + *

      + */ + public static final String PARAM_BREAK_WORDS = "breakWords"; + /** + * See {@link WordBreakSpellChecker#setMaxChanges} + */ + public static final String PARAM_MAX_CHANGES = "maxChanges"; + /** + * See {@link WordBreakSpellChecker#setMaxCombineWordLength} + */ + public static final String PARAM_MAX_COMBINE_WORD_LENGTH = "maxCombinedLength"; + /** + * See {@link WordBreakSpellChecker#setMinBreakWordLength} + */ + public static final String PARAM_MIN_BREAK_WORD_LENGTH = "minBreakLength"; + /** + * See {@link BreakSuggestionTieBreaker} for options. + */ + public static final String PARAM_BREAK_SUGGESTION_TIE_BREAKER = "breakSugestionTieBreaker"; + /** + * See {@link WordBreakSpellChecker#setMaxEvaluations} + */ + public static final String PARAM_MAX_EVALUATIONS = "maxEvaluations"; + /** + * See {@link WordBreakSpellChecker#setMinSuggestionFrequency} + */ + public static final String PARAM_MIN_SUGGESTION_FREQUENCY = "minSuggestionFreq"; + + /** + *

      + * Specify a value on the "breakSugestionTieBreaker" parameter. + * The default is MAX_FREQ. + *

      + */ + public enum BreakSuggestionTieBreaker { + /** + * See + * {@link BreakSuggestionSortMethod#NUM_CHANGES_THEN_MAX_FREQUENCY} + * # + */ + MAX_FREQ, + /** + * See + * {@link BreakSuggestionSortMethod#NUM_CHANGES_THEN_SUMMED_FREQUENCY} + */ + SUM_FREQ + }; + + private WordBreakSpellChecker wbsp = null; + private boolean combineWords = false; + private boolean breakWords = false; + private BreakSuggestionSortMethod sortMethod = BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY; + private static final Pattern spacePattern = Pattern.compile("\\s+"); + + @Override + public String init(@SuppressWarnings("unchecked") NamedList config, + SolrCore core) { + String name = super.init(config, core); + combineWords = boolParam(config, PARAM_COMBINE_WORDS); + breakWords = boolParam(config, PARAM_BREAK_WORDS); + wbsp = new WordBreakSpellChecker(); + String bstb = strParam(config, PARAM_BREAK_SUGGESTION_TIE_BREAKER); + if (bstb != null) { + bstb = bstb.toUpperCase(); + if (bstb.equals(BreakSuggestionTieBreaker.SUM_FREQ.name())) { + sortMethod = BreakSuggestionSortMethod.NUM_CHANGES_THEN_SUMMED_FREQUENCY; + } else if (bstb.equals(BreakSuggestionTieBreaker.MAX_FREQ.name())) { + sortMethod = BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY; + } else { + throw new IllegalArgumentException("Invalid value for parameter " + + PARAM_BREAK_SUGGESTION_TIE_BREAKER + " : " + bstb); + } + } else { + sortMethod = BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY; + } + int mc = intParam(config, PARAM_MAX_CHANGES); + if (mc > 0) { + wbsp.setMaxChanges(mc); + } + int mcl = intParam(config, PARAM_MAX_COMBINE_WORD_LENGTH); + if (mcl > 0) { + wbsp.setMaxCombineWordLength(mcl); + } + int mbwl = intParam(config, PARAM_MIN_BREAK_WORD_LENGTH); + if (mbwl > 0) { + wbsp.setMinBreakWordLength(mbwl); + } + int me = intParam(config, PARAM_MAX_EVALUATIONS); + if (me > 0) { + wbsp.setMaxEvaluations(me); + } + int msf = intParam(config, PARAM_MIN_SUGGESTION_FREQUENCY); + if (msf > 0) { + wbsp.setMinSuggestionFrequency(msf); + } + return name; + } + + private String strParam(@SuppressWarnings("unchecked") NamedList config, + String paramName) { + Object o = config.get(paramName); + return o == null ? null : o.toString(); + } + + private boolean boolParam(@SuppressWarnings("unchecked") NamedList config, + String paramName) { + String s = strParam(config, paramName); + if ("true".equalsIgnoreCase(s) || "on".equalsIgnoreCase(s)) { + return true; + } + return false; + } + + private int intParam(@SuppressWarnings("unchecked") NamedList config, + String paramName) { + Object o = config.get(paramName); + if (o == null) { + return 0; + } + try { + return Integer.parseInt(o.toString()); + } catch (NumberFormatException nfe) { + throw new IllegalArgumentException("Invalid integer for parameter " + + paramName + " : " + o); + } + } + + @Override + public SpellingResult getSuggestions(SpellingOptions options) + throws IOException { + IndexReader ir = options.reader; + int numSuggestions = options.count; + + StringBuilder sb = new StringBuilder(); + Token[] tokenArr = options.tokens.toArray(new Token[options.tokens.size()]); + List termArr = new ArrayList(options.tokens.size() + 2); + + List breakSuggestionList = new ArrayList(); + boolean lastOneProhibited = false; + boolean lastOneRequired = false; + boolean lastOneprocedesNewBooleanOp = false; + for (int i = 0; i < tokenArr.length; i++) { + boolean prohibited = + (tokenArr[i].getFlags() & QueryConverter.PROHIBITED_TERM_FLAG) == + QueryConverter.PROHIBITED_TERM_FLAG; + boolean required = + (tokenArr[i].getFlags() & QueryConverter.REQUIRED_TERM_FLAG) == + QueryConverter.REQUIRED_TERM_FLAG; + boolean procedesNewBooleanOp = + (tokenArr[i].getFlags() & QueryConverter.TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG) == + QueryConverter.TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG; + if (i > 0 + && (prohibited != lastOneProhibited || required != lastOneRequired || lastOneprocedesNewBooleanOp)) { + termArr.add(WordBreakSpellChecker.SEPARATOR_TERM); + } + lastOneProhibited = prohibited; + lastOneRequired = required; + lastOneprocedesNewBooleanOp = procedesNewBooleanOp; + + Term thisTerm = new Term(field, tokenArr[i].toString()); + termArr.add(thisTerm); + if (breakWords) { + SuggestWord[][] breakSuggestions = wbsp.suggestWordBreaks(thisTerm, + numSuggestions, ir, options.suggestMode, sortMethod); + for (SuggestWord[] breakSuggestion : breakSuggestions) { + sb.delete(0, sb.length()); + boolean firstOne = true; + int freq = 0; + for (SuggestWord word : breakSuggestion) { + if (!firstOne) { + sb.append(" "); + } + firstOne = false; + sb.append(word.string); + if (sortMethod == BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY) { + freq = Math.max(freq, word.freq); + } else { + freq += word.freq; + } + } + breakSuggestionList.add(new ResultEntry(tokenArr[i], sb.toString(), + freq)); + } + } + } + List combineSuggestionList = Collections.emptyList(); + CombineSuggestion[] combineSuggestions = wbsp.suggestWordCombinations( + termArr.toArray(new Term[termArr.size()]), numSuggestions, ir, options.suggestMode); + if (combineWords) { + combineSuggestionList = new ArrayList( + combineSuggestions.length); + for (CombineSuggestion cs : combineSuggestions) { + int firstTermIndex = cs.originalTermIndexes[0]; + int lastTermIndex = cs.originalTermIndexes[cs.originalTermIndexes.length - 1]; + sb.delete(0, sb.length()); + for (int i = firstTermIndex; i <= lastTermIndex; i++) { + if (i > firstTermIndex) { + sb.append(" "); + } + sb.append(tokenArr[i].toString()); + } + Token token = new Token(sb.toString(), tokenArr[firstTermIndex] + .startOffset(), tokenArr[lastTermIndex].endOffset()); + combineSuggestionList.add(new ResultEntry(token, cs.suggestion.string, + cs.suggestion.freq)); + } + } + + // Interleave the two lists of suggestions into one SpellingResult + SpellingResult result = new SpellingResult(); + Iterator breakIter = breakSuggestionList.iterator(); + Iterator combineIter = combineSuggestionList.iterator(); + ResultEntry lastBreak = breakIter.hasNext() ? breakIter.next() : null; + ResultEntry lastCombine = combineIter.hasNext() ? combineIter.next() : null; + int breakCount = 0; + int combineCount = 0; + while (lastBreak != null || lastCombine != null) { + if (lastBreak == null) { + result.add(lastCombine.token, lastCombine.suggestion, lastCombine.freq); + result.addFrequency(lastCombine.token, getCombineFrequency(ir, lastCombine.token)); + lastCombine = null; + } else if (lastCombine == null) { + result.add(lastBreak.token, lastBreak.suggestion, lastBreak.freq); + result.addFrequency(lastBreak.token, ir.docFreq(new Term(field, lastBreak.token.toString()))); + lastBreak = null; + } else if (lastBreak.freq < lastCombine.freq) { + result.add(lastCombine.token, lastCombine.suggestion, lastCombine.freq); + result.addFrequency(lastCombine.token, getCombineFrequency(ir, lastCombine.token)); + lastCombine = null; + } else if (lastCombine.freq < lastBreak.freq) { + result.add(lastBreak.token, lastBreak.suggestion, lastBreak.freq); + result.addFrequency(lastBreak.token, ir.docFreq(new Term(field, lastBreak.token.toString()))); + lastBreak = null; + } else if (breakCount >= combineCount) { + result.add(lastCombine.token, lastCombine.suggestion, lastCombine.freq); + result.addFrequency(lastCombine.token, getCombineFrequency(ir, lastCombine.token)); + lastCombine = null; + } else { + result.add(lastBreak.token, lastBreak.suggestion, lastBreak.freq); + result.addFrequency(lastBreak.token, ir.docFreq(new Term(field, lastBreak.token.toString()))); + lastBreak = null; + } + if (result.getSuggestions().size() > numSuggestions) { + break; + } + if (lastBreak == null && breakIter.hasNext()) { + lastBreak = breakIter.next(); + breakCount++; + } + if (lastCombine == null && combineIter.hasNext()) { + lastCombine = combineIter.next(); + combineCount++; + } + } + return result; + } + + private int getCombineFrequency(IndexReader ir, Token token) throws IOException { + String[] words = spacePattern.split(token.toString()); + int result = 0; + if(sortMethod==BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY) { + for(String word : words) { + result = Math.max(result, ir.docFreq(new Term(field, word))); + } + } else { + for(String word : words) { + result += ir.docFreq(new Term(field, word)); + } + } + return result; + } + + @Override + public void build(SolrCore core, SolrIndexSearcher searcher) { + /* no-op */ + } + + @Override + public void reload(SolrCore core, SolrIndexSearcher searcher) + throws IOException { + /* no-op */ + } + + @Override + public boolean isSuggestionsMayOverlap() { + return true; + } +} diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/LookupFactory.java b/solr/core/src/java/org/apache/solr/spelling/suggest/LookupFactory.java index d621295..147b261 100644 --- a/solr/core/src/java/org/apache/solr/spelling/suggest/LookupFactory.java +++ b/solr/core/src/java/org/apache/solr/spelling/suggest/LookupFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.spelling.suggest; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java b/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java index 34c7679..18a8f97 100644 --- a/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java +++ b/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FSTLookupFactory.java b/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FSTLookupFactory.java index b32af8e..8676937 100644 --- a/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FSTLookupFactory.java +++ b/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FSTLookupFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.spelling.suggest.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/fst/WFSTLookupFactory.java b/solr/core/src/java/org/apache/solr/spelling/suggest/fst/WFSTLookupFactory.java index 523816a..24c5005 100644 --- a/solr/core/src/java/org/apache/solr/spelling/suggest/fst/WFSTLookupFactory.java +++ b/solr/core/src/java/org/apache/solr/spelling/suggest/fst/WFSTLookupFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.spelling.suggest.fst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellLookupFactory.java b/solr/core/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellLookupFactory.java index 94a7f7a..a57f962 100644 --- a/solr/core/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellLookupFactory.java +++ b/solr/core/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellLookupFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.spelling.suggest.jaspell; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/tst/TSTLookupFactory.java b/solr/core/src/java/org/apache/solr/spelling/suggest/tst/TSTLookupFactory.java index a5aa8ab..ee5a0f6 100644 --- a/solr/core/src/java/org/apache/solr/spelling/suggest/tst/TSTLookupFactory.java +++ b/solr/core/src/java/org/apache/solr/spelling/suggest/tst/TSTLookupFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.spelling.suggest.tst; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java b/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java index 528969c..3c11779 100644 --- a/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java +++ b/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,6 +19,7 @@ package org.apache.solr.update; import org.apache.lucene.document.Document; import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; @@ -27,6 +28,8 @@ import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; +import java.util.List; + /** * */ @@ -42,6 +45,7 @@ public class AddUpdateCommand extends UpdateCommand { public boolean overwrite = true; public Term updateTerm; + public int commitWithin = -1; public AddUpdateCommand(SolrQueryRequest req) { diff --git a/solr/core/src/java/org/apache/solr/update/CommitTracker.java b/solr/core/src/java/org/apache/solr/update/CommitTracker.java index 17521b8..8df3d1e 100644 --- a/solr/core/src/java/org/apache/solr/update/CommitTracker.java +++ b/solr/core/src/java/org/apache/solr/update/CommitTracker.java @@ -1,6 +1,6 @@ package org.apache.solr.update; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java b/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java index d8d5271..2a7649b 100644 --- a/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java +++ b/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java index 8b49df5..3b4ee53 100644 --- a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java +++ b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java @@ -1,6 +1,6 @@ package org.apache.solr.update; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,7 +21,6 @@ import java.io.IOException; import org.apache.lucene.index.IndexWriter; import org.apache.solr.cloud.RecoveryStrategy; -import org.apache.solr.common.SolrException; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.DirectoryFactory; import org.apache.solr.core.SolrCore; @@ -64,7 +63,7 @@ public final class DefaultSolrCoreState extends SolrCoreState { } @Override - public void decref(IndexWriterCloser closer) throws IOException { + public void decref(IndexWriterCloser closer) { synchronized (this) { refCnt--; if (refCnt == 0) { diff --git a/solr/core/src/java/org/apache/solr/update/DeleteUpdateCommand.java b/solr/core/src/java/org/apache/solr/update/DeleteUpdateCommand.java index 6ab8ed2..50bcaf7 100644 --- a/solr/core/src/java/org/apache/solr/update/DeleteUpdateCommand.java +++ b/solr/core/src/java/org/apache/solr/update/DeleteUpdateCommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java index 31e738c..5c189d3 100644 --- a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java +++ b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,7 +22,9 @@ package org.apache.solr.update; import java.io.IOException; import java.net.URL; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -42,11 +44,9 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.SolrConfig.UpdateHandlerInfo; @@ -93,7 +93,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState protected final CommitTracker commitTracker; protected final CommitTracker softCommitTracker; - public DirectUpdateHandler2(SolrCore core) throws IOException { + public DirectUpdateHandler2(SolrCore core) { super(core); solrCoreState = new DefaultSolrCoreState(core.getDirectoryFactory()); @@ -109,7 +109,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, true, true); } - public DirectUpdateHandler2(SolrCore core, UpdateHandler updateHandler) throws IOException { + public DirectUpdateHandler2(SolrCore core, UpdateHandler updateHandler) { super(core); if (updateHandler instanceof DirectUpdateHandler2) { this.solrCoreState = ((DirectUpdateHandler2) updateHandler).solrCoreState; @@ -162,37 +162,68 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState try { if (cmd.overwrite) { - Term updateTerm; - Term idTerm = new Term(idField.getName(), cmd.getIndexedId()); - boolean del = false; - if (cmd.updateTerm == null) { - updateTerm = idTerm; - } else { - del = true; - updateTerm = cmd.updateTerm; + + // Check for delete by query commands newer (i.e. reordered). This should always be null on a leader + List deletesAfter = null; + if (ulog != null && cmd.version > 0) { + deletesAfter = ulog.getDBQNewer(cmd.version); } - Document luceneDocument = cmd.getLuceneDocument(); - // SolrCore.verbose("updateDocument",updateTerm,luceneDocument,writer); - writer.updateDocument(updateTerm, luceneDocument); - // SolrCore.verbose("updateDocument",updateTerm,"DONE"); + if (deletesAfter != null) { + log.info("Reordered DBQs detected. Update="+cmd+" DBQs="+deletesAfter); + List dbqList = new ArrayList(deletesAfter.size()); + for (UpdateLog.DBQ dbq : deletesAfter) { + try { + DeleteUpdateCommand tmpDel = new DeleteUpdateCommand(cmd.req); + tmpDel.query = dbq.q; + tmpDel.version = -dbq.version; + dbqList.add( getQuery(tmpDel) ); + } catch (Exception e) { + log.error("Exception parsing reordered query : " + dbq, e); + } + } + + addAndDelete(cmd, dbqList); + } else { + // normal update + + Term updateTerm; + Term idTerm = new Term(idField.getName(), cmd.getIndexedId()); + boolean del = false; + if (cmd.updateTerm == null) { + updateTerm = idTerm; + } else { + del = true; + updateTerm = cmd.updateTerm; + } + + Document luceneDocument = cmd.getLuceneDocument(); + // SolrCore.verbose("updateDocument",updateTerm,luceneDocument,writer); + writer.updateDocument(updateTerm, luceneDocument, schema.getAnalyzer()); + // SolrCore.verbose("updateDocument",updateTerm,"DONE"); + + + if(del) { // ensure id remains unique + BooleanQuery bq = new BooleanQuery(); + bq.add(new BooleanClause(new TermQuery(updateTerm), Occur.MUST_NOT)); + bq.add(new BooleanClause(new TermQuery(idTerm), Occur.MUST)); + writer.deleteDocuments(bq); + } + - if(del) { // ensure id remains unique - BooleanQuery bq = new BooleanQuery(); - bq.add(new BooleanClause(new TermQuery(updateTerm), Occur.MUST_NOT)); - bq.add(new BooleanClause(new TermQuery(idTerm), Occur.MUST)); - writer.deleteDocuments(bq); + // Add to the transaction log *after* successfully adding to the index, if there was no error. + // This ordering ensures that if we log it, it's definitely been added to the the index. + // This also ensures that if a commit sneaks in-between, that we know everything in a particular + // log version was definitely committed. + if (ulog != null) ulog.add(cmd); } + } else { // allow duplicates - writer.addDocument(cmd.getLuceneDocument()); + writer.addDocument(cmd.getLuceneDocument(), schema.getAnalyzer()); + if (ulog != null) ulog.add(cmd); } - // Add to the transaction log *after* successfully adding to the index, if there was no error. - // This ordering ensures that if we log it, it's definitely been added to the the index. - // This also ensures that if a commit sneaks in-between, that we know everything in a particular - // log version was definitely committed. - if (ulog != null) ulog.add(cmd); if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) { commitTracker.addedDocument( -1 ); @@ -245,6 +276,43 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState updateDeleteTrackers(cmd); } + + public void clearIndex() throws IOException { + deleteAll(); + if (ulog != null) { + ulog.deleteAll(); + } + } + + + private Query getQuery(DeleteUpdateCommand cmd) { + Query q; + try { + // move this higher in the stack? + QParser parser = QParser.getParser(cmd.getQuery(), "lucene", cmd.req); + q = parser.getQuery(); + q = QueryUtils.makeQueryable(q); + + // Make sure not to delete newer versions + if (ulog != null && cmd.getVersion() != 0 && cmd.getVersion() != -Long.MAX_VALUE) { + BooleanQuery bq = new BooleanQuery(); + bq.add(q, Occur.MUST); + SchemaField sf = ulog.getVersionInfo().getVersionField(); + ValueSource vs = sf.getType().getValueSource(sf, null); + ValueSourceRangeFilter filt = new ValueSourceRangeFilter(vs, null, Long.toString(Math.abs(cmd.getVersion())), true, true); + FunctionRangeQuery range = new FunctionRangeQuery(filt); + bq.add(range, Occur.MUST); + q = bq; + } + + return q; + + } catch (ParseException e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); + } + } + + // we don't return the number of docs deleted because it's not always possible to quickly know that info. @Override public void deleteByQuery(DeleteUpdateCommand cmd) throws IOException { @@ -252,35 +320,19 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState deleteByQueryCommandsCumulative.incrementAndGet(); boolean madeIt=false; try { - Query q; - try { - // TODO: move this higher in the stack? - QParser parser = QParser.getParser(cmd.query, "lucene", cmd.req); - q = parser.getQuery(); - q = QueryUtils.makeQueryable(q); - - // peer-sync can cause older deleteByQueries to be executed and could - // delete newer documents. We prevent this by adding a clause restricting - // version. - if ((cmd.getFlags() & UpdateCommand.PEER_SYNC) != 0) { - BooleanQuery bq = new BooleanQuery(); - bq.add(q, Occur.MUST); - SchemaField sf = core.getSchema().getField(VersionInfo.VERSION_FIELD); - ValueSource vs = sf.getType().getValueSource(sf, null); - ValueSourceRangeFilter filt = new ValueSourceRangeFilter(vs, null, Long.toString(Math.abs(cmd.version)), true, true); - FunctionRangeQuery range = new FunctionRangeQuery(filt); - bq.add(range, Occur.MUST); - q = bq; - } - - - - } catch (ParseException e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); - } + Query q = getQuery(cmd); boolean delAll = MatchAllDocsQuery.class == q.getClass(); + // currently for testing purposes. Do a delete of complete index w/o worrying about versions, don't log, clean up most state in update log, etc + if (delAll && cmd.getVersion() == -Long.MAX_VALUE) { + synchronized (this) { + deleteAll(); + ulog.deleteAll(); + return; + } + } + // // synchronized to prevent deleteByQuery from running during the "open new searcher" // part of a commit. DBQ needs to signal that a fresh reader will be needed for @@ -309,6 +361,31 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState } } + + + /** Add a document execute the deletes as atomically as possible */ + private void addAndDelete(AddUpdateCommand cmd, List dbqList) throws IOException { + Document luceneDocument = cmd.getLuceneDocument(); + Term idTerm = new Term(idField.getName(), cmd.getIndexedId()); + + // see comment in deleteByQuery + synchronized (this) { + IndexWriter writer = solrCoreState.getIndexWriter(core); + + writer.updateDocument(idTerm, luceneDocument, core.getSchema().getAnalyzer()); + + for (Query q : dbqList) { + writer.deleteDocuments(q); + } + + if (ulog != null) ulog.add(cmd, true); + } + + } + + + + @Override public int mergeIndexes(MergeIndexesCommand cmd) throws IOException { mergeIndexesCommands.incrementAndGet(); diff --git a/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java b/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java index 06e8b54..9a0b4cb 100644 --- a/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java +++ b/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/MergeIndexesCommand.java b/solr/core/src/java/org/apache/solr/update/MergeIndexesCommand.java index 28053e0..e060a41 100644 --- a/solr/core/src/java/org/apache/solr/update/MergeIndexesCommand.java +++ b/solr/core/src/java/org/apache/solr/update/MergeIndexesCommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/PeerSync.java b/solr/core/src/java/org/apache/solr/update/PeerSync.java index abff7ba..49f96eb 100644 --- a/solr/core/src/java/org/apache/solr/update/PeerSync.java +++ b/solr/core/src/java/org/apache/solr/update/PeerSync.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -27,12 +27,10 @@ import java.util.List; import java.util.Set; import org.apache.http.NoHttpResponseException; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; -import org.apache.http.params.CoreConnectionPNames; +import org.apache.http.client.HttpClient; import org.apache.lucene.util.BytesRef; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.cloud.ZkController; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; @@ -80,17 +78,15 @@ public class PeerSync { private Set requestedUpdateSet; private long ourLowThreshold; // 20th percentile private long ourHighThreshold; // 80th percentile - private static ThreadSafeClientConnManager mgr = new ThreadSafeClientConnManager(); - private static DefaultHttpClient client = new DefaultHttpClient(mgr); + private static final HttpClient client; static { - mgr.setDefaultMaxPerRoute(20); - mgr.setMaxTotal(10000); - client.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 30000); - client.getParams().setIntParameter(CoreConnectionPNames.SO_TIMEOUT, 30000); - - // prevent retries (note: this didn't work when set on mgr.. needed to be set on client) - DefaultHttpRequestRetryHandler retryhandler = new DefaultHttpRequestRetryHandler(0, false); - client.setHttpRequestRetryHandler(retryhandler); + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 20); + params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 10000); + params.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, 30000); + params.set(HttpClientUtil.PROP_SO_TIMEOUT, 30000); + params.set(HttpClientUtil.PROP_USE_RETRY, false); + client = HttpClientUtil.createClient(params); } // comparator that sorts by absolute value, putting highest first diff --git a/solr/core/src/java/org/apache/solr/update/RollbackUpdateCommand.java b/solr/core/src/java/org/apache/solr/update/RollbackUpdateCommand.java index d0d0f5e..2766135 100644 --- a/solr/core/src/java/org/apache/solr/update/RollbackUpdateCommand.java +++ b/solr/core/src/java/org/apache/solr/update/RollbackUpdateCommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java index b0a82bd..a79e663 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java +++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java @@ -1,6 +1,6 @@ package org.apache.solr.update; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -34,8 +34,8 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.http.client.HttpClient; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.client.solrj.request.AbstractUpdateRequest; import org.apache.solr.client.solrj.request.UpdateRequestExt; @@ -45,24 +45,29 @@ import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; import org.apache.solr.util.DefaultSolrThreadFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SolrCmdDistributor { + private static final int MAX_RETRIES_ON_FORWARD = 6; + public static Logger log = LoggerFactory.getLogger(SolrCmdDistributor.class); + // TODO: shut this thing down // TODO: this cannot be per instance... static ThreadPoolExecutor commExecutor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 5, TimeUnit.SECONDS, new SynchronousQueue(), new DefaultSolrThreadFactory("cmdDistribExecutor")); - static HttpClient client; + static final HttpClient client; static { - ThreadSafeClientConnManager mgr = new ThreadSafeClientConnManager(); - mgr.setDefaultMaxPerRoute(8); - mgr.setMaxTotal(200); - client = new DefaultHttpClient(mgr); + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 200); + params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 8); + client = HttpClientUtil.createClient(params); } CompletionService completionService; @@ -167,7 +172,7 @@ public class SolrCmdDistributor { } private void doDelete(DeleteUpdateCommand cmd, List nodes, - ModifiableSolrParams params) throws IOException { + ModifiableSolrParams params) { flushAdds(1); @@ -351,11 +356,32 @@ public class SolrCmdDistributor { // error during request // if there is a retry url, we want to retry... - // TODO: but we really should only retry on connection errors... - if (sreq.retries < 5 && sreq.node.checkRetry()) { + boolean isRetry = sreq.node.checkRetry(); + boolean doRetry = false; + int rspCode = sreq.rspCode; + + // this can happen in certain situations such as shutdown + if (isRetry) { + if (rspCode == 404 || rspCode == 403 || rspCode == 503 + || rspCode == 500) { + doRetry = true; + } + + // if its an ioexception, lets try again + if (sreq.exception instanceof IOException) { + doRetry = true; + } else if (sreq.exception instanceof SolrServerException) { + if (((SolrServerException) sreq.exception).getRootCause() instanceof IOException) { + doRetry = true; + } + } + } + + if (isRetry && sreq.retries < MAX_RETRIES_ON_FORWARD && doRetry) { sreq.retries++; sreq.rspCode = 0; sreq.exception = null; + SolrException.log(SolrCmdDistributor.log, "forwarding update to " + sreq.node.getUrl() + " failed - retrying ... "); Thread.sleep(500); submit(sreq); checkResponses(block); @@ -366,7 +392,7 @@ public class SolrCmdDistributor { error.node = sreq.node; response.errors.add(error); response.sreq = sreq; - SolrException.log(SolrCore.log, "shard update error " + SolrException.log(SolrCmdDistributor.log, "shard update error " + sreq.node, sreq.exception); } } diff --git a/solr/core/src/java/org/apache/solr/update/SolrCoreState.java b/solr/core/src/java/org/apache/solr/update/SolrCoreState.java index 14922bb..d8d2e72 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrCoreState.java +++ b/solr/core/src/java/org/apache/solr/update/SolrCoreState.java @@ -1,6 +1,6 @@ package org.apache.solr.update; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java index 4f79f78..a5693c1 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -68,19 +68,17 @@ public class SolrIndexConfig { @SuppressWarnings("deprecation") private SolrIndexConfig(SolrConfig solrConfig) { luceneVersion = solrConfig.luceneMatchVersion; - useCompoundFile = luceneVersion.onOrAfter(Version.LUCENE_36) ? false : true; + useCompoundFile = false; maxBufferedDocs = -1; maxMergeDocs = -1; mergeFactor = -1; - ramBufferSizeMB = luceneVersion.onOrAfter(Version.LUCENE_36) ? 32 : 16; + ramBufferSizeMB = 32; writeLockTimeout = -1; - lockType = luceneVersion.onOrAfter(Version.LUCENE_36) ? - LOCK_TYPE_NATIVE : - LOCK_TYPE_SIMPLE; + lockType = LOCK_TYPE_NATIVE; termIndexInterval = IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL; mergePolicyInfo = null; mergeSchedulerInfo = null; - defaultMergePolicyClassName = luceneVersion.onOrAfter(Version.LUCENE_33) ? TieredMergePolicy.class.getName() : LogByteSizeMergePolicy.class.getName(); + defaultMergePolicyClassName = TieredMergePolicy.class.getName(); } /** @@ -106,13 +104,13 @@ public class SolrIndexConfig { // Warn for luceneMatchVersion's before LUCENE_36, fail fast above assertWarnOrFail("The myclass syntax is no longer supported in solrconfig.xml. Please use syntax instead.", !((solrConfig.get(prefix+"/mergeScheduler/text()",null) != null) && (solrConfig.get(prefix+"/mergeScheduler/@class",null) == null)), - luceneVersion.onOrAfter(Version.LUCENE_36)); + true); assertWarnOrFail("The myclass syntax is no longer supported in solrconfig.xml. Please use syntax instead.", !((solrConfig.get(prefix+"/mergePolicy/text()",null) != null) && (solrConfig.get(prefix+"/mergePolicy/@class",null) == null)), - luceneVersion.onOrAfter(Version.LUCENE_36)); + true); assertWarnOrFail("The true|false parameter is no longer valid in solrconfig.xml.", solrConfig.get(prefix+"/luceneAutoCommit", null) == null, - luceneVersion.onOrAfter(Version.LUCENE_36)); + true); defaultMergePolicyClassName = def.defaultMergePolicyClassName; useCompoundFile=solrConfig.getBool(prefix+"/useCompoundFile", def.useCompoundFile); @@ -157,7 +155,11 @@ public class SolrIndexConfig { } public IndexWriterConfig toIndexWriterConfig(IndexSchema schema) { - IndexWriterConfig iwc = new IndexWriterConfig(luceneVersion, schema.getAnalyzer()); + // so that we can update the analyzer on core reload, we pass null + // for the default analyzer, and explicitly pass an analyzer on + // appropriate calls to IndexWriter + + IndexWriterConfig iwc = new IndexWriterConfig(luceneVersion, null); if (maxBufferedDocs != -1) iwc.setMaxBufferedDocs(maxBufferedDocs); diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java b/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java index b9ca142..d3a9f92 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/TransactionLog.java b/solr/core/src/java/org/apache/solr/update/TransactionLog.java index 618b83a..23bcbc3 100644 --- a/solr/core/src/java/org/apache/solr/update/TransactionLog.java +++ b/solr/core/src/java/org/apache/solr/update/TransactionLog.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,7 +18,6 @@ package org.apache.solr.update; import org.apache.lucene.util.BytesRef; -import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.util.FastInputStream; @@ -27,13 +26,19 @@ import org.apache.solr.common.util.JavaBinCodec; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.nio.channels.FileChannel; -import java.rmi.registry.LocateRegistry; -import java.util.*; -import java.util.concurrent.CountDownLatch; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; /** @@ -129,11 +134,11 @@ public class TransactionLog { } - TransactionLog(File tlogFile, Collection globalStrings) throws IOException { + TransactionLog(File tlogFile, Collection globalStrings) { this(tlogFile, globalStrings, false); } - TransactionLog(File tlogFile, Collection globalStrings, boolean openExisting) throws IOException { + TransactionLog(File tlogFile, Collection globalStrings, boolean openExisting) { try { if (debug) { log.debug("New TransactionLog file=" + tlogFile + ", exists=" + tlogFile.exists() + ", size=" + tlogFile.length() + ", openExisting=" + openExisting); @@ -596,7 +601,7 @@ public class TransactionLog { ChannelFastInputStream fis; private LogCodec codec = new LogCodec() { @Override - public SolrInputDocument readSolrInputDocument(FastInputStream dis) throws IOException { + public SolrInputDocument readSolrInputDocument(FastInputStream dis) { // Given that the SolrInputDocument is last in an add record, it's OK to just skip // reading it completely. return null; diff --git a/solr/core/src/java/org/apache/solr/update/UpdateCommand.java b/solr/core/src/java/org/apache/solr/update/UpdateCommand.java index ece1f4d..6d5e15d 100644 --- a/solr/core/src/java/org/apache/solr/update/UpdateCommand.java +++ b/solr/core/src/java/org/apache/solr/update/UpdateCommand.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -34,6 +34,7 @@ public abstract class UpdateCommand implements Cloneable { public static int REPLAY = 0x00000002; // update command is from replaying a log. public static int PEER_SYNC = 0x00000004; // update command is a missing update being provided by a peer. public static int IGNORE_AUTOCOMMIT = 0x00000008; // this update should not count toward triggering of autocommits. + public static int CLEAR_CACHES = 0x00000010; // clear caches associated with the update log. used when applying reordered DBQ updates when doing an add. public UpdateCommand(SolrQueryRequest req) { this.req = req; diff --git a/solr/core/src/java/org/apache/solr/update/UpdateHandler.java b/solr/core/src/java/org/apache/solr/update/UpdateHandler.java index 9336f58..99de26b 100644 --- a/solr/core/src/java/org/apache/solr/update/UpdateHandler.java +++ b/solr/core/src/java/org/apache/solr/update/UpdateHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.java index 452201b..74e71cb 100644 --- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java +++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,10 +17,10 @@ package org.apache.solr.update; +import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; @@ -31,7 +31,6 @@ import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.search.SolrIndexSearcher; -import org.apache.solr.update.processor.DistributedUpdateProcessor; import org.apache.solr.update.processor.DistributedUpdateProcessorFactory; import org.apache.solr.update.processor.RunUpdateProcessorFactory; import org.apache.solr.update.processor.UpdateRequestProcessor; @@ -106,7 +105,9 @@ public class UpdateLog implements PluginInfoInitialized { private TransactionLog prevMapLog2; // the transaction log used to look up entries found in prevMap private final int numDeletesToKeep = 1000; + private final int numDeletesByQueryToKeep = 100; public final int numRecordsToKeep = 100; + // keep track of deletes only... this is not updated on an add private LinkedHashMap oldDeletes = new LinkedHashMap(numDeletesToKeep) { protected boolean removeEldestEntry(Map.Entry eldest) { @@ -114,6 +115,18 @@ public class UpdateLog implements PluginInfoInitialized { } }; + public class DBQ { + public String q; // the query string + public long version; // positive version of the DBQ + + @Override + public String toString() { + return "DBQ{version=" + version + ",q="+q+"}"; + } + } + + private LinkedList deleteByQueries = new LinkedList(); + private String[] tlogFiles; private File tlogDir; private Collection globalStrings; @@ -209,6 +222,16 @@ public class UpdateLog implements PluginInfoInitialized { DeleteUpdate du = startingUpdates.deleteList.get(i); oldDeletes.put(new BytesRef(du.id), new LogPtr(-1,du.version)); } + + // populate recent deleteByQuery commands + for (int i=startingUpdates.deleteByQueryList.size()-1; i>=0; i--) { + Update update = startingUpdates.deleteByQueryList.get(i); + List dbq = (List) update.log.lookup(update.pointer); + long version = (Long) dbq.get(1); + String q = (String) dbq.get(2); + trackDeleteByQuery(q, version); + } + } finally { startingUpdates.close(); } @@ -282,6 +305,11 @@ public class UpdateLog implements PluginInfoInitialized { public void add(AddUpdateCommand cmd) { + add(cmd, false); + } + + + public void add(AddUpdateCommand cmd, boolean clearCaches) { // don't log if we are replaying from another log // TODO: we currently need to log to maintain correct versioning, rtg, etc // if ((cmd.getFlags() & UpdateCommand.REPLAY) != 0) return; @@ -295,21 +323,45 @@ public class UpdateLog implements PluginInfoInitialized { pos = tlog.write(cmd, operationFlags); } - // TODO: in the future we could support a real position for a REPLAY update. - // Only currently would be useful for RTG while in recovery mode though. - LogPtr ptr = new LogPtr(pos, cmd.getVersion()); + if (!clearCaches) { + // TODO: in the future we could support a real position for a REPLAY update. + // Only currently would be useful for RTG while in recovery mode though. + LogPtr ptr = new LogPtr(pos, cmd.getVersion()); - // only update our map if we're not buffering - if ((cmd.getFlags() & UpdateCommand.BUFFERING) == 0) { - map.put(cmd.getIndexedId(), ptr); - } + // only update our map if we're not buffering + if ((cmd.getFlags() & UpdateCommand.BUFFERING) == 0) { + map.put(cmd.getIndexedId(), ptr); + } + + if (trace) { + log.trace("TLOG: added id " + cmd.getPrintableId() + " to " + tlog + " " + ptr + " map=" + System.identityHashCode(map)); + } + + } else { + // replicate the deleteByQuery logic. See deleteByQuery for comments. + + if (map != null) map.clear(); + if (prevMap != null) prevMap.clear(); + if (prevMap2 != null) prevMap2.clear(); + + try { + RefCounted holder = uhandler.core.openNewSearcher(true, true); + holder.decref(); + } catch (Throwable e) { + SolrException.log(log, "Error opening realtime searcher for deleteByQuery", e); + } + + if (trace) { + log.trace("TLOG: added id " + cmd.getPrintableId() + " to " + tlog + " clearCaches=true"); + } - if (trace) { - log.trace("TLOG: added id " + cmd.getPrintableId() + " to " + tlog + " " + ptr + " map=" + System.identityHashCode(map)); } + } } + + public void delete(DeleteUpdateCommand cmd) { BytesRef br = cmd.getIndexedId(); @@ -350,7 +402,11 @@ public class UpdateLog implements PluginInfoInitialized { if ((cmd.getFlags() & UpdateCommand.BUFFERING) == 0) { // given that we just did a delete-by-query, we don't know what documents were // affected and hence we must purge our caches. - map.clear(); + if (map != null) map.clear(); + if (prevMap != null) prevMap.clear(); + if (prevMap2 != null) prevMap2.clear(); + + trackDeleteByQuery(cmd.getQuery(), cmd.getVersion()); // oldDeletes.clear(); @@ -375,6 +431,75 @@ public class UpdateLog implements PluginInfoInitialized { } } + /** currently for testing only */ + public void deleteAll() { + synchronized (this) { + + try { + RefCounted holder = uhandler.core.openNewSearcher(true, true); + holder.decref(); + } catch (Throwable e) { + SolrException.log(log, "Error opening realtime searcher for deleteByQuery", e); + } + + if (map != null) map.clear(); + if (prevMap != null) prevMap.clear(); + if (prevMap2 != null) prevMap2.clear(); + + oldDeletes.clear(); + deleteByQueries.clear(); + } + } + + + void trackDeleteByQuery(String q, long version) { + version = Math.abs(version); + DBQ dbq = new DBQ(); + dbq.q = q; + dbq.version = version; + + synchronized (this) { + if (deleteByQueries.isEmpty() || deleteByQueries.getFirst().version < version) { + // common non-reordered case + deleteByQueries.addFirst(dbq); + } else { + // find correct insertion point + ListIterator iter = deleteByQueries.listIterator(); + iter.next(); // we already checked the first element in the previous "if" clause + while (iter.hasNext()) { + DBQ oldDBQ = iter.next(); + if (oldDBQ.version < version) { + iter.previous(); + break; + } else if (oldDBQ.version == version && oldDBQ.q.equals(q)) { + // a duplicate + return; + } + } + iter.add(dbq); // this also handles the case of adding at the end when hasNext() == false + } + + if (deleteByQueries.size() > numDeletesByQueryToKeep) { + deleteByQueries.removeLast(); + } + } + } + + public List getDBQNewer(long version) { + synchronized (this) { + if (deleteByQueries.isEmpty() || deleteByQueries.getFirst().version < version) { + // fast common case + return null; + } + + List dbqList = new ArrayList(); + for (DBQ dbq : deleteByQueries) { + if (dbq.version <= version) break; + dbqList.add(dbq); + } + return dbqList; + } + } private void newMap() { prevMap2 = prevMap; @@ -632,11 +757,7 @@ public class UpdateLog implements PluginInfoInitialized { private void ensureLog() { if (tlog == null) { String newLogName = String.format(Locale.ENGLISH, LOG_FILENAME_PATTERN, TLOG_NAME, id); - try { - tlog = new TransactionLog(new File(tlogDir, newLogName), globalStrings); - } catch (IOException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Can't open new tlog!", e); - } + tlog = new TransactionLog(new File(tlogDir, newLogName), globalStrings); } } diff --git a/solr/core/src/java/org/apache/solr/update/VersionBucket.java b/solr/core/src/java/org/apache/solr/update/VersionBucket.java index 2a0fae1..f7c2c2a 100644 --- a/solr/core/src/java/org/apache/solr/update/VersionBucket.java +++ b/solr/core/src/java/org/apache/solr/update/VersionBucket.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/VersionInfo.java b/solr/core/src/java/org/apache/solr/update/VersionInfo.java index 9abce53..b9aaae4 100644 --- a/solr/core/src/java/org/apache/solr/update/VersionInfo.java +++ b/solr/core/src/java/org/apache/solr/update/VersionInfo.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -159,7 +159,7 @@ public class VersionInfo { ValueSource vs = versionField.getType().getValueSource(versionField, null); Map context = ValueSource.newContext(searcher); vs.createWeight(context, searcher); - FunctionValues fv = vs.getValues(context, searcher.getTopReaderContext().leaves()[(int)(lookup>>32)]); + FunctionValues fv = vs.getValues(context, searcher.getTopReaderContext().leaves().get((int)(lookup>>32))); long ver = fv.longVal((int)lookup); return ver; diff --git a/solr/core/src/java/org/apache/solr/update/processor/AbstractDefaultValueUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/AbstractDefaultValueUpdateProcessorFactory.java new file mode 100644 index 0000000..9c87ec9 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/update/processor/AbstractDefaultValueUpdateProcessorFactory.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update.processor; + +import java.io.IOException; + +import org.apache.solr.common.SolrException; +import static org.apache.solr.common.SolrException.ErrorCode.*; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.update.AddUpdateCommand; + +/** + *

      + * Base class that can be extended by any + * UpdateRequestProcessorFactory designed to add a default value + * to the document in an AddUpdateCommand when that field is not + * already specified. + *

      + *

      + * This base class handles initialization of the fieldName init + * param, and provides an {@link DefaultValueUpdateProcessor} that Factory + * subclasses may choose to return from their getInstance + * implementation. + *

      + */ +public abstract class AbstractDefaultValueUpdateProcessorFactory + extends UpdateRequestProcessorFactory { + + protected String fieldName = null; + + @SuppressWarnings("unchecked") + @Override + public void init(NamedList args) { + + Object obj = args.remove("fieldName"); + if (null == obj && null == fieldName) { + throw new SolrException + (SERVER_ERROR, "'fieldName' init param must be specified and non-null"); + } else { + fieldName = obj.toString(); + } + + if (0 < args.size()) { + throw new SolrException(SERVER_ERROR, + "Unexpected init param(s): '" + + args.getName(0) + "'"); + } + + super.init(args); + } + + /** + * A simple processor that adds the results of {@link #getDefaultValue} + * to any document which does not already have a value in + * fieldName + */ + protected static abstract class DefaultValueUpdateProcessor + extends UpdateRequestProcessor { + + final String fieldName; + + public DefaultValueUpdateProcessor(final String fieldName, + final UpdateRequestProcessor next) { + super(next); + this.fieldName = fieldName; + } + + @Override + public void processAdd(AddUpdateCommand cmd) throws IOException { + final SolrInputDocument doc = cmd.getSolrInputDocument(); + + if (! doc.containsKey(fieldName)) { + doc.addField(fieldName, getDefaultValue()); + } + + super.processAdd(cmd); + } + + public abstract Object getDefaultValue(); + } +} + + + diff --git a/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java new file mode 100644 index 0000000..5a9d8ac --- /dev/null +++ b/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java @@ -0,0 +1,265 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.update.processor; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.ArrayList; +import java.util.List; +import java.util.HashSet; + +import org.apache.solr.schema.IndexSchema; + +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.util.plugin.SolrCoreAware; + +import org.apache.solr.common.util.NamedList; + +import org.apache.solr.common.SolrInputField; +import org.apache.solr.common.SolrInputDocument; + +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import static org.apache.solr.common.SolrException.ErrorCode.*; + +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; + +import org.apache.solr.update.AddUpdateCommand; + +import org.apache.solr.update.processor.FieldMutatingUpdateProcessorFactory; +import org.apache.solr.update.processor.FieldMutatingUpdateProcessorFactory.SelectorParams; +import org.apache.solr.update.processor.FieldMutatingUpdateProcessor.FieldNameSelector; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Clones the values found in any matching source field into + * the configured dest field. + *

      + * While the dest field must be a single <str>, + * the source fields can be configured as either: + *

      + *
        + *
      • One or more <str>
        • + *
      • An <arr> of <str>
        • + *
      • A <lst> containing {@link FieldMutatingUpdateProcessorFactory FieldMutatingUpdateProcessorFactory style selector arguments}
        • + *
      + *

      + * If the dest field already exists in the document, then the + * values from the source fields will be added to it. The + * "boost" value associated with the dest will not be changed, + * and any boost specified on the source fields will be ignored. + * (If the dest field did not exist prior to this processor, the + * newly created dest field will have the default boost of 1.0) + *

      + *

      + * In the example below, the category field will be cloned + * into the category_s field, both the authors and + * editors fields will be cloned into the contributors + * field, and any field with a name ending in _price -- except for + * list_price -- will be cloned into the all_prices + * field. + *

      + * + * 
      + *   <updateRequestProcessorChain name="multiple-clones">
+ *     <processor class="solr.CloneFieldUpdateProcessorFactory">
+ *       <str name="source">category</str>
+ *       <str name="dest">category_s</str>
+ *     </processor>
+ *     <processor class="solr.CloneFieldUpdateProcessorFactory">
+ *       <arr name="source">
+ *         <str>authors</str>
+ *         <str>editors</str>
+ *       </arr>
+ *       <str name="dest">contributors</str>
+ *     </processor>
+ *     <processor class="solr.CloneFieldUpdateProcessorFactory">
+ *       <lst name="source">
+ *         <str name="fieldRegex">.*_price</str>
+ *         <lst name="exclude">
+ *           <str name="fieldName">list_price</str>
+ *         </lst>
+ *       </lst>
+ *       <str name="dest">all_prices</str>
+ *     </processor>
+ *   </updateRequestProcessorChain>
+ *
      + */ +public class CloneFieldUpdateProcessorFactory + extends UpdateRequestProcessorFactory implements SolrCoreAware { + + private final static Logger log = LoggerFactory.getLogger(CloneFieldUpdateProcessorFactory.class); + + public static final String SOURCE_PARAM = "source"; + public static final String DEST_PARAM = "dest"; + + private SelectorParams srcInclusions = new SelectorParams(); + private Collection srcExclusions + = new ArrayList(); + + private FieldNameSelector srcSelector = null; + private String dest = null; + + protected final FieldNameSelector getSourceSelector() { + if (null != srcSelector) return srcSelector; + + throw new SolrException(SERVER_ERROR, "selector was never initialized, "+ + " inform(SolrCore) never called???"); + } + + @SuppressWarnings("unchecked") + @Override + public void init(NamedList args) { + Object d = args.remove(DEST_PARAM); + if (null == d) { + throw new SolrException + (SERVER_ERROR, "Init param '" + DEST_PARAM + "' must be specified"); + } else if (! (d instanceof CharSequence) ) { + throw new SolrException + (SERVER_ERROR, "Init param '" + DEST_PARAM + "' must be a string (ie: 'str')"); + } + dest = d.toString(); + + List sources = args.getAll(SOURCE_PARAM); + if (0 == sources.size()) { + throw new SolrException + (SERVER_ERROR, "Init param '" + SOURCE_PARAM + "' must be specified"); + } + if (1 == sources.size() && sources.get(0) instanceof NamedList) { + // nested set of selector options + NamedList selectorConfig = (NamedList) args.remove(SOURCE_PARAM); + + srcInclusions = parseSelectorParams(selectorConfig); + + List excList = selectorConfig.getAll("exclude"); + + for (Object excObj : excList) { + if (null == excObj) { + throw new SolrException + (SERVER_ERROR, "Init param '" + SOURCE_PARAM + + "' child 'exclude' can not be null"); + } + if (! (excObj instanceof NamedList) ) { + throw new SolrException + (SERVER_ERROR, "Init param '" + SOURCE_PARAM + + "' child 'exclude' must be "); + } + NamedList exc = (NamedList) excObj; + srcExclusions.add(parseSelectorParams(exc)); + if (0 < exc.size()) { + throw new SolrException(SERVER_ERROR, "Init param '" + SOURCE_PARAM + + "' has unexpected 'exclude' sub-param(s): '" + + selectorConfig.getName(0) + "'"); + } + // call once per instance + selectorConfig.remove("exclude"); + } + + if (0 < selectorConfig.size()) { + throw new SolrException(SERVER_ERROR, "Init param '" + SOURCE_PARAM + + "' contains unexpected child param(s): '" + + selectorConfig.getName(0) + "'"); + } + } else { + // source better be one or more strings + srcInclusions.fieldName = new HashSet + (FieldMutatingUpdateProcessorFactory.oneOrMany(args, "source")); + } + + + + if (0 < args.size()) { + throw new SolrException(SERVER_ERROR, + "Unexpected init param(s): '" + + args.getName(0) + "'"); + } + + super.init(args); + } + + public void inform(final SolrCore core) { + + final IndexSchema schema = core.getSchema(); + + srcSelector = + FieldMutatingUpdateProcessor.createFieldNameSelector + (core.getResourceLoader(), + core.getSchema(), + srcInclusions.fieldName, + srcInclusions.typeName, + srcInclusions.typeClass, + srcInclusions.fieldRegex, + FieldMutatingUpdateProcessor.SELECT_NO_FIELDS); + + for (SelectorParams exc : srcExclusions) { + srcSelector = FieldMutatingUpdateProcessor.wrap + (srcSelector, + FieldMutatingUpdateProcessor.createFieldNameSelector + (core.getResourceLoader(), + core.getSchema(), + exc.fieldName, + exc.typeName, + exc.typeClass, + exc.fieldRegex, + FieldMutatingUpdateProcessor.SELECT_NO_FIELDS)); + } + } + + @Override + public final UpdateRequestProcessor getInstance(SolrQueryRequest req, + SolrQueryResponse rsp, + UpdateRequestProcessor next) { + return new UpdateRequestProcessor(next) { + @Override + public void processAdd(AddUpdateCommand cmd) throws IOException { + + final SolrInputDocument doc = cmd.getSolrInputDocument(); + + // preserve initial values and boost (if any) + SolrInputField destField = doc.containsKey(dest) ? + doc.getField(dest) : new SolrInputField(dest); + + boolean modified = false; + for (final String fname : doc.getFieldNames()) { + if (! srcSelector.shouldMutate(fname)) continue; + + for (Object val : doc.getFieldValues(fname)) { + // preserve existing dest boost (multiplicitive), ignore src boost + destField.addValue(val, 1.0f); + } + modified=true; + } + + if (modified) doc.put(dest, destField); + + super.processAdd(cmd); + } + }; + } + + /** macro */ + private static SelectorParams parseSelectorParams(NamedList args) { + return FieldMutatingUpdateProcessorFactory.parseSelectorParams(args); + } + +} diff --git a/solr/core/src/java/org/apache/solr/update/processor/ConcatFieldUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/ConcatFieldUpdateProcessorFactory.java index 19dbdfd..b24a88b 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/ConcatFieldUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/ConcatFieldUpdateProcessorFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/DefaultValueUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/DefaultValueUpdateProcessorFactory.java new file mode 100644 index 0000000..d19d78a --- /dev/null +++ b/solr/core/src/java/org/apache/solr/update/processor/DefaultValueUpdateProcessorFactory.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update.processor; + +import java.io.IOException; + +import org.apache.solr.common.SolrException; +import static org.apache.solr.common.SolrException.ErrorCode.*; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.update.AddUpdateCommand; + +/** + *

      + * An update processor that adds a constant default value to any document + * being added that does not already have a value in the specified field. + *

      + * + *

      + * In the example configuration below, if a document does not contain a value + * in the price and/or type fields, it will be given + * default values of 0.0 and/or unknown + * (respectively). + *

      + * + * 

      + * <processor class="solr.DefaultValueUpdateProcessorFactory">
+ *   <str name="fieldName">price</str>
+ *   <float name="value">0.0</float>
+ * </processor>
+ * <processor class="solr.DefaultValueUpdateProcessorFactory">
+ *   <str name="fieldName">type</str>
+ *   <str name="value">unknown</str>
+ * </processor>
+ *
      + */ +public class DefaultValueUpdateProcessorFactory + extends AbstractDefaultValueUpdateProcessorFactory { + + protected Object defaultValue = null; + + @SuppressWarnings("unchecked") + @Override + public void init(NamedList args) { + + Object obj = args.remove("value"); + if (null == obj) { + throw new SolrException + (SERVER_ERROR, "'value' init param must be specified and non-null"); + } else { + defaultValue = obj; + } + + super.init(args); + } + + public UpdateRequestProcessor getInstance(SolrQueryRequest req, + SolrQueryResponse rsp, + UpdateRequestProcessor next ) { + return new DefaultValueUpdateProcessor(fieldName, next) { + public Object getDefaultValue() { return defaultValue; } + }; + } + +} + + + diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java index c10be7a..b40b321 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java @@ -1,6 +1,6 @@ package org.apache.solr.update.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -443,6 +443,9 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor { // realtime-get to work reliably. // TODO: if versions aren't stored, do we need to set on the cmd anyway for some reason? // there may be other reasons in the future for a version on the commands + + boolean checkDeleteByQueries = false; + if (versionsStored) { long bucketVersion = bucket.highest; @@ -484,7 +487,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor { // if we aren't the leader, then we need to check that updates were not re-ordered if (bucketVersion != 0 && bucketVersion < versionOnUpdate) { // we're OK... this update has a version higher than anything we've seen - // in this bucket so far, so we know that no reordering has yet occured. + // in this bucket so far, so we know that no reordering has yet occurred. bucket.updateHighest(versionOnUpdate); } else { // there have been updates higher than the current update. we need to check @@ -494,11 +497,16 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor { // This update is a repeat, or was reordered. We need to drop this update. return true; } + + // also need to re-apply newer deleteByQuery commands + checkDeleteByQueries = true; } } } + // TODO: possibly set checkDeleteByQueries as a flag on the command? doLocalAdd(cmd); + } // end synchronized (bucket) } finally { vinfo.unlockForUpdate(); @@ -654,7 +662,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor { // FROM: we are a replica receiving a DBQ from our leader // - log + execute the local DBQ DistribPhase phase = - DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM)); + DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM)); if (zkEnabled && DistribPhase.NONE == phase) { boolean leaderForAnyShard = false; // start off by assuming we are not a leader for any shard @@ -772,8 +780,6 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor { } - - // TODO: need to handle reorders to replicas somehow // forward to all replicas if (leaderLogic && replicas != null) { ModifiableSolrParams params = new ModifiableSolrParams(req.getParams()); diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessorFactory.java index 8a03e55..bc0403c 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessorFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.update.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/FieldLengthUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/FieldLengthUpdateProcessorFactory.java index 8ffa6fe..25ebc3f 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/FieldLengthUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/FieldLengthUpdateProcessorFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessor.java index 53ef98f..ac81762 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessorFactory.java index 018b96f..14a4b3d 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessorFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -108,7 +108,7 @@ public abstract class FieldMutatingUpdateProcessorFactory extends UpdateRequestProcessorFactory implements SolrCoreAware { - private static class SelectorParams { + public static final class SelectorParams { public Set fieldName = Collections.emptySet(); public Set typeName = Collections.emptySet(); public Collection typeClass = Collections.emptyList(); @@ -129,7 +129,7 @@ public abstract class FieldMutatingUpdateProcessorFactory } @SuppressWarnings("unchecked") - private static final SelectorParams parseSelectorParams(NamedList args) { + public static SelectorParams parseSelectorParams(NamedList args) { SelectorParams params = new SelectorParams(); params.fieldName = new HashSet(oneOrMany(args, "fieldName")); @@ -246,7 +246,7 @@ public abstract class FieldMutatingUpdateProcessorFactory * to one or more strings (or arrays of strings) * @exception SolrException invalid arr/str structure. */ - private static Collection oneOrMany(final NamedList args, final String key) { + public static Collection oneOrMany(final NamedList args, final String key) { List result = new ArrayList(args.size() / 2); final String err = "init arg '" + key + "' must be a string " + "(ie: 'str'), or an array (ie: 'arr') containing strings; found: "; diff --git a/solr/core/src/java/org/apache/solr/update/processor/FieldValueMutatingUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/FieldValueMutatingUpdateProcessor.java index 2782559..81ceafa 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/FieldValueMutatingUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/FieldValueMutatingUpdateProcessor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/FieldValueSubsetUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/FieldValueSubsetUpdateProcessorFactory.java index b5eb086..8a9919c 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/FieldValueSubsetUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/FieldValueSubsetUpdateProcessorFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/FirstFieldValueUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/FirstFieldValueUpdateProcessorFactory.java index 5243f73..a40dcf7 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/FirstFieldValueUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/FirstFieldValueUpdateProcessorFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/HTMLStripFieldUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/HTMLStripFieldUpdateProcessorFactory.java index d0dcb80..16be5fb 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/HTMLStripFieldUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/HTMLStripFieldUpdateProcessorFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/IgnoreFieldUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/IgnoreFieldUpdateProcessorFactory.java new file mode 100644 index 0000000..52f6ef4 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/update/processor/IgnoreFieldUpdateProcessorFactory.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update.processor; + +import org.apache.solr.core.SolrCore; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.FieldType; + +import org.apache.solr.common.SolrInputField; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; + +/** + * Ignores & removes fields matching the specified + * conditions from any document being added to the index. + * + *

      + * By default, this processor ignores any field name which does not + * exist according to the schema + *

      + * + *

      + * For example, in the configuration below, any field name which would cause + * an error because it does not exist, or match a dynamicField, in the + * schema.xml would be silently removed from any added documents... + *

      + * + * 
      + *  <updateRequestProcessorChain>
+ *    <processor class="solr.IgnoreFieldUpdateProcessorFactory" />
+ *  </updateRequestProcessorChain>
+ *
      + * + *

      + * In this second example, any field name ending in "_raw" found in a + * document being added would be removed... + *

      + * 
      + *  <updateRequestProcessorChain>
+ *    <processor class="solr.IgnoreFieldUpdateProcessorFactory">
+ *      <str name="fieldRegex">.*_raw</str>
+ *    </processor>
+ *  </updateRequestProcessorChain>
+ *
      + */ +public final class IgnoreFieldUpdateProcessorFactory extends FieldMutatingUpdateProcessorFactory { + + @Override + public UpdateRequestProcessor getInstance(SolrQueryRequest req, + SolrQueryResponse rsp, + UpdateRequestProcessor next) { + return new FieldMutatingUpdateProcessor(getSelector(), next) { + protected SolrInputField mutate(final SolrInputField src) { + return null; + } + }; + } + + @Override + public FieldMutatingUpdateProcessor.FieldNameSelector + getDefaultSelector(final SolrCore core) { + + final IndexSchema schema = core.getSchema(); + return new FieldMutatingUpdateProcessor.FieldNameSelector() { + public boolean shouldMutate(final String fieldName) { + + FieldType type = schema.getFieldTypeNoEx(fieldName); + return (null == type); + + } + }; + } + +} + diff --git a/solr/core/src/java/org/apache/solr/update/processor/LastFieldValueUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/LastFieldValueUpdateProcessorFactory.java index 491328e..2138d41 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/LastFieldValueUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/LastFieldValueUpdateProcessorFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/LogUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/LogUpdateProcessorFactory.java index 1fdfa09..dbb955d 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/LogUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/LogUpdateProcessorFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/Lookup3Signature.java b/solr/core/src/java/org/apache/solr/update/processor/Lookup3Signature.java index d2e5353..ab81262 100755 --- a/solr/core/src/java/org/apache/solr/update/processor/Lookup3Signature.java +++ b/solr/core/src/java/org/apache/solr/update/processor/Lookup3Signature.java @@ -1,5 +1,5 @@ package org.apache.solr.update.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/MD5Signature.java b/solr/core/src/java/org/apache/solr/update/processor/MD5Signature.java index 4ad6f37..be84930 100755 --- a/solr/core/src/java/org/apache/solr/update/processor/MD5Signature.java +++ b/solr/core/src/java/org/apache/solr/update/processor/MD5Signature.java @@ -1,5 +1,5 @@ package org.apache.solr.update.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/MaxFieldValueUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/MaxFieldValueUpdateProcessorFactory.java index 7ad030b..1ce15fa 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/MaxFieldValueUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/MaxFieldValueUpdateProcessorFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/MinFieldValueUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/MinFieldValueUpdateProcessorFactory.java index 995d4bd..fe4e23b 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/MinFieldValueUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/MinFieldValueUpdateProcessorFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/RegexReplaceProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/RegexReplaceProcessorFactory.java index 2b1ba1c..98c23f5 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/RegexReplaceProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/RegexReplaceProcessorFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -71,25 +71,26 @@ public final class RegexReplaceProcessorFactory extends FieldMutatingUpdateProce @Override public void init(NamedList args) { - String patternParam = args.remove(PATTERN_PARAM).toString(); + Object patternParam = args.remove(PATTERN_PARAM); if(patternParam == null) { throw new SolrException(ErrorCode.SERVER_ERROR, "Missing required init parameter: " + PATTERN_PARAM); } + try { - pattern = Pattern.compile(patternParam); + pattern = Pattern.compile(patternParam.toString()); } catch (PatternSyntaxException e) { throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid regex: " + patternParam, e); } - String replacementParam = args.remove(REPLACEMENT_PARAM).toString(); + Object replacementParam = args.remove(REPLACEMENT_PARAM); if(replacementParam == null) { throw new SolrException(ErrorCode.SERVER_ERROR, "Missing required init parameter: " + REPLACEMENT_PARAM); } - replacement = Matcher.quoteReplacement(replacementParam); + replacement = Matcher.quoteReplacement(replacementParam.toString()); super.init(args); } diff --git a/solr/core/src/java/org/apache/solr/update/processor/RemoveBlankFieldUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/RemoveBlankFieldUpdateProcessorFactory.java index 86a0ca4..079f09d 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/RemoveBlankFieldUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/RemoveBlankFieldUpdateProcessorFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/RunUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/RunUpdateProcessorFactory.java index bd9289d..d02e21c 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/RunUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/RunUpdateProcessorFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/Signature.java b/solr/core/src/java/org/apache/solr/update/processor/Signature.java index c196bcf..b9684f0 100755 --- a/solr/core/src/java/org/apache/solr/update/processor/Signature.java +++ b/solr/core/src/java/org/apache/solr/update/processor/Signature.java @@ -1,5 +1,5 @@ package org.apache.solr.update.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/SignatureUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/SignatureUpdateProcessorFactory.java index d63e0cb..98032b7 100755 --- a/solr/core/src/java/org/apache/solr/update/processor/SignatureUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/SignatureUpdateProcessorFactory.java @@ -1,5 +1,5 @@ package org.apache.solr.update.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/TextProfileSignature.java b/solr/core/src/java/org/apache/solr/update/processor/TextProfileSignature.java index e94a733..f459e22 100755 --- a/solr/core/src/java/org/apache/solr/update/processor/TextProfileSignature.java +++ b/solr/core/src/java/org/apache/solr/update/processor/TextProfileSignature.java @@ -1,5 +1,5 @@ package org.apache.solr.update.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/TimestampUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/TimestampUpdateProcessorFactory.java new file mode 100644 index 0000000..7ba90fd --- /dev/null +++ b/solr/core/src/java/org/apache/solr/update/processor/TimestampUpdateProcessorFactory.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update.processor; + +import java.io.IOException; +import java.util.Date; + +import org.apache.solr.common.SolrException; +import static org.apache.solr.common.SolrException.ErrorCode.*; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrRequestInfo; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.update.AddUpdateCommand; + +import org.apache.solr.common.params.CommonParams; // javadoc + +/** + *

      + * An update processor that adds a newly generated Date value + * of "NOW" to any document being added that does not already have a value + * in the specified field. + *

      + * + *

      + * In the example configuration below, if a document does not contain a value + * in the timestamp field, a new Date will be + * generated and added as the value of that field. + *

      + * + * 

      + * <processor class="solr.TimestampUpdateProcessorFactory">
+ *   <str name="fieldName">timestamp</str>
+ * </processor>
+ *
      + * + * @see Date + * @see CommonParams#NOW + */ +public class TimestampUpdateProcessorFactory + extends AbstractDefaultValueUpdateProcessorFactory { + + public UpdateRequestProcessor getInstance(SolrQueryRequest req, + SolrQueryResponse rsp, + UpdateRequestProcessor next ) { + return new DefaultValueUpdateProcessor(fieldName, next) { + public Object getDefaultValue() { + return SolrRequestInfo.getRequestInfo().getNOW(); + } + }; + } +} + + + diff --git a/solr/core/src/java/org/apache/solr/update/processor/TrimFieldUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/TrimFieldUpdateProcessorFactory.java index 1754ac3..d3f5028 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/TrimFieldUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/TrimFieldUpdateProcessorFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/TruncateFieldUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/TruncateFieldUpdateProcessorFactory.java new file mode 100644 index 0000000..226464c --- /dev/null +++ b/solr/core/src/java/org/apache/solr/update/processor/TruncateFieldUpdateProcessorFactory.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update.processor; + +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.util.NamedList; + +import org.apache.solr.core.SolrCore; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; + +/** + * Truncates any CharSequence values found in fields matching the specified + * conditions to a maximum character length. + *

      + * By default this processor matches no fields + *

      + * + *

      For example, with the configuration listed below any documents + * containing a String in any field declared in the schema using + * StrField will be truncated to no more then 100 characters + *

      + * 
      + * <processor class="solr.TruncateFieldUpdateProcessorFactory">
+ *   <str name="typeClass">solr.StrField</str>
+ *   <int name="maxLength">100</int>
+ * </processor>
+ *
      + */ +public final class TruncateFieldUpdateProcessorFactory + extends FieldMutatingUpdateProcessorFactory { + + private static final String MAX_LENGTH_PARAM = "maxLength"; + + private int maxLength = 0; + + @SuppressWarnings("unchecked") + @Override + public void init(NamedList args) { + + Object lengthParam = args.remove(MAX_LENGTH_PARAM); + if (null == lengthParam) { + throw new SolrException(ErrorCode.SERVER_ERROR, + "Missing required init parameter: " + + MAX_LENGTH_PARAM); + } + if ( ! (lengthParam instanceof Number) ) { + throw new SolrException(ErrorCode.SERVER_ERROR, + "Init param " + MAX_LENGTH_PARAM + + "must be a number; found: \"" + + lengthParam.toString()); + } + maxLength = ((Number)lengthParam).intValue(); + if (maxLength < 0) { + throw new SolrException(ErrorCode.SERVER_ERROR, + "Init param " + MAX_LENGTH_PARAM + + "must be >= 0; found: " + maxLength); + } + + super.init(args); + } + + @Override + public FieldMutatingUpdateProcessor.FieldNameSelector + getDefaultSelector(final SolrCore core) { + + return FieldMutatingUpdateProcessor.SELECT_NO_FIELDS; + } + + @Override + public UpdateRequestProcessor getInstance(SolrQueryRequest req, + SolrQueryResponse rsp, + UpdateRequestProcessor next) { + return new FieldValueMutatingUpdateProcessor(getSelector(), next) { + protected Object mutateValue(final Object src) { + if (src instanceof CharSequence) { + CharSequence s = (CharSequence)src; + if (maxLength < s.length()) { + return s.subSequence(0, maxLength); + } + } + return src; + } + }; + } +} + diff --git a/solr/core/src/java/org/apache/solr/update/processor/URLClassifyProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/URLClassifyProcessor.java index 2376ca0..894ff2d 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/URLClassifyProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/URLClassifyProcessor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/URLClassifyProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/URLClassifyProcessorFactory.java index 79eb8e7..e8d8c55 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/URLClassifyProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/URLClassifyProcessorFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/UUIDUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/UUIDUpdateProcessorFactory.java new file mode 100644 index 0000000..0d3fd40 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/update/processor/UUIDUpdateProcessorFactory.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update.processor; + +import java.io.IOException; +import java.util.UUID; +import java.util.Locale; + +import org.apache.solr.common.SolrException; +import static org.apache.solr.common.SolrException.ErrorCode.*; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.update.AddUpdateCommand; + + +/** + *

      + * An update processor that adds a newly generated UUID value + * to any document being added that does not already have a value in the + * specified field. + *

      + * + *

      + * In the example configuration below, if a document does not contain a value + * in the id field, a new UUID will be generated + * and added as the value of that field. + *

      + * + * 

      + * <processor class="solr.UUIDUpdateProcessorFactory">
+ *   <str name="fieldName">id</str>
+ * </processor>
+ *
      + * + * @see UUID + */ +public class UUIDUpdateProcessorFactory + extends AbstractDefaultValueUpdateProcessorFactory { + + public UpdateRequestProcessor getInstance(SolrQueryRequest req, + SolrQueryResponse rsp, + UpdateRequestProcessor next ) { + return new DefaultValueUpdateProcessor(fieldName, next) { + public Object getDefaultValue() { + return UUID.randomUUID().toString().toLowerCase(Locale.ENGLISH); + } + }; + } +} + + + diff --git a/solr/core/src/java/org/apache/solr/update/processor/UniqFieldsUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/UniqFieldsUpdateProcessorFactory.java index 5b139f0..07a6571 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/UniqFieldsUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/UniqFieldsUpdateProcessorFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.update.processor; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessor.java index a5a02cd..adf9c69 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java index 729e4a1..78cfda6 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java +++ b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorFactory.java index 3d1d985..da559ac 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/util/BCDUtils.java b/solr/core/src/java/org/apache/solr/util/BCDUtils.java index 66c52d3..c5c252d 100644 --- a/solr/core/src/java/org/apache/solr/util/BCDUtils.java +++ b/solr/core/src/java/org/apache/solr/util/BCDUtils.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/util/BoundedTreeSet.java b/solr/core/src/java/org/apache/solr/util/BoundedTreeSet.java index 4abe7b4..397b816 100644 --- a/solr/core/src/java/org/apache/solr/util/BoundedTreeSet.java +++ b/solr/core/src/java/org/apache/solr/util/BoundedTreeSet.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/util/ConcurrentLFUCache.java b/solr/core/src/java/org/apache/solr/util/ConcurrentLFUCache.java index bbb5964..567f9ef 100644 --- a/solr/core/src/java/org/apache/solr/util/ConcurrentLFUCache.java +++ b/solr/core/src/java/org/apache/solr/util/ConcurrentLFUCache.java @@ -1,5 +1,5 @@ package org.apache.solr.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/util/ConcurrentLRUCache.java b/solr/core/src/java/org/apache/solr/util/ConcurrentLRUCache.java index b98e790..0f44d89 100644 --- a/solr/core/src/java/org/apache/solr/util/ConcurrentLRUCache.java +++ b/solr/core/src/java/org/apache/solr/util/ConcurrentLRUCache.java @@ -1,5 +1,5 @@ package org.apache.solr.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/util/DateMathParser.java b/solr/core/src/java/org/apache/solr/util/DateMathParser.java index 404443c..1efe844 100644 --- a/solr/core/src/java/org/apache/solr/util/DateMathParser.java +++ b/solr/core/src/java/org/apache/solr/util/DateMathParser.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/util/DefaultSolrThreadFactory.java b/solr/core/src/java/org/apache/solr/util/DefaultSolrThreadFactory.java index f97ba9b..2e21108 100644 --- a/solr/core/src/java/org/apache/solr/util/DefaultSolrThreadFactory.java +++ b/solr/core/src/java/org/apache/solr/util/DefaultSolrThreadFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/util/LongPriorityQueue.java b/solr/core/src/java/org/apache/solr/util/LongPriorityQueue.java index f0f04ae..5833249 100755 --- a/solr/core/src/java/org/apache/solr/util/LongPriorityQueue.java +++ b/solr/core/src/java/org/apache/solr/util/LongPriorityQueue.java @@ -2,7 +2,7 @@ package org.apache.solr.util; import java.util.Arrays; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/util/NumberUtils.java b/solr/core/src/java/org/apache/solr/util/NumberUtils.java index 130f8e1..d8dd26b 100644 --- a/solr/core/src/java/org/apache/solr/util/NumberUtils.java +++ b/solr/core/src/java/org/apache/solr/util/NumberUtils.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/util/PrimUtils.java b/solr/core/src/java/org/apache/solr/util/PrimUtils.java index 11580c0..970de80 100644 --- a/solr/core/src/java/org/apache/solr/util/PrimUtils.java +++ b/solr/core/src/java/org/apache/solr/util/PrimUtils.java @@ -1,5 +1,5 @@ package org.apache.solr.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/util/RefCounted.java b/solr/core/src/java/org/apache/solr/util/RefCounted.java index 6cd5abc..f06228a 100644 --- a/solr/core/src/java/org/apache/solr/util/RefCounted.java +++ b/solr/core/src/java/org/apache/solr/util/RefCounted.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/util/SimplePostTool.java b/solr/core/src/java/org/apache/solr/util/SimplePostTool.java index c54c0c7..7863deb 100644 --- a/solr/core/src/java/org/apache/solr/util/SimplePostTool.java +++ b/solr/core/src/java/org/apache/solr/util/SimplePostTool.java @@ -1,6 +1,6 @@ package org.apache.solr.util; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -72,8 +72,8 @@ public class SimplePostTool { " java -jar post.jar *.xml\n"+ " java -Ddata=args -jar post.jar '42'\n"+ " java -Ddata=stdin -jar post.jar < hd.xml\n"+ - " java -Durl=http://localhost:8983/solr/update/csv -Dtype=text/csv -jar post.jar *.csv\n"+ - " java -Durl=http://localhost:8983/solr/update/json -Dtype=application/json -jar post.jar *.json\n"+ + " java -Dtype=text/csv -jar post.jar *.csv\n"+ + " java -Dtype=application/json -jar post.jar *.json\n"+ " java -Durl=http://localhost:8983/solr/update/extract?literal.id=a -Dtype=application/pdf -jar post.jar a.pdf\n"+ "Other options controlled by System Properties include the Solr\n"+ "URL to POST to, the Content-Type of the data, whether a commit\n"+ @@ -218,8 +218,7 @@ public class SimplePostTool { /** * Opens the file and posts it's contents to the solrUrl, - * writes to response to output. - * @throws UnsupportedEncodingException + * writes to response to output. */ public void postFile(File file, OutputStream output, String type) { diff --git a/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java b/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java index 1e3aaef..3b35ba9 100644 --- a/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java +++ b/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -434,9 +434,10 @@ public class SolrPluginUtils { * * @param fieldLists - an array of Strings eg. {"fieldOne^2.3", "fieldTwo", fieldThree~5^-0.4} * @param wordGrams - (0=all words, 2,3 = shingle size) + * @param defaultSlop - the default slop for this param * @return - FieldParams containing the fieldname,boost,slop,and shingle size */ - public static List parseFieldBoostsAndSlop(String[] fieldLists,int wordGrams) { + public static List parseFieldBoostsAndSlop(String[] fieldLists,int wordGrams,int defaultSlop) { if (null == fieldLists || 0 == fieldLists.length) { return new ArrayList(); } @@ -454,7 +455,7 @@ public class SolrPluginUtils { String[] fieldAndSlopVsBoost = caratPattern.split(s); String[] fieldVsSlop = tildePattern.split(fieldAndSlopVsBoost[0]); String field = fieldVsSlop[0]; - int slop = (2 == fieldVsSlop.length) ? Integer.valueOf(fieldVsSlop[1]) : 0; + int slop = (2 == fieldVsSlop.length) ? Integer.valueOf(fieldVsSlop[1]) : defaultSlop; Float boost = (1 == fieldAndSlopVsBoost.length) ? 1 : Float.valueOf(fieldAndSlopVsBoost[1]); FieldParams fp = new FieldParams(field,wordGrams,slop,boost); out.add(fp); diff --git a/solr/core/src/java/org/apache/solr/util/SuggestMissingFactories.java b/solr/core/src/java/org/apache/solr/util/SuggestMissingFactories.java index 013f6bc..fe2a5ce 100644 --- a/solr/core/src/java/org/apache/solr/util/SuggestMissingFactories.java +++ b/solr/core/src/java/org/apache/solr/util/SuggestMissingFactories.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/util/SystemIdResolver.java b/solr/core/src/java/org/apache/solr/util/SystemIdResolver.java index fbcb1ca..cf263aa 100644 --- a/solr/core/src/java/org/apache/solr/util/SystemIdResolver.java +++ b/solr/core/src/java/org/apache/solr/util/SystemIdResolver.java @@ -96,7 +96,7 @@ public final class SystemIdResolver implements EntityResolver, EntityResolver2 { }; } - URI resolveRelativeURI(String baseURI, String systemId) throws IOException,URISyntaxException { + URI resolveRelativeURI(String baseURI, String systemId) throws URISyntaxException { URI uri; // special case for backwards compatibility: if relative systemId starts with "/" (we convert that to an absolute solrres:-URI) diff --git a/solr/core/src/java/org/apache/solr/util/TimeZoneUtils.java b/solr/core/src/java/org/apache/solr/util/TimeZoneUtils.java index 5d47491..0aa5875 100644 --- a/solr/core/src/java/org/apache/solr/util/TimeZoneUtils.java +++ b/solr/core/src/java/org/apache/solr/util/TimeZoneUtils.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/util/VersionedFile.java b/solr/core/src/java/org/apache/solr/util/VersionedFile.java index ad39242..1c703ce 100644 --- a/solr/core/src/java/org/apache/solr/util/VersionedFile.java +++ b/solr/core/src/java/org/apache/solr/util/VersionedFile.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java b/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java index 3dd6906..47a59b8 100644 --- a/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java +++ b/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/util/plugin/MapInitializedPlugin.java b/solr/core/src/java/org/apache/solr/util/plugin/MapInitializedPlugin.java index 2bc1200..f7fbb60 100644 --- a/solr/core/src/java/org/apache/solr/util/plugin/MapInitializedPlugin.java +++ b/solr/core/src/java/org/apache/solr/util/plugin/MapInitializedPlugin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/util/plugin/MapPluginLoader.java b/solr/core/src/java/org/apache/solr/util/plugin/MapPluginLoader.java index dbcd788..c7026ad 100644 --- a/solr/core/src/java/org/apache/solr/util/plugin/MapPluginLoader.java +++ b/solr/core/src/java/org/apache/solr/util/plugin/MapPluginLoader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/util/plugin/NamedListInitializedPlugin.java b/solr/core/src/java/org/apache/solr/util/plugin/NamedListInitializedPlugin.java index ad8a533..6945bba 100644 --- a/solr/core/src/java/org/apache/solr/util/plugin/NamedListInitializedPlugin.java +++ b/solr/core/src/java/org/apache/solr/util/plugin/NamedListInitializedPlugin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/util/plugin/NamedListPluginLoader.java b/solr/core/src/java/org/apache/solr/util/plugin/NamedListPluginLoader.java index 5d89a66..e57c853 100644 --- a/solr/core/src/java/org/apache/solr/util/plugin/NamedListPluginLoader.java +++ b/solr/core/src/java/org/apache/solr/util/plugin/NamedListPluginLoader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/util/plugin/PluginInfoInitialized.java b/solr/core/src/java/org/apache/solr/util/plugin/PluginInfoInitialized.java index 2b2b020..811ed81 100644 --- a/solr/core/src/java/org/apache/solr/util/plugin/PluginInfoInitialized.java +++ b/solr/core/src/java/org/apache/solr/util/plugin/PluginInfoInitialized.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/util/plugin/SolrCoreAware.java b/solr/core/src/java/org/apache/solr/util/plugin/SolrCoreAware.java index 5462c92..8439717 100644 --- a/solr/core/src/java/org/apache/solr/util/plugin/SolrCoreAware.java +++ b/solr/core/src/java/org/apache/solr/util/plugin/SolrCoreAware.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/java/org/apache/solr/util/xslt/TransformerProvider.java b/solr/core/src/java/org/apache/solr/util/xslt/TransformerProvider.java index a6bcc14..686d2d3 100644 --- a/solr/core/src/java/org/apache/solr/util/xslt/TransformerProvider.java +++ b/solr/core/src/java/org/apache/solr/util/xslt/TransformerProvider.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -64,7 +64,7 @@ public class TransformerProvider { } /** Return a new Transformer, possibly created from our cached Templates object - * @throws TransformerConfigurationException + * @throws IOException */ public synchronized Transformer getTransformer(SolrConfig solrConfig, String filename,int cacheLifetimeSeconds) throws IOException { // For now, the Templates are blindly reloaded once cacheExpires is over. diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-mp-solrconfig.xml b/solr/core/src/test-files/solr/collection1/conf/bad-mp-solrconfig.xml new file mode 100644 index 0000000..af5d8fb --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-mp-solrconfig.xml @@ -0,0 +1,34 @@ + + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + 8 + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-dup-dynamicField.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-dup-dynamicField.xml new file mode 100644 index 0000000..460fbda --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-dup-dynamicField.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + id + id + + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-dup-field.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-dup-field.xml new file mode 100644 index 0000000..4272362 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-dup-field.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + id + id + + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-dup-fieldType.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-dup-fieldType.xml new file mode 100644 index 0000000..34ef44b --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-dup-fieldType.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + id + id + + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-external-filefield.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-external-filefield.xml new file mode 100644 index 0000000..e7874c8 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-external-filefield.xml @@ -0,0 +1,27 @@ + + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-nontext-analyzer.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-nontext-analyzer.xml new file mode 100644 index 0000000..06a689a --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-nontext-analyzer.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + id + id + + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-not-indexed-but-norms.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-not-indexed-but-norms.xml new file mode 100644 index 0000000..f7c4e9b --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-not-indexed-but-norms.xml @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + id + id + + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-not-indexed-but-pos.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-not-indexed-but-pos.xml new file mode 100644 index 0000000..774d587 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-not-indexed-but-pos.xml @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + id + id + + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-not-indexed-but-tf.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-not-indexed-but-tf.xml new file mode 100644 index 0000000..d153793 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-not-indexed-but-tf.xml @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + id + id + + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-omit-tf-but-not-pos.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-omit-tf-but-not-pos.xml new file mode 100644 index 0000000..116f116 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-omit-tf-but-not-pos.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + id + id + + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-sim-global-vs-ft-mismatch.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-sim-global-vs-ft-mismatch.xml new file mode 100644 index 0000000..a776d10 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-sim-global-vs-ft-mismatch.xml @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + sim1text + + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-uniquekey-is-copyfield-dest.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-uniquekey-is-copyfield-dest.xml new file mode 100644 index 0000000..bf1d532 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-uniquekey-is-copyfield-dest.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + id + id + + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-uniquekey-uses-default.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-uniquekey-uses-default.xml new file mode 100644 index 0000000..026b529 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-uniquekey-uses-default.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + id + id + + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad_solrconfig.xml b/solr/core/src/test-files/solr/collection1/conf/bad_solrconfig.xml new file mode 100644 index 0000000..8c8e960 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad_solrconfig.xml @@ -0,0 +1,31 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + ${unset.sys.property} + + diff --git a/solr/core/src/test-files/solr/collection1/conf/compoundDictionary.txt b/solr/core/src/test-files/solr/collection1/conf/compoundDictionary.txt new file mode 100644 index 0000000..f4977b5 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/compoundDictionary.txt @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A set of words for testing the DictionaryCompound factory +soft +ball +team diff --git a/solr/core/src/test-files/solr/collection1/conf/currency.xml b/solr/core/src/test-files/solr/collection1/conf/currency.xml new file mode 100644 index 0000000..f74f6e9 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/currency.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/da_UTF8.xml b/solr/core/src/test-files/solr/collection1/conf/da_UTF8.xml new file mode 100644 index 0000000..2c8d203 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/da_UTF8.xml @@ -0,0 +1,1208 @@ + + + + + + + + + + +aA +bB +cC +dD +eE +fF +gG +hH +iI +jJ +kK +lL +mM +nN +oO +pP +qQ +rR +sS +tT +uU +vV +wW +xX +yY +zZ +æÆ +øØ +åÅ + + + +.ae3 +.an3k +.an1s +.be5la +.be1t +.bi4tr +.der3i +.diagno5 +.her3 +.hoved3 +.ne4t5 +.om1 +.ove4 +.po1 +.til3 +.yd5r +ab5le +3abst +a3c +ade5la +5adg +a1e +5afg +5a4f1l +af3r +af4ri +5afs +a4gef +a4gi +ag5in +ag5si +3agti +a4gy +a3h +ais5t +a3j +a5ka +a3ke +a5kr +aku5 +a3la +a1le +a1li +al3k +4alkv +a1lo +al5si +a3lu +a1ly +am4pa +3analy +an4k5r +a3nu +3anv +a5o +a5pe +a3pi +a5po +a1ra +ar5af +1arb +a1re +5arg +a1ri +a3ro +a3sa +a3sc +a1si +a3sk +a3so +3a3sp +a3ste +a3sti +a1ta1 +a1te +a1ti +a4t5in +a1to +ato5v +a5tr +a1tu +a5va +a1ve +a5z +1ba +ba4ti +4bd +1be +be1k +be3ro +be5ru +be1s4 +be1tr +1bi +bi5sk +b1j +4b1n +1bo +bo4gr +bo3ra +bo5re +1br4 +4bs +bs5k +b3so +b1st +b5t +3bu +bu4s5tr +b5w +1by +by5s +4c1c +1ce +ce5ro +3ch +4ch. +ci4o +ck3 +5cy +3da +4d3af +d5anta +da4s +d1b +d1d4 +1de +de5d +4de4lem +der5eri +de4rig +de5sk +d1f +d1g +d3h +1di +di1e +di5l +d3j +d1k +d1l +d1m +4d1n +3do +4dop +d5ov +d1p +4drett +5d4reve +3drif +3driv +d5ros +d5ru +ds5an +ds5in +d1ski +d4sm +d4su +dsu5l +ds5vi +d3ta +d1te +dt5o +d5tr +dt5u +1du +dub5 +d1v +3dy +e5ad +e3af +e5ag +e3ak +e1al +ea4la +e3an +e5ap +e3at +e3bl +ebs3 +e1ci +ed5ar +edde4 +eddel5 +e4do +ed5ra +ed3re +ed3rin +ed4str +e3e +3eff +e3fr +3eft +e3gu +e1h +e3in +ei5s +e3je +e4j5el +e1ka +e3ke +e3kl +4e1ko +e5kr +ek5sa +3eksem +3eksp +e3ku +e1kv +e5ky +e3lad +el3ak +el3ar +e1las +e3le +e4lek +3elem +e1li +5elim +e3lo +el5sa +e5lu +e3ly +e4mad +em4p5le +em1s +en5ak +e4nan +4enn +e4no +en3so +e5nu +e5ol +e3op +e1or +e3ov +epi3 +e1pr +e3ra +er3af +e4rag +e4rak +e1re +e4ref +er5ege +5erhv +e1ri +e4rib +er1k +ero5d +er5ov +er3s +er5tr +e3rum +er5un +e5ry +e1ta +e1te +etek4s +e1ti +e3tj +e1to +e3tr +e3tu +e1ty +e3um +e3un +3eur +e1va +e3ve +e4v3erf +e1vi +e5x +1fa +fa4ce +fags3 +f1b +f1d +1fe +fej4 +fejl1 +f1f +f1g +f1h +1fi +f1k +3fl +1fo +for1en +fo4ri +f1p +f1s4 +4ft +f3ta +f1te +f1ti +f5to +f5tvi +1fu +f1v +3fy +1ga +g3art +g1b +g1d +1ge +4g5enden +ger3in +ge3s +g3f +g1g +g1h +1gi +gi4b +gi3st +5gj +g3k +g1l +g1m +3go +4g5om +g5ov +g3p +1gr +gs1a +gsde4len +g4se +gsha4 +g5sla +gs3or +gs1p +g5s4tide +g4str +gs1v +g3ta +g1te +g1ti +g5to +g3tr +gt4s +g3ud +gun5 +g3v +1gy +g5yd +4ha. +heds3 +he5s +4het +hi4e +hi4n5 +hi3s +ho5ko +ho5ve +4h3t +hun4 +hund3 +hvo4 +i1a +i3b +i4ble +i1c +i3dr +ids5k +i1el +i1en +i3er +i3et. +if3r +i3gu +i3h +i5i +i5j +i1ka +i1ke +ik1l +i5ko +ik3re +ik5ri +iks5t +ik4tu +i3ku +ik3v +i3lag +il3eg +il5ej +il5el +i3li +i4l5id +il3k +i1lo +il5u +i3mu +ind3t +5inf +ings1 +in3s +in4sv +inter1 +i3nu +i3od +i3og +i5ok +i3ol +ion4 +ions1 +i5o5r +i3ot +i5pi +i3pli +i5pr +i3re +i3ri +ir5t +i3sc +i3si +i4sm +is3p +i1ster +i3sti +i5sua +i1ta +i1te +i1ti +i3to +i3tr +it5re. +i1tu +i3ty +i1u +i1va +i1ve +i1vi +j3ag +jde4rer +jds1 +jek4to +4j5en. +j5k +j3le +j3li +jlmeld5 +jlmel4di +j3r +jre5 +ju3s +5kap +k5au +5kav +k5b +kel5s +ke3sk +ke5st +ke4t5a +k3h +ki3e +ki3st +k1k +k5lak +k1le +3klu +k4ny +5kod +1kon +ko3ra +3kort +ko3v +1kra +5kry +ks3an +k1si +ks3k +ks1p +k3ste +k5stu +ks5v +k1t +k4tar +k4terh +kti4e +kt5re +kt5s +3kur +1kus +3kut +k4vo +k4vu +5lab +lad3r +5lagd +la4g3r +5lam +1lat +l1b +ldiagnos5 +l3dr +ld3st +1le. +5led +4lele +le4mo +3len +1ler +1les +4leu +l1f +lfin4 +lfind5 +l1go1 +l3h +li4ga +4l5ins +4l3int +li5o +l3j +l1ke +l1ko +l3ky +l1l +l5mu +lo4du +l3op +4l5or +3lov +4l3p +l4ps +l3r +4ls +lses1 +ls5in +l5sj +l1ta +l4taf +l1te +l4t5erf +l3ti +lt3o +l3tr +l3tu +lu5l +l3ve +l3vi +1ma +m1b +m3d +1me +4m5ej +m3f +m1g +m3h +1mi +mi3k +m5ing +mi4o +mi5sty +m3k +m1l +m1m +mmen5 +m1n +3mo +mo4da +4mop +4m5ov +m1pe +m3pi +m3pl +m1po +m3pr +m1r +mse5s +ms5in +m5sk +ms3p +m3ste +ms5v +m3ta +m3te +m3ti +m3tr +m1ud +1mul +mu1li +3my +3na +4nak +1nal +n1b +n1c +4nd +n3dr +nd5si +nd5sk +nd5sp +1ne +ne5a +ne4da +nemen4 +nement5e +neo4 +n3erk +n5erl +ne5sl +ne5st +n1f +n4go +4n1h +1ni +4nim +ni5o +ni3st +n1ke +n1ko +n3kr +n3ku +n5kv +4n1l +n1m +n1n +1no +n3ord +n5p +n3r +4ns +n3si +n1sku +ns3po +n1sta +n5sti +n1ta +nta4le +n1te +n1ti +ntiali4 +n3to +n1tr +nt4s5t +nt4su +n3tu +n3ty +4n1v +3ny +n3z +o3a +o4as +ob3li +o1c +o4din +od5ri +od5s +od5un +o1e +of5r +o4gek +o4gel +o4g5o +og5re +og5sk +o5h +o5in +oi6s5e +o1j +o3ka +o1ke +o3ku +o3la +o3le +o1li +o1lo +o3lu +o5ly +1omr +on3k +ook5 +o3or +o5ov +o3pi +op3l +op3r +op3s +3opta +4or. +or1an +3ordn +ord5s +o3re. +o3reg +o3rek +o3rer +o3re3s +o3ret +o3ri +3orient +or5im +o4r5in +or3k +or5o +or3sl +or3st +o3si +o3so +o3t +o1te +o5un +ov4s +3pa +pa5gh +p5anl +p3d +4pec +3pen +1per +pe1ra +pe5s +pe3u +p3f +4p5h +1pla +p4lan +4ple. +4pler +4ples +p3m +p3n +5pok +4po3re +3pot +4p5p4 +p4ro +1proc +p3sk +p5so +ps4p +p3st +p1t +1pu +pu5b +p5ule +p5v +5py3 +qu4 +4raf +ra5is +4rarb +r1b +r4d5ar +r3dr +rd4s3 +4reks +1rel +re5la +r5enss +5rese +re5spo +4ress +re3st +re5s4u +5rett +r1f +r1gu +r1h +ri1e +ri5la +4rimo +r4ing +ringse4 +ringso4r +4rinp +4rint +r3ka +r1ke +r1ki +rk3so +r3ku +r1l +rmo4 +r5mu +r1n +ro1b +ro3p +r3or +r3p +r1r +rre5s +rro4n5 +r1sa +r1si +r5skr +r4sk5v +rs4n +r3sp +r5stu +r5su +r3sv +r5tal +r1te +r4teli +r1ti +r3to +r4t5or +rt5rat +rt3re +r5tri +r5tro +rt3s +r5ty +r3ud +run4da +5rut +r3va +r1ve +r3vi +ry4s +s3af +1sam +sa4ma +s3ap +s1ar +1sat +4s1b +s1d +sdy4 +1se +s4ed +5s4er +se4se +s1f +4s1g4 +4s3h +si4bl +1sig +s5int +5sis +5sit +5siu +s5ju +4sk. +1skab +1ske +s3kl +sk5s4 +5sky +s1le +s1li +slo3 +5slu +s5ly +s1m +s4my +4snin +s4nit +so5k +5sol +5som. +3somm +s5oms +5somt +3son +4s1op +sp4 +3spec +4sper +3s4pi +s1pl +3sprog. +s5r4 +s1s4 +4st. +5s4tam +1stan +st5as +3stat +1stav +1ste. +1sted +3stel +5stemo +1sten +5step +3ster. +3stes +5stet +5stj +3sto +st5om +1str +s1ud +3sul +s3un +3sur +s3ve +3s4y +1sy1s +5ta. +1tag +tands3 +4tanv +4tb +tede4l +teds5 +3teg +5tekn +teo1 +5term +te5ro +4t1f +6t3g +t1h +tialis5t +3tid +ti4en +ti3st +4t3k +4t1l +tli4s5 +t1m +t1n +to5ra +to1re +to1ri +tor4m +4t3p +t4ra +4tres +tro5v +1try +4ts +t3si +ts4pa +ts5pr +t3st +ts5ul +4t1t +t5uds +5tur +t5ve +1typ +u1a +5udl +ud5r +ud3s +3udv +u1e +ue4t5 +uge4ri +ugs3 +u5gu +u3i +u5kl +uk4ta +uk4tr +u1la +u1le +u5ly +u5pe +up5l +u5q +u3ra +u3re +u4r3eg +u1rer +u3ro +us5a +u3si +u5ska +u5so +us5v +u1te +u1ti +u1to +ut5r +ut5s4 +5u5v +va5d +3varm +1ved +ve4l5e +ve4reg +ve3s +5vet +v5h +vi4l3in +1vis +v5j +v5k +vl4 +v3le +v5li +vls1 +1vo +4v5om +v5p +v5re +v3st +v5su +v5t +3vu +y3a +y5dr +y3e +y3ke +y5ki +yk3li +y3ko +yk4s5 +y3kv +y5li +y5lo +y5mu +yns5 +y5o +y1pe +y3pi +y3re +yr3ek +y3ri +y3si +y3ti +y5t3r +y5ve +zi5o + +.sÃ¥3 +.ær5i +.øv3r +a3tø +a5væ +brød3 +5bæ +5drøv +dstÃ¥4 +3dæ +3dø +e3læ +e3lø +e3rø +er5øn +e5tæ +e5tø +e1væ +e3æ +e5Ã¥ +3fæ +3fø +fø4r5en +giø4 +g4sø +g5sÃ¥ +3gæ +3gø1 +3gÃ¥ +i5tæ +i3ø +3kø +3kÃ¥ +lingeniø4 +l3væ +5løs +m5tÃ¥ +1mæ +3mø +3mÃ¥ +n3kæ +n5tæ +3næ +4n5æb +5nø +o5læ +or3ø +o5Ã¥ +5præ +5pæd +pÃ¥3 +r5kæ +r5tæ +r5tø +r3væ +r5æl +4røn +5rør +3rÃ¥d +r5Ã¥r +s4kÃ¥ +3slÃ¥ +s4næ +5stø +1stÃ¥ +1sæ +4s5æn +1sø +s5øk +sÃ¥4r5 +ti4ø +3træk. +t4sø +t5sÃ¥ +t3væ +u3læ +3værd +1værk +5vÃ¥ +y5væ +æb3l +æ3c +æ3e +æg5a +æ4gek +æ4g5r +ægs5 +æ5i +æ5kv +ælle4 +æn1dr +æ5o +æ1re +ær4g5r +æ3ri +ær4ma +ær4mo +ær5s +æ5si +æ3so +æ3ste +æ3ve +øde5 +ø3e +ø1je +ø3ke +ø3le +øms5 +øn3st +øn4t3 +ø1re +ø3ri +ørne3 +ør5o +ø1ve +Ã¥1d +Ã¥1e +Ã¥5h +Ã¥3l +Ã¥3re +Ã¥rs5t +Ã¥5sk +Ã¥3t + + diff --git a/solr/core/src/test-files/solr/collection1/conf/da_compoundDictionary.txt b/solr/core/src/test-files/solr/collection1/conf/da_compoundDictionary.txt new file mode 100644 index 0000000..9a14f40 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/da_compoundDictionary.txt @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A set of words for testing the HyphenationCompound factory, +# in conjunction with the danish hyphenation grammar. +læse +hest diff --git a/solr/core/src/test-files/solr/collection1/conf/elevate.xml b/solr/core/src/test-files/solr/collection1/conf/elevate.xml new file mode 100644 index 0000000..d15bd5e --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/elevate.xml @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/frenchArticles.txt b/solr/core/src/test-files/solr/collection1/conf/frenchArticles.txt new file mode 100644 index 0000000..9141611 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/frenchArticles.txt @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A set of articles for testing the French Elision filter. +# Requiring a text file is a bit weird here... +l +m +t +qu +n +s +j diff --git a/solr/core/src/test-files/solr/collection1/conf/hunspell-test.aff b/solr/core/src/test-files/solr/collection1/conf/hunspell-test.aff new file mode 100644 index 0000000..d035ad1 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/hunspell-test.aff @@ -0,0 +1,13 @@ +SET UTF-8 +TRY abcdefghijklmopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ + +SFX A Y 2 +SFX A 0 e n +SFX A 0 e t + +SFX C Y 2 +SFX C 0 d/C c +SFX C 0 c b + +PFX B Y 1 +PFX B 0 s o \ No newline at end of file diff --git a/solr/core/src/test-files/solr/collection1/conf/hunspell-test.dic b/solr/core/src/test-files/solr/collection1/conf/hunspell-test.dic new file mode 100644 index 0000000..92c35d2 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/hunspell-test.dic @@ -0,0 +1,6 @@ +5 +lucen/A +lucene +mahout/A +olr/B +ab/C \ No newline at end of file diff --git a/solr/core/src/test-files/solr/collection1/conf/hyphenation.dtd b/solr/core/src/test-files/solr/collection1/conf/hyphenation.dtd new file mode 100644 index 0000000..083c2bd --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/hyphenation.dtd @@ -0,0 +1,68 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/keep-1.txt b/solr/core/src/test-files/solr/collection1/conf/keep-1.txt new file mode 100644 index 0000000..8dfe809 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/keep-1.txt @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +foo +bar \ No newline at end of file diff --git a/solr/core/src/test-files/solr/collection1/conf/keep-2.txt b/solr/core/src/test-files/solr/collection1/conf/keep-2.txt new file mode 100644 index 0000000..646b7ff --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/keep-2.txt @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +junk +more \ No newline at end of file diff --git a/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt b/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt new file mode 100644 index 0000000..ede7742 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt @@ -0,0 +1,246 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Syntax: +# "source" => "target" +# "source".length() > 0 (source cannot be empty.) +# "target".length() >= 0 (target can be empty.) + +# example: +# "À" => "A" +# "\u00C0" => "A" +# "\u00C0" => "\u0041" +# "ß" => "ss" +# "\t" => " " +# "\n" => "" + +# À => A +"\u00C0" => "A" + +# à => A +"\u00C1" => "A" + +#  => A +"\u00C2" => "A" + +# à => A +"\u00C3" => "A" + +# Ä => A +"\u00C4" => "A" + +# Ã… => A +"\u00C5" => "A" + +# Æ => AE +"\u00C6" => "AE" + +# Ç => C +"\u00C7" => "C" + +# È => E +"\u00C8" => "E" + +# É => E +"\u00C9" => "E" + +# Ê => E +"\u00CA" => "E" + +# Ë => E +"\u00CB" => "E" + +# ÃŒ => I +"\u00CC" => "I" + +# à => I +"\u00CD" => "I" + +# ÃŽ => I +"\u00CE" => "I" + +# à => I +"\u00CF" => "I" + +# IJ => IJ +"\u0132" => "IJ" + +# à => D +"\u00D0" => "D" + +# Ñ => N +"\u00D1" => "N" + +# Ã’ => O +"\u00D2" => "O" + +# Ó => O +"\u00D3" => "O" + +# Ô => O +"\u00D4" => "O" + +# Õ => O +"\u00D5" => "O" + +# Ö => O +"\u00D6" => "O" + +# Ø => O +"\u00D8" => "O" + +# Å’ => OE +"\u0152" => "OE" + +# Þ +"\u00DE" => "TH" + +# Ù => U +"\u00D9" => "U" + +# Ú => U +"\u00DA" => "U" + +# Û => U +"\u00DB" => "U" + +# Ãœ => U +"\u00DC" => "U" + +# à => Y +"\u00DD" => "Y" + +# Ÿ => Y +"\u0178" => "Y" + +# à => a +"\u00E0" => "a" + +# á => a +"\u00E1" => "a" + +# â => a +"\u00E2" => "a" + +# ã => a +"\u00E3" => "a" + +# ä => a +"\u00E4" => "a" + +# Ã¥ => a +"\u00E5" => "a" + +# æ => ae +"\u00E6" => "ae" + +# ç => c +"\u00E7" => "c" + +# è => e +"\u00E8" => "e" + +# é => e +"\u00E9" => "e" + +# ê => e +"\u00EA" => "e" + +# ë => e +"\u00EB" => "e" + +# ì => i +"\u00EC" => "i" + +# í => i +"\u00ED" => "i" + +# î => i +"\u00EE" => "i" + +# ï => i +"\u00EF" => "i" + +# ij => ij +"\u0133" => "ij" + +# ð => d +"\u00F0" => "d" + +# ñ => n +"\u00F1" => "n" + +# ò => o +"\u00F2" => "o" + +# ó => o +"\u00F3" => "o" + +# ô => o +"\u00F4" => "o" + +# õ => o +"\u00F5" => "o" + +# ö => o +"\u00F6" => "o" + +# ø => o +"\u00F8" => "o" + +# Å“ => oe +"\u0153" => "oe" + +# ß => ss +"\u00DF" => "ss" + +# þ => th +"\u00FE" => "th" + +# ù => u +"\u00F9" => "u" + +# ú => u +"\u00FA" => "u" + +# û => u +"\u00FB" => "u" + +# ü => u +"\u00FC" => "u" + +# ý => y +"\u00FD" => "y" + +# ÿ => y +"\u00FF" => "y" + +# ff => ff +"\uFB00" => "ff" + +# ï¬ => fi +"\uFB01" => "fi" + +# fl => fl +"\uFB02" => "fl" + +# ffi => ffi +"\uFB03" => "ffi" + +# ffl => ffl +"\uFB04" => "ffl" + +# ſt => ft +"\uFB05" => "ft" + +# st => st +"\uFB06" => "st" diff --git a/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt b/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt new file mode 100644 index 0000000..a7624f0 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +a => aa +b => b1 b2 +c => c1,c2 +a\=>a => b\=>b +a\,a => b\,b +foo,bar,baz + +Television,TV,Televisions diff --git a/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json b/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json new file mode 100644 index 0000000..c7bf1ed --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json @@ -0,0 +1,167 @@ +{ + "disclaimer": "This data is collected from various providers and provided free of charge for informational purposes only, with no guarantee whatsoever of accuracy, validity, availability or fitness for any purpose; use at your own risk. Other than that - have fun, and please share/watch/fork if you think data like this should be free!", + "license": "Data collected from various providers with public-facing APIs; copyright may apply; not for resale; no warranties given.", + "timestamp": 1332070464, + "base": "USD", + "rates": { + "AED": 3.6732, + "AFN": 48.299999, + "ALL": 105.919998, + "AMD": 388.890015, + "ANG": 1.79, + "AOA": 94.769997, + "ARS": 4.35, + "AUD": 0.943931, + "AWG": 1.7899, + "AZN": 0.7863, + "BAM": 1.48775, + "BBD": 2, + "BDT": 82, + "BGN": 1.4962, + "BHD": 0.37703, + "BIF": 1304.170044, + "BMD": 1, + "BND": 1.2575, + "BOB": 6.91, + "BRL": 1.8003, + "BSD": 1, + "BTN": 50.185001, + "BWP": 7.2307, + "BYR": 8150, + "BZD": 1.9135, + "CAD": 0.9921, + "CDF": 917.276917, + "CHF": 0.9164, + "CLF": 0.02146, + "CLP": 482.75, + "CNY": 6.3239, + "COP": 1760, + "CRC": 507.600006, + "CUP": 1, + "CVE": 84.190002, + "CZK": 18.606001, + "DJF": 179.490005, + "DKK": 5.64424, + "DOP": 39.025002, + "DZD": 74.544998, + "EGP": 6.0385, + "ETB": 17.720449, + "EUR": 0.758956, + "FJD": 1.7734, + "FKP": 0.6316, + "GBP": 0.631373, + "GEL": 1.6469, + "GHS": 1.7455, + "GIP": 0.63165, + "GMD": 31.5, + "GNF": 7100, + "GTQ": 7.6975, + "GYD": 203.699997, + "HKD": 7.76306, + "HNL": 19.055, + "HRK": 5.7333, + "HTG": 41, + "HUF": 219.850006, + "IDR": 9118, + "IEP": 0.5978, + "ILS": 3.7542, + "INR": 50.165001, + "IQD": 1165.5, + "IRR": 12308, + "ISK": 127.440002, + "JMD": 86.699997, + "JOD": 0.7095, + "JPY": 83.445, + "KES": 83.18, + "KGS": 46.699402, + "KHR": 4010.300049, + "KMF": 373.424255, + "KPW": 900, + "KRW": 1125.849976, + "KWD": 0.27925, + "KZT": 147.690002, + "LAK": 7993.799805, + "LBP": 1504, + "LKR": 125.224998, + "LRD": 73.459999, + "LSL": 7.5768, + "LTL": 2.6219, + "LVL": 0.5291, + "LYD": 1.2572, + "MAD": 8.4611, + "MDL": 11.89, + "MGA": 2155, + "MKD": 46.705002, + "MMK": 6.51, + "MNT": 1322.5, + "MOP": 7.9958, + "MRO": 293, + "MUR": 29.110001, + "MVR": 15.36, + "MWK": 165.206207, + "MXN": 12.6745, + "MYR": 3.0575, + "MZN": 27.200001, + "NAD": 7.58, + "NGN": 157.600006, + "NIO": 23.215, + "NOK": 5.73163, + "NPR": 80.620003, + "NZD": 1.212269, + "OMR": 0.38485, + "PAB": 1, + "PEN": 2.674, + "PGK": 2.0627, + "PHP": 43.02, + "PKR": 90.800003, + "PLN": 3.1285, + "PYG": 4245, + "QAR": 3.6415, + "RON": 3.3256, + "RSD": 84.100502, + "RUB": 29.2342, + "RWF": 606.717468, + "SAR": 3.7505, + "SBD": 7.075973, + "SCR": 14.0447, + "SDG": 2.6765, + "SEK": 6.74525, + "SGD": 1.258, + "SHP": 0.63165, + "SLL": 4364.5, + "SOS": 1629, + "SRD": 3.2875, + "STD": 18650, + "SVC": 8.7475, + "SYP": 57.450001, + "SZL": 7.5752, + "THB": 30.700001, + "TJS": 4.7588, + "TMT": 2.85, + "TND": 1.5178, + "TOP": 1.693601, + "TRY": 1.796, + "TTD": 6.40015, + "TWD": 29.532, + "TZS": 1595, + "UAH": 8.029, + "UGX": 2481.699951, + "USD": 1, + "UYU": 19.469999, + "UZS": 1835.75, + "VEF": 4.295, + "VND": 20820, + "VUV": 90.199997, + "WST": 2.247475, + "XAF": 497.898987, + "XCD": 2.7, + "XDR": 0.652794, + "XOF": 498.399994, + "XPF": 90.639999, + "YER": 216.005005, + "ZAR": 7.5688, + "ZMK": 5271.5, + "ZWD": 378.700012, + "ZWL": 322.355011 + } +} \ No newline at end of file diff --git a/solr/core/src/test-files/solr/collection1/conf/phrasesuggest.txt b/solr/core/src/test-files/solr/collection1/conf/phrasesuggest.txt new file mode 100644 index 0000000..fd4984d --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/phrasesuggest.txt @@ -0,0 +1,8 @@ +# simple auto-suggest phrase dictionary for testing +# note this uses tabs as separator! +the first phrase 1.0 +the second phrase 2.0 +testing 1234 3.0 +foo 5.0 +the fifth phrase 2.0 +the final phrase 4.0 diff --git a/solr/core/src/test-files/solr/collection1/conf/protwords.txt b/solr/core/src/test-files/solr/collection1/conf/protwords.txt new file mode 100644 index 0000000..ab7e3e2 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/protwords.txt @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#use a protected word file to avoid stemming two +#unrelated words to the same base word. +#to test, we will use words that would normally obviously be stemmed. +cats +ridding +c# +c++ +.net diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml b/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml new file mode 100644 index 0000000..bf0e3e9 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml @@ -0,0 +1,100 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-bm25.xml b/solr/core/src/test-files/solr/collection1/conf/schema-bm25.xml new file mode 100644 index 0000000..54bdc05 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-bm25.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + 1.2 + 0.76 + + + + + + + + + + + + text + id + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-collate.xml b/solr/core/src/test-files/solr/collection1/conf/schema-collate.xml new file mode 100644 index 0000000..7feb73a --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-collate.xml @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text + id + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml b/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml new file mode 100644 index 0000000..d3e09d9 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml @@ -0,0 +1,473 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text + id + + + + + + + + + + + + + + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-dfr.xml b/solr/core/src/test-files/solr/collection1/conf/schema-dfr.xml new file mode 100644 index 0000000..c4f7d83 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-dfr.xml @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + I(F) + B + H2 + + + + + + + + I(F) + B + H3 + 900 + + + + + + + + P + L + H2 + 7 + + + + + + + + + + + + + text + id + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-folding.xml b/solr/core/src/test-files/solr/collection1/conf/schema-folding.xml new file mode 100644 index 0000000..95b37c3 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-folding.xml @@ -0,0 +1,259 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + content + id + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-ib.xml b/solr/core/src/test-files/solr/collection1/conf/schema-ib.xml new file mode 100644 index 0000000..3d55b2a --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-ib.xml @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + SPL + DF + H2 + + + + + + + + LL + TTF + H3 + 900 + + + + + + + + + + + + text + id + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-lmdirichlet.xml b/solr/core/src/test-files/solr/collection1/conf/schema-lmdirichlet.xml new file mode 100644 index 0000000..f39922f --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-lmdirichlet.xml @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + 1000 + + + + + + + + + + + + text + id + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-lmjelinekmercer.xml b/solr/core/src/test-files/solr/collection1/conf/schema-lmjelinekmercer.xml new file mode 100644 index 0000000..49b692e --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-lmjelinekmercer.xml @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + 0.4 + + + + + + + + + + + + text + id + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-luceneMatchVersion.xml b/solr/core/src/test-files/solr/collection1/conf/schema-luceneMatchVersion.xml new file mode 100644 index 0000000..13e12f3 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-luceneMatchVersion.xml @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-minimal.xml b/solr/core/src/test-files/solr/collection1/conf/schema-minimal.xml new file mode 100644 index 0000000..9e2f947 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-minimal.xml @@ -0,0 +1,25 @@ + + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-not-required-unique-key.xml b/solr/core/src/test-files/solr/collection1/conf/schema-not-required-unique-key.xml new file mode 100644 index 0000000..f3e5a1a --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-not-required-unique-key.xml @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + subject + id + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-numeric.xml b/solr/core/src/test-files/solr/collection1/conf/schema-numeric.xml new file mode 100644 index 0000000..0a12736 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-numeric.xml @@ -0,0 +1,78 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-phrasesuggest.xml b/solr/core/src/test-files/solr/collection1/conf/schema-phrasesuggest.xml new file mode 100644 index 0000000..da11caf --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-phrasesuggest.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text + id + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-replication1.xml b/solr/core/src/test-files/solr/collection1/conf/schema-replication1.xml new file mode 100644 index 0000000..48ecd9f --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-replication1.xml @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + id + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-replication2.xml b/solr/core/src/test-files/solr/collection1/conf/schema-replication2.xml new file mode 100644 index 0000000..65ad6f3 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-replication2.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + id + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-required-fields.xml b/solr/core/src/test-files/solr/collection1/conf/schema-required-fields.xml new file mode 100644 index 0000000..0945629 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-required-fields.xml @@ -0,0 +1,439 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text + id + + + + + + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-reversed.xml b/solr/core/src/test-files/solr/collection1/conf/schema-reversed.xml new file mode 100644 index 0000000..22844af --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-reversed.xml @@ -0,0 +1,85 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + one + id + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-sim.xml b/solr/core/src/test-files/solr/collection1/conf/schema-sim.xml new file mode 100644 index 0000000..ca2bd78 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-sim.xml @@ -0,0 +1,68 @@ + + + + + + + + + + + + + + + + + + + + + + + + is there an echo? + + + + + + + + + + + + + + + + + + + + + + + + sim1text + id + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-spellchecker.xml b/solr/core/src/test-files/solr/collection1/conf/schema-spellchecker.xml new file mode 100644 index 0000000..7124065 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-spellchecker.xml @@ -0,0 +1,87 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + text + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-stop-keep.xml b/solr/core/src/test-files/solr/collection1/conf/schema-stop-keep.xml new file mode 100644 index 0000000..e3d7839 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-stop-keep.xml @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + one + id + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-tfidf.xml b/solr/core/src/test-files/solr/collection1/conf/schema-tfidf.xml new file mode 100644 index 0000000..eacea90 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-tfidf.xml @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + false + + + + + + + + + + + + text + id + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-trie.xml b/solr/core/src/test-files/solr/collection1/conf/schema-trie.xml new file mode 100644 index 0000000..5c7b4cc --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-trie.xml @@ -0,0 +1,333 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + text + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema.xml b/solr/core/src/test-files/solr/collection1/conf/schema.xml new file mode 100644 index 0000000..b6aad0b --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema.xml @@ -0,0 +1,681 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + I am your default sim + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema11.xml b/solr/core/src/test-files/solr/collection1/conf/schema11.xml new file mode 100755 index 0000000..6d5fd1a --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema11.xml @@ -0,0 +1,385 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + text + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema12.xml b/solr/core/src/test-files/solr/collection1/conf/schema12.xml new file mode 100755 index 0000000..b6cc3c0 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema12.xml @@ -0,0 +1,605 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text + id + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema15.xml b/solr/core/src/test-files/solr/collection1/conf/schema15.xml new file mode 100755 index 0000000..48c2c9e --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema15.xml @@ -0,0 +1,591 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text + id + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml b/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml new file mode 100644 index 0000000..e08ab8d --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + string_f + string_f + diff --git a/solr/core/src/test-files/solr/collection1/conf/schemasurround.xml b/solr/core/src/test-files/solr/collection1/conf/schemasurround.xml new file mode 100644 index 0000000..6020cf1 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schemasurround.xml @@ -0,0 +1,613 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text + id + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-SOLR-749.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-SOLR-749.xml new file mode 100644 index 0000000..3a0a0cc --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-SOLR-749.xml @@ -0,0 +1,33 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-altdirectory.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-altdirectory.xml new file mode 100755 index 0000000..96cfd2d --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-altdirectory.xml @@ -0,0 +1,25 @@ + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-basic.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-basic.xml new file mode 100644 index 0000000..b23c612 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-basic.xml @@ -0,0 +1,27 @@ + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + ${solr.data.dir:} + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-caching.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-caching.xml new file mode 100644 index 0000000..c6f7cbf --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-caching.xml @@ -0,0 +1,38 @@ + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + + + + + + \ No newline at end of file diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy1.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy1.xml new file mode 100644 index 0000000..751663c --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy1.xml @@ -0,0 +1,40 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + + + + + true + 3 + 100MILLISECONDS + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy2.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy2.xml new file mode 100644 index 0000000..585a1cd --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy2.xml @@ -0,0 +1,32 @@ + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + value1 + value2 + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-elevate.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-elevate.xml new file mode 100644 index 0000000..b32572a --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-elevate.xml @@ -0,0 +1,191 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + ${solr.data.dir:} + + + + + + single + + + + + + + + + + + + + + + + + 1024 + + + + + + + + + + + true + + 10 + + + + + + + + + + + + + + + + + string + ${elevate.file:elevate.xml} + + + + + + string + ${elevate.data.file:elevate-data.xml} + + + + + explicit + + + elevate + + + + + + explicit + + + dataElevate + + + + + + + + + + max-age=30, public + + + + + solr + solrconfig.xml scheam.xml admin-extra.html + + + + prefix-${solr.test.sys.prop2}-suffix + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-enableplugin.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-enableplugin.xml new file mode 100644 index 0000000..5d97ab1 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-enableplugin.xml @@ -0,0 +1,68 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + ${solr.data.dir:} + + + + + + single + + + + + + + true + + + + + + + + + + + + + + + + + + + + + + max-age=30, public + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-functionquery.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-functionquery.xml new file mode 100755 index 0000000..12bf3a2 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-functionquery.xml @@ -0,0 +1,42 @@ + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + + + + + + + + + + + 0.0 + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-highlight.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-highlight.xml new file mode 100644 index 0000000..9da3245 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-highlight.xml @@ -0,0 +1,58 @@ + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + + + + + + + + + + + + 100 + + + + + + 70 + + + + + + + ]]> + ]]> + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-lazywriter.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-lazywriter.xml new file mode 100644 index 0000000..1e2939f --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-lazywriter.xml @@ -0,0 +1,27 @@ + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-master.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-master.xml new file mode 100644 index 0000000..cece0ab --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-master.xml @@ -0,0 +1,76 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + ${solr.data.dir:} + + + single + + + + + + + true + + + + + commit + schema.xml + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + + + + max-age=30, public + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-master1-keepOneBackup.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-master1-keepOneBackup.xml new file mode 100644 index 0000000..5347975 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-master1-keepOneBackup.xml @@ -0,0 +1,51 @@ + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + ${solr.data.dir:} + + + + single + + + + + + + + + + commit + schema-replication2.xml:schema.xml + + 1 + + + + + + + + + max-age=30, public + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-master1.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-master1.xml new file mode 100644 index 0000000..019f47e --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-master1.xml @@ -0,0 +1,76 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + ${solr.data.dir:} + + + + single + + + + + + + true + + + + + commit + schema-replication2.xml:schema.xml + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + + + + max-age=30, public + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-master2.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-master2.xml new file mode 100644 index 0000000..b088fb9 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-master2.xml @@ -0,0 +1,76 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + ${solr.data.dir:} + + + + single + + + + + + + true + + + + + startup + schema.xml + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + + + + max-age=30, public + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-master3.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-master3.xml new file mode 100644 index 0000000..cf3ee59 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-master3.xml @@ -0,0 +1,77 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + ${solr.data.dir:} + + + + single + + + + + + + true + + + + + commit + startup + schema.xml + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + + + + max-age=30, public + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy.xml new file mode 100644 index 0000000..8fa12a7 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy.xml @@ -0,0 +1,40 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + 7 + + 19 + 9 + 1.0 + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-nativelock.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-nativelock.xml new file mode 100644 index 0000000..ac0f70d --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-nativelock.xml @@ -0,0 +1,33 @@ + + + + + + + + ${solr.data.dir:} + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + native + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-nocache.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-nocache.xml new file mode 100644 index 0000000..cb6c92d --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-nocache.xml @@ -0,0 +1,45 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + + ${solr.data.dir:} + + + + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml new file mode 100644 index 0000000..4cdd038 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml @@ -0,0 +1,63 @@ + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + ${solr.data.dir:} + + + + + + + suggest_wfst + org.apache.solr.spelling.suggest.Suggester + org.apache.solr.spelling.suggest.fst.WFSTLookupFactory + suggest_wfst + false + + + true + + phrasesuggest.txt + + + + phrase_suggest + + + + + + + + + true + suggest_wfst + false + + true + + + suggest_wfst + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-propinject-indexdefault.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-propinject-indexdefault.xml new file mode 100644 index 0000000..efa1e6c --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-propinject-indexdefault.xml @@ -0,0 +1,60 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + + + + 32.0 + + + 4 + 4 + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-propinject.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-propinject.xml new file mode 100644 index 0000000..f3e5391 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-propinject.xml @@ -0,0 +1,61 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + + + + + 64.0 + + + + 2 + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender-noquery.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender-noquery.xml new file mode 100644 index 0000000..206350d --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender-noquery.xml @@ -0,0 +1,79 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender.xml new file mode 100644 index 0000000..a23c734 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender.xml @@ -0,0 +1,75 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + + + + + + + + + + + + + solr 0 10 mock + rocks 0 10 mock + + + + + + + + + fast_warm 0 10 + mock + + + + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-repeater.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-repeater.xml new file mode 100644 index 0000000..14d540c --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-repeater.xml @@ -0,0 +1,72 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + ${solr.data.dir:} + + + single + + + + + + + true + + + + + + + + + + + + + + + + commit + schema.xml + + + http://localhost:TEST_PORT/solr/replication + 00:00:01 + + + + + + + + + max-age=30, public + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-reqHandler.incl b/solr/core/src/test-files/solr/collection1/conf/solrconfig-reqHandler.incl new file mode 100644 index 0000000..03f236f --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-reqHandler.incl @@ -0,0 +1,5 @@ + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-simplelock.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-simplelock.xml new file mode 100644 index 0000000..1760267 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-simplelock.xml @@ -0,0 +1,33 @@ + + + + + + + + ${solr.data.dir:} + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + simple + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-slave.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-slave.xml new file mode 100644 index 0000000..5ee5459 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-slave.xml @@ -0,0 +1,68 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + ${solr.data.dir:} + + + single + + + + + + + true + + + + + + + + + + + + + + + + http://localhost:TEST_PORT/solr + 00:00:01 + + + + + + + + + max-age=30, public + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-slave1.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-slave1.xml new file mode 100644 index 0000000..afcd2ef --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-slave1.xml @@ -0,0 +1,65 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + ${solr.data.dir:} + + + + single + + + + + + + true + + + + + + + + + + + + + + + + + + + + + + + max-age=30, public + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-solcoreproperties.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-solcoreproperties.xml new file mode 100644 index 0000000..39398ba --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-solcoreproperties.xml @@ -0,0 +1,66 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + ${solr.data.dir:} + + + + + + single + + + + + + + true + + + + + + + ${foo.foo1} + ${foo.foo2} + + + + + + + + + + + + max-age=30, public + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml new file mode 100644 index 0000000..87bd0d4 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml @@ -0,0 +1,184 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + ${solr.data.dir:} + + + + + + + lowerpunctfilt + + + default + lowerfilt + spellchecker1 + true + + + default_teststop + default_teststop + true + teststop + + + direct + solr.DirectSolrSpellChecker + 3 + 100 + teststop + + + direct_lowerfilt + solr.DirectSolrSpellChecker + 3 + 100 + lowerfilt + + + wordbreak + solr.WordBreakSolrSpellChecker + lowerfilt + true + true + MAX_FREQ + 10 + + + threshold + lowerfilt + spellcheckerThreshold + true + .29 + + + threshold_direct + solr.DirectSolrSpellChecker + lowerfilt + spellcheckerThreshold + true + .29 + + + multipleFields + lowerfilt1and2 + spellcheckerMultipleFields + true + + + + jarowinkler + lowerfilt + + org.apache.lucene.search.spell.JaroWinklerDistance + spellchecker2 + + + + solr.FileBasedSpellChecker + external + spellings.txt + UTF-8 + spellchecker3 + + + + freq + lowerfilt + spellcheckerFreq + + freq + true + + + fqcn + lowerfilt + spellcheckerFQCN + org.apache.solr.spelling.SampleComparator + true + + + perDict + org.apache.solr.handler.component.DummyCustomParamSpellChecker + lowerfilt + + + + + + + + + + false + + false + + 1 + + + spellcheck + + + + + dismax + lowerfilt1^1 + + + spellcheck + + + + + default + wordbreak + 20 + + + spellcheck + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellchecker.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellchecker.xml new file mode 100644 index 0000000..72fc0f6 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellchecker.xml @@ -0,0 +1,142 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + + + + + + + + suggest + org.apache.solr.spelling.suggest.Suggester + org.apache.solr.spelling.suggest.jaspell.JaspellLookup + suggest + suggest + true + + + 0.0 + + + + + + + suggest_tst + org.apache.solr.spelling.suggest.Suggester + org.apache.solr.spelling.suggest.tst.TSTLookup + suggest + suggest_tst + true + + + 0.0 + + + + + + + suggest_fst + org.apache.solr.spelling.suggest.Suggester + org.apache.solr.spelling.suggest.fst.FSTLookup + suggest + suggest_fst + true + + + 5 + true + + + + + + + suggest_wfst + org.apache.solr.spelling.suggest.Suggester + org.apache.solr.spelling.suggest.fst.WFSTLookupFactory + suggest + suggest_wfst + true + + + true + + + + + + + true + suggest + true + + + suggest_jaspell + + + + + + + true + suggest_tst + true + + + suggest_tst + + + + + + + true + suggest_fst + false + + + suggest_fst + + + + + + + true + suggest_wfst + false + + + suggest_wfst + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-termindex.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-termindex.xml new file mode 100644 index 0000000..d3c4375 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-termindex.xml @@ -0,0 +1,62 @@ + + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + + + + + + + + 256 + + + + + + + + + 12 + + + + + + solr + solrconfig.xml scheam.xml admin-extra.html + + + + prefix-${solr.test.sys.prop2}-suffix + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml new file mode 100644 index 0000000..9933392 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml @@ -0,0 +1,42 @@ + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + ${solr.data.dir:} + + + + + + + true + + + + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-transformers.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-transformers.xml new file mode 100644 index 0000000..e59bad0 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-transformers.xml @@ -0,0 +1,84 @@ + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + + + + + + x1 + x2 + + + + 100 + + + + x1 + x2 + + + + + xA + xA + + + + + + + + 88 + 99 + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml new file mode 100644 index 0000000..dc49b14 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml @@ -0,0 +1,373 @@ + + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + + + + solr.TrieIntField + solr.TrieLongField + + + + min_foo_l + + + max_foo_l + + + ; + + primary_author_s1 + + + + primary_author_s1 + first_foo_l + + + + + + + + + + + + + + foo_t + + + + + + + + + foo_t + + + + + + foo.* + bar.* + + .*HOSS.* + + + + + + foo.* + bar.* + + + solr.DateField + + + .*HOSS.* + + + + + + foo.* + bar.* + + + solr.DateField + .*HOSS.* + + + + + + + name + foo_t + + + + + + name + foo_t + + + + + + + foo.* + bar.*_s + + + + + + nametext + text_sw + + + + + + solr.DateField + solr.StrField + + + + + + solr.DateField + solr.StrField + + foo.* + + + + + + + + + + + + + + + + + + foo.* + yak.* + + + + + + + + + + + foo_s + + + + + string + ; + + + + + + foo_s + bar_s + + + + + foo_s + bar_s + + + + + foo_i + foo_s + bar_s + + + + + foo_i + foo_s + bar_s + + + + + + html_s + + + + + + + trunc + 5 + + + + + + + + + + .*_raw + + + + + + source1_s + dest_s + + + + + source1_s + source2_s + dest_s + + + + + + + source1_s + source2_s + + dest_s + + + + + + + source\d_.* + + source0_.* + + + dest_s + + + + + + field1 + toField + + + toField + 3 + + + + + + field1 + toField + + + field1 + + + + + + toField + + + field1 + toField + + + + + + field1 + field2 + toField + + + ; + toField + + + + + + + category + category_s + + + + authors + editors + + contributors + + + + .*_price + + list_price + + + all_prices + + + + + + content + title + \s+ + X + + + + + + processor_default_s + X + + + processor_default_i + 42 + + + uuid + + + timestamp + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-xinclude.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-xinclude.xml new file mode 100644 index 0000000..5135a7e --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-xinclude.xml @@ -0,0 +1,28 @@ + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml new file mode 100644 index 0000000..7a7ba7d --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml @@ -0,0 +1,563 @@ + + + + + + + + + + + + ${solr.data.dir:} + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + 1000 + 8 + + + 10 + + single + true + + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + 1024 + + + + + + + + + + + true + + + + + + + 10 + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + true + + + + + + + dismax + *:* + 0.01 + + text^0.5 features_t^1.0 subject^1.4 title_stemmed^2.0 + + + text^0.2 features_t^1.1 subject^1.4 title_stemmed^2.0 title^1.5 + + + ord(weight)^0.5 recip(rord(iind),1,1000,1000)^0.3 + + + 3<-1 5<-2 6<90% + + 100 + + + + + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + + + lowerpunctfilt + + + default + lowerfilt + spellchecker1 + false + + + direct + DirectSolrSpellChecker + lowerfilt + 3 + + + wordbreak + solr.WordBreakSolrSpellChecker + lowerfilt + true + true + 10 + + + multipleFields + lowerfilt1and2 + spellcheckerMultipleFields + false + + + + jarowinkler + lowerfilt + + org.apache.lucene.search.spell.JaroWinklerDistance + spellchecker2 + + + + solr.FileBasedSpellChecker + external + spellings.txt + UTF-8 + spellchecker3 + + + + freq + lowerfilt + spellcheckerFreq + + freq + false + + + fqcn + lowerfilt + spellcheckerFQCN + org.apache.solr.spelling.SampleComparator + false + + + perDict + org.apache.solr.handler.component.DummyCustomParamSpellChecker + lowerfilt + + + + + + + + termsComp + + + + + + + + + false + + false + + 1 + + + spellcheck + + + + + direct + false + false + 1 + + + spellcheck + + + + + default + wordbreak + 20 + + + spellcheck + + + + + direct + wordbreak + 20 + + + spellcheck + + + + + dismax + lowerfilt1^1 + + + spellcheck + + + + + + + + + + + tvComponent + + + + + + + + + + + + 100 + + + + + + 70 + + + + + + + ]]> + ]]> + + + + + + + + + + + + + 10 + .,!? + + + + + + WORD + en + US + + + + + + + + + + + max-age=30, public + + + + + + + explicit + true + + + + + solr + solrconfig.xml scheam.xml admin-extra.html + + + + prefix-${solr.test.sys.prop2}-suffix + + + + + + false + true + v_t,t_field + org.apache.solr.update.processor.TextProfileSignature + + + + + + false + false + id + + org.apache.solr.update.processor.Lookup3Signature + + + + + + + true + non_indexed_signature_sS + false + v_t,t_field + org.apache.solr.update.processor.TextProfileSignature + + + + + + + uniq + uniq2 + uniq3 + + + + + + + + + regex_dup_A_s + x + x_x + + + + regex_dup_B_s + x + x_x + + + + + + + + regex_dup_A_s + x + x_x + + + regex_dup_B_s + x + x_x + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig_codec.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig_codec.xml new file mode 100644 index 0000000..28b4608 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig_codec.xml @@ -0,0 +1,23 @@ + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig_perf.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig_perf.xml new file mode 100755 index 0000000..0c5e205 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig_perf.xml @@ -0,0 +1,75 @@ + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + ${solr.data.dir:} + + + + + + + + + + + + + + + true + 20 + 200 + false + 2 + + + + + + + + + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/stemdict.txt b/solr/core/src/test-files/solr/collection1/conf/stemdict.txt new file mode 100644 index 0000000..f57a4ad --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/stemdict.txt @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# test that we can override the stemming algorithm with our own mappings +# these must be tab-separated +monkeys monkey +otters otter +# some crazy ones that a stemmer would never do +dogs cat diff --git a/solr/core/src/test-files/solr/collection1/conf/stop-1.txt b/solr/core/src/test-files/solr/collection1/conf/stop-1.txt new file mode 100644 index 0000000..8dfe809 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/stop-1.txt @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +foo +bar \ No newline at end of file diff --git a/solr/core/src/test-files/solr/collection1/conf/stop-2.txt b/solr/core/src/test-files/solr/collection1/conf/stop-2.txt new file mode 100644 index 0000000..646b7ff --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/stop-2.txt @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +junk +more \ No newline at end of file diff --git a/solr/core/src/test-files/solr/collection1/conf/stop-snowball.txt b/solr/core/src/test-files/solr/collection1/conf/stop-snowball.txt new file mode 100644 index 0000000..1c0c6f5 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/stop-snowball.txt @@ -0,0 +1,10 @@ + | This is a file in snowball format, empty lines are ignored, '|' is a comment + | Additionally, multiple words can be on the same line, allowing stopwords to be + | arranged in tables (useful in some languages where they might inflect) + + | fictitious table below + +|third person singular +|Subject Object Possessive Reflexive +he him his himself| masculine +she her hers herself| feminine diff --git a/solr/core/src/test-files/solr/collection1/conf/stoptypes-1.txt b/solr/core/src/test-files/solr/collection1/conf/stoptypes-1.txt new file mode 100644 index 0000000..456348e --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/stoptypes-1.txt @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/stoptypes-2.txt b/solr/core/src/test-files/solr/collection1/conf/stoptypes-2.txt new file mode 100644 index 0000000..d8a3810 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/stoptypes-2.txt @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/stopwithbom.txt b/solr/core/src/test-files/solr/collection1/conf/stopwithbom.txt new file mode 100644 index 0000000..eb5f6e1 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/stopwithbom.txt @@ -0,0 +1 @@ +BOMsAreEvil diff --git a/solr/core/src/test-files/solr/collection1/conf/stopwords.txt b/solr/core/src/test-files/solr/collection1/conf/stopwords.txt new file mode 100644 index 0000000..b5824da --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/stopwords.txt @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +#Standard english stop words taken from Lucene's StopAnalyzer +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +s +such +t +that +the +their +then +there +these +they +this +to +was +will +with + diff --git a/solr/core/src/test-files/solr/collection1/conf/stopwordsWrongEncoding.txt b/solr/core/src/test-files/solr/collection1/conf/stopwordsWrongEncoding.txt new file mode 100644 index 0000000..0d305c8 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/stopwordsWrongEncoding.txt @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# stopwords in the wrong encoding (ISO-8859-1). +# tests resourceloader's ability to report wrongly encoded files. +bañadores diff --git a/solr/core/src/test-files/solr/collection1/conf/synonyms.txt b/solr/core/src/test-files/solr/collection1/conf/synonyms.txt new file mode 100644 index 0000000..b0e31cb --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/synonyms.txt @@ -0,0 +1,31 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +aaa => aaaa +bbb => bbbb1 bbbb2 +ccc => cccc1,cccc2 +a\=>a => b\=>b +a\,a => b\,b +fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +pixima => pixma + diff --git a/solr/core/src/test-files/solr/collection1/conf/wdftypes.txt b/solr/core/src/test-files/solr/collection1/conf/wdftypes.txt new file mode 100644 index 0000000..7378b08 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/wdftypes.txt @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A customized type mapping for WordDelimiterFilterFactory +# the allowable types are: LOWER, UPPER, ALPHA, DIGIT, ALPHANUM, SUBWORD_DELIM +# +# the default for any character without a mapping is always computed from +# Unicode character properties + +# Map the $, %, '.', and ',' characters to DIGIT +# This might be useful for financial data. +$ => DIGIT +% => DIGIT +. => DIGIT +\u002C => DIGIT + +# in some cases you might not want to split on ZWJ +# this also tests the case where we need a bigger byte[] +# see http://en.wikipedia.org/wiki/Zero-width_joiner +\u200D => ALPHANUM diff --git a/solr/core/src/test-files/solr/collection1/conf/xslt/dummy-using-include.xsl b/solr/core/src/test-files/solr/collection1/conf/xslt/dummy-using-include.xsl new file mode 100644 index 0000000..f10cfbf --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/xslt/dummy-using-include.xsl @@ -0,0 +1,31 @@ + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/xslt/dummy.xsl b/solr/core/src/test-files/solr/collection1/conf/xslt/dummy.xsl new file mode 100644 index 0000000..fbbd8f7 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/xslt/dummy.xsl @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/xslt/xsl-update-handler-test.xsl b/solr/core/src/test-files/solr/collection1/conf/xslt/xsl-update-handler-test.xsl new file mode 100644 index 0000000..2e7359a --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/xslt/xsl-update-handler-test.xsl @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/lib/README b/solr/core/src/test-files/solr/collection1/lib/README new file mode 100644 index 0000000..b7ca5b8 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/lib/README @@ -0,0 +1,18 @@ + + +Items under this directory are used by TestConfig.testLibs() diff --git a/solr/core/src/test-files/solr/collection1/lib/classes/empty-file-main-lib.txt b/solr/core/src/test-files/solr/collection1/lib/classes/empty-file-main-lib.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/lib/classes/empty-file-main-lib.txt @@ -0,0 +1 @@ + diff --git a/solr/core/src/test-files/solr/conf/bad-mp-solrconfig.xml b/solr/core/src/test-files/solr/conf/bad-mp-solrconfig.xml deleted file mode 100644 index af5d8fb..0000000 --- a/solr/core/src/test-files/solr/conf/bad-mp-solrconfig.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - 8 - - - - - - diff --git a/solr/core/src/test-files/solr/conf/bad-schema-dup-dynamicField.xml b/solr/core/src/test-files/solr/conf/bad-schema-dup-dynamicField.xml deleted file mode 100644 index 460fbda..0000000 --- a/solr/core/src/test-files/solr/conf/bad-schema-dup-dynamicField.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - id - id - - diff --git a/solr/core/src/test-files/solr/conf/bad-schema-dup-field.xml b/solr/core/src/test-files/solr/conf/bad-schema-dup-field.xml deleted file mode 100644 index 4272362..0000000 --- a/solr/core/src/test-files/solr/conf/bad-schema-dup-field.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - id - id - - diff --git a/solr/core/src/test-files/solr/conf/bad-schema-dup-fieldType.xml b/solr/core/src/test-files/solr/conf/bad-schema-dup-fieldType.xml deleted file mode 100644 index 34ef44b..0000000 --- a/solr/core/src/test-files/solr/conf/bad-schema-dup-fieldType.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - id - id - - diff --git a/solr/core/src/test-files/solr/conf/bad-schema-external-filefield.xml b/solr/core/src/test-files/solr/conf/bad-schema-external-filefield.xml deleted file mode 100644 index e7874c8..0000000 --- a/solr/core/src/test-files/solr/conf/bad-schema-external-filefield.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/bad-schema-nontext-analyzer.xml b/solr/core/src/test-files/solr/conf/bad-schema-nontext-analyzer.xml deleted file mode 100644 index 06a689a..0000000 --- a/solr/core/src/test-files/solr/conf/bad-schema-nontext-analyzer.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - id - id - - diff --git a/solr/core/src/test-files/solr/conf/bad-schema-not-indexed-but-norms.xml b/solr/core/src/test-files/solr/conf/bad-schema-not-indexed-but-norms.xml deleted file mode 100644 index f7c4e9b..0000000 --- a/solr/core/src/test-files/solr/conf/bad-schema-not-indexed-but-norms.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - id - id - - diff --git a/solr/core/src/test-files/solr/conf/bad-schema-not-indexed-but-pos.xml b/solr/core/src/test-files/solr/conf/bad-schema-not-indexed-but-pos.xml deleted file mode 100644 index 774d587..0000000 --- a/solr/core/src/test-files/solr/conf/bad-schema-not-indexed-but-pos.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - id - id - - diff --git a/solr/core/src/test-files/solr/conf/bad-schema-not-indexed-but-tf.xml b/solr/core/src/test-files/solr/conf/bad-schema-not-indexed-but-tf.xml deleted file mode 100644 index d153793..0000000 --- a/solr/core/src/test-files/solr/conf/bad-schema-not-indexed-but-tf.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - id - id - - diff --git a/solr/core/src/test-files/solr/conf/bad-schema-omit-tf-but-not-pos.xml b/solr/core/src/test-files/solr/conf/bad-schema-omit-tf-but-not-pos.xml deleted file mode 100644 index 116f116..0000000 --- a/solr/core/src/test-files/solr/conf/bad-schema-omit-tf-but-not-pos.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - id - id - - diff --git a/solr/core/src/test-files/solr/conf/bad_solrconfig.xml b/solr/core/src/test-files/solr/conf/bad_solrconfig.xml deleted file mode 100644 index 8c8e960..0000000 --- a/solr/core/src/test-files/solr/conf/bad_solrconfig.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - ${unset.sys.property} - - diff --git a/solr/core/src/test-files/solr/conf/compoundDictionary.txt b/solr/core/src/test-files/solr/conf/compoundDictionary.txt deleted file mode 100644 index f4977b5..0000000 --- a/solr/core/src/test-files/solr/conf/compoundDictionary.txt +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# A set of words for testing the DictionaryCompound factory -soft -ball -team diff --git a/solr/core/src/test-files/solr/conf/currency.xml b/solr/core/src/test-files/solr/conf/currency.xml deleted file mode 100644 index f74f6e9..0000000 --- a/solr/core/src/test-files/solr/conf/currency.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - - - - - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/da_UTF8.xml b/solr/core/src/test-files/solr/conf/da_UTF8.xml deleted file mode 100644 index 2c8d203..0000000 --- a/solr/core/src/test-files/solr/conf/da_UTF8.xml +++ /dev/null @@ -1,1208 +0,0 @@ - - - - - - - - - - -aA -bB -cC -dD -eE -fF -gG -hH -iI -jJ -kK -lL -mM -nN -oO -pP -qQ -rR -sS -tT -uU -vV -wW -xX -yY -zZ -æÆ -øØ -åÅ - - - -.ae3 -.an3k -.an1s -.be5la -.be1t -.bi4tr -.der3i -.diagno5 -.her3 -.hoved3 -.ne4t5 -.om1 -.ove4 -.po1 -.til3 -.yd5r -ab5le -3abst -a3c -ade5la -5adg -a1e -5afg -5a4f1l -af3r -af4ri -5afs -a4gef -a4gi -ag5in -ag5si -3agti -a4gy -a3h -ais5t -a3j -a5ka -a3ke -a5kr -aku5 -a3la -a1le -a1li -al3k -4alkv -a1lo -al5si -a3lu -a1ly -am4pa -3analy -an4k5r -a3nu -3anv -a5o -a5pe -a3pi -a5po -a1ra -ar5af -1arb -a1re -5arg -a1ri -a3ro -a3sa -a3sc -a1si -a3sk -a3so -3a3sp -a3ste -a3sti -a1ta1 -a1te -a1ti -a4t5in -a1to -ato5v -a5tr -a1tu -a5va -a1ve -a5z -1ba -ba4ti -4bd -1be -be1k -be3ro -be5ru -be1s4 -be1tr -1bi -bi5sk -b1j -4b1n -1bo -bo4gr -bo3ra -bo5re -1br4 -4bs -bs5k -b3so -b1st -b5t -3bu -bu4s5tr -b5w -1by -by5s -4c1c -1ce -ce5ro -3ch -4ch. -ci4o -ck3 -5cy -3da -4d3af -d5anta -da4s -d1b -d1d4 -1de -de5d -4de4lem -der5eri -de4rig -de5sk -d1f -d1g -d3h -1di -di1e -di5l -d3j -d1k -d1l -d1m -4d1n -3do -4dop -d5ov -d1p -4drett -5d4reve -3drif -3driv -d5ros -d5ru -ds5an -ds5in -d1ski -d4sm -d4su -dsu5l -ds5vi -d3ta -d1te -dt5o -d5tr -dt5u -1du -dub5 -d1v -3dy -e5ad -e3af -e5ag -e3ak -e1al -ea4la -e3an -e5ap -e3at -e3bl -ebs3 -e1ci -ed5ar -edde4 -eddel5 -e4do -ed5ra -ed3re -ed3rin -ed4str -e3e -3eff -e3fr -3eft -e3gu -e1h -e3in -ei5s -e3je -e4j5el -e1ka -e3ke -e3kl -4e1ko -e5kr -ek5sa -3eksem -3eksp -e3ku -e1kv -e5ky -e3lad -el3ak -el3ar -e1las -e3le -e4lek -3elem -e1li -5elim -e3lo -el5sa -e5lu -e3ly -e4mad -em4p5le -em1s -en5ak -e4nan -4enn -e4no -en3so -e5nu -e5ol -e3op -e1or -e3ov -epi3 -e1pr -e3ra -er3af -e4rag -e4rak -e1re -e4ref -er5ege -5erhv -e1ri -e4rib -er1k -ero5d -er5ov -er3s -er5tr -e3rum -er5un -e5ry -e1ta -e1te -etek4s -e1ti -e3tj -e1to -e3tr -e3tu -e1ty -e3um -e3un -3eur -e1va -e3ve -e4v3erf -e1vi -e5x -1fa -fa4ce -fags3 -f1b -f1d -1fe -fej4 -fejl1 -f1f -f1g -f1h -1fi -f1k -3fl -1fo -for1en -fo4ri -f1p -f1s4 -4ft -f3ta -f1te -f1ti -f5to -f5tvi -1fu -f1v -3fy -1ga -g3art -g1b -g1d -1ge -4g5enden -ger3in -ge3s -g3f -g1g -g1h -1gi -gi4b -gi3st -5gj -g3k -g1l -g1m -3go -4g5om -g5ov -g3p -1gr -gs1a -gsde4len -g4se -gsha4 -g5sla -gs3or -gs1p -g5s4tide -g4str -gs1v -g3ta -g1te -g1ti -g5to -g3tr -gt4s -g3ud -gun5 -g3v -1gy -g5yd -4ha. -heds3 -he5s -4het -hi4e -hi4n5 -hi3s -ho5ko -ho5ve -4h3t -hun4 -hund3 -hvo4 -i1a -i3b -i4ble -i1c -i3dr -ids5k -i1el -i1en -i3er -i3et. -if3r -i3gu -i3h -i5i -i5j -i1ka -i1ke -ik1l -i5ko -ik3re -ik5ri -iks5t -ik4tu -i3ku -ik3v -i3lag -il3eg -il5ej -il5el -i3li -i4l5id -il3k -i1lo -il5u -i3mu -ind3t -5inf -ings1 -in3s -in4sv -inter1 -i3nu -i3od -i3og -i5ok -i3ol -ion4 -ions1 -i5o5r -i3ot -i5pi -i3pli -i5pr -i3re -i3ri -ir5t -i3sc -i3si -i4sm -is3p -i1ster -i3sti -i5sua -i1ta -i1te -i1ti -i3to -i3tr -it5re. -i1tu -i3ty -i1u -i1va -i1ve -i1vi -j3ag -jde4rer -jds1 -jek4to -4j5en. -j5k -j3le -j3li -jlmeld5 -jlmel4di -j3r -jre5 -ju3s -5kap -k5au -5kav -k5b -kel5s -ke3sk -ke5st -ke4t5a -k3h -ki3e -ki3st -k1k -k5lak -k1le -3klu -k4ny -5kod -1kon -ko3ra -3kort -ko3v -1kra -5kry -ks3an -k1si -ks3k -ks1p -k3ste -k5stu -ks5v -k1t -k4tar -k4terh -kti4e -kt5re -kt5s -3kur -1kus -3kut -k4vo -k4vu -5lab -lad3r -5lagd -la4g3r -5lam -1lat -l1b -ldiagnos5 -l3dr -ld3st -1le. -5led -4lele -le4mo -3len -1ler -1les -4leu -l1f -lfin4 -lfind5 -l1go1 -l3h -li4ga -4l5ins -4l3int -li5o -l3j -l1ke -l1ko -l3ky -l1l -l5mu -lo4du -l3op -4l5or -3lov -4l3p -l4ps -l3r -4ls -lses1 -ls5in -l5sj -l1ta -l4taf -l1te -l4t5erf -l3ti -lt3o -l3tr -l3tu -lu5l -l3ve -l3vi -1ma -m1b -m3d -1me -4m5ej -m3f -m1g -m3h -1mi -mi3k -m5ing -mi4o -mi5sty -m3k -m1l -m1m -mmen5 -m1n -3mo -mo4da -4mop -4m5ov -m1pe -m3pi -m3pl -m1po -m3pr -m1r -mse5s -ms5in -m5sk -ms3p -m3ste -ms5v -m3ta -m3te -m3ti -m3tr -m1ud -1mul -mu1li -3my -3na -4nak -1nal -n1b -n1c -4nd -n3dr -nd5si -nd5sk -nd5sp -1ne -ne5a -ne4da -nemen4 -nement5e -neo4 -n3erk -n5erl -ne5sl -ne5st -n1f -n4go -4n1h -1ni -4nim -ni5o -ni3st -n1ke -n1ko -n3kr -n3ku -n5kv -4n1l -n1m -n1n -1no -n3ord -n5p -n3r -4ns -n3si -n1sku -ns3po -n1sta -n5sti -n1ta -nta4le -n1te -n1ti -ntiali4 -n3to -n1tr -nt4s5t -nt4su -n3tu -n3ty -4n1v -3ny -n3z -o3a -o4as -ob3li -o1c -o4din -od5ri -od5s -od5un -o1e -of5r -o4gek -o4gel -o4g5o -og5re -og5sk -o5h -o5in -oi6s5e -o1j -o3ka -o1ke -o3ku -o3la -o3le -o1li -o1lo -o3lu -o5ly -1omr -on3k -ook5 -o3or -o5ov -o3pi -op3l -op3r -op3s -3opta -4or. -or1an -3ordn -ord5s -o3re. -o3reg -o3rek -o3rer -o3re3s -o3ret -o3ri -3orient -or5im -o4r5in -or3k -or5o -or3sl -or3st -o3si -o3so -o3t -o1te -o5un -ov4s -3pa -pa5gh -p5anl -p3d -4pec -3pen -1per -pe1ra -pe5s -pe3u -p3f -4p5h -1pla -p4lan -4ple. -4pler -4ples -p3m -p3n -5pok -4po3re -3pot -4p5p4 -p4ro -1proc -p3sk -p5so -ps4p -p3st -p1t -1pu -pu5b -p5ule -p5v -5py3 -qu4 -4raf -ra5is -4rarb -r1b -r4d5ar -r3dr -rd4s3 -4reks -1rel -re5la -r5enss -5rese -re5spo -4ress -re3st -re5s4u -5rett -r1f -r1gu -r1h -ri1e -ri5la -4rimo -r4ing -ringse4 -ringso4r -4rinp -4rint -r3ka -r1ke -r1ki -rk3so -r3ku -r1l -rmo4 -r5mu -r1n -ro1b -ro3p -r3or -r3p -r1r -rre5s -rro4n5 -r1sa -r1si -r5skr -r4sk5v -rs4n -r3sp -r5stu -r5su -r3sv -r5tal -r1te -r4teli -r1ti -r3to -r4t5or -rt5rat -rt3re -r5tri -r5tro -rt3s -r5ty -r3ud -run4da -5rut -r3va -r1ve -r3vi -ry4s -s3af -1sam -sa4ma -s3ap -s1ar -1sat -4s1b -s1d -sdy4 -1se -s4ed -5s4er -se4se -s1f -4s1g4 -4s3h -si4bl -1sig -s5int -5sis -5sit -5siu -s5ju -4sk. -1skab -1ske -s3kl -sk5s4 -5sky -s1le -s1li -slo3 -5slu -s5ly -s1m -s4my -4snin -s4nit -so5k -5sol -5som. -3somm -s5oms -5somt -3son -4s1op -sp4 -3spec -4sper -3s4pi -s1pl -3sprog. -s5r4 -s1s4 -4st. -5s4tam -1stan -st5as -3stat -1stav -1ste. -1sted -3stel -5stemo -1sten -5step -3ster. -3stes -5stet -5stj -3sto -st5om -1str -s1ud -3sul -s3un -3sur -s3ve -3s4y -1sy1s -5ta. -1tag -tands3 -4tanv -4tb -tede4l -teds5 -3teg -5tekn -teo1 -5term -te5ro -4t1f -6t3g -t1h -tialis5t -3tid -ti4en -ti3st -4t3k -4t1l -tli4s5 -t1m -t1n -to5ra -to1re -to1ri -tor4m -4t3p -t4ra -4tres -tro5v -1try -4ts -t3si -ts4pa -ts5pr -t3st -ts5ul -4t1t -t5uds -5tur -t5ve -1typ -u1a -5udl -ud5r -ud3s -3udv -u1e -ue4t5 -uge4ri -ugs3 -u5gu -u3i -u5kl -uk4ta -uk4tr -u1la -u1le -u5ly -u5pe -up5l -u5q -u3ra -u3re -u4r3eg -u1rer -u3ro -us5a -u3si -u5ska -u5so -us5v -u1te -u1ti -u1to -ut5r -ut5s4 -5u5v -va5d -3varm -1ved -ve4l5e -ve4reg -ve3s -5vet -v5h -vi4l3in -1vis -v5j -v5k -vl4 -v3le -v5li -vls1 -1vo -4v5om -v5p -v5re -v3st -v5su -v5t -3vu -y3a -y5dr -y3e -y3ke -y5ki -yk3li -y3ko -yk4s5 -y3kv -y5li -y5lo -y5mu -yns5 -y5o -y1pe -y3pi -y3re -yr3ek -y3ri -y3si -y3ti -y5t3r -y5ve -zi5o - -.sÃ¥3 -.ær5i -.øv3r -a3tø -a5væ -brød3 -5bæ -5drøv -dstÃ¥4 -3dæ -3dø -e3læ -e3lø -e3rø -er5øn -e5tæ -e5tø -e1væ -e3æ -e5Ã¥ -3fæ -3fø -fø4r5en -giø4 -g4sø -g5sÃ¥ -3gæ -3gø1 -3gÃ¥ -i5tæ -i3ø -3kø -3kÃ¥ -lingeniø4 -l3væ -5løs -m5tÃ¥ -1mæ -3mø -3mÃ¥ -n3kæ -n5tæ -3næ -4n5æb -5nø -o5læ -or3ø -o5Ã¥ -5præ -5pæd -pÃ¥3 -r5kæ -r5tæ -r5tø -r3væ -r5æl -4røn -5rør -3rÃ¥d -r5Ã¥r -s4kÃ¥ -3slÃ¥ -s4næ -5stø -1stÃ¥ -1sæ -4s5æn -1sø -s5øk -sÃ¥4r5 -ti4ø -3træk. -t4sø -t5sÃ¥ -t3væ -u3læ -3værd -1værk -5vÃ¥ -y5væ -æb3l -æ3c -æ3e -æg5a -æ4gek -æ4g5r -ægs5 -æ5i -æ5kv -ælle4 -æn1dr -æ5o -æ1re -ær4g5r -æ3ri -ær4ma -ær4mo -ær5s -æ5si -æ3so -æ3ste -æ3ve -øde5 -ø3e -ø1je -ø3ke -ø3le -øms5 -øn3st -øn4t3 -ø1re -ø3ri -ørne3 -ør5o -ø1ve -Ã¥1d -Ã¥1e -Ã¥5h -Ã¥3l -Ã¥3re -Ã¥rs5t -Ã¥5sk -Ã¥3t - - diff --git a/solr/core/src/test-files/solr/conf/da_compoundDictionary.txt b/solr/core/src/test-files/solr/conf/da_compoundDictionary.txt deleted file mode 100644 index 9a14f40..0000000 --- a/solr/core/src/test-files/solr/conf/da_compoundDictionary.txt +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# A set of words for testing the HyphenationCompound factory, -# in conjunction with the danish hyphenation grammar. -læse -hest diff --git a/solr/core/src/test-files/solr/conf/elevate.xml b/solr/core/src/test-files/solr/conf/elevate.xml deleted file mode 100644 index d15bd5e..0000000 --- a/solr/core/src/test-files/solr/conf/elevate.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/frenchArticles.txt b/solr/core/src/test-files/solr/conf/frenchArticles.txt deleted file mode 100644 index 9141611..0000000 --- a/solr/core/src/test-files/solr/conf/frenchArticles.txt +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# A set of articles for testing the French Elision filter. -# Requiring a text file is a bit weird here... -l -m -t -qu -n -s -j diff --git a/solr/core/src/test-files/solr/conf/hunspell-test.aff b/solr/core/src/test-files/solr/conf/hunspell-test.aff deleted file mode 100644 index d035ad1..0000000 --- a/solr/core/src/test-files/solr/conf/hunspell-test.aff +++ /dev/null @@ -1,13 +0,0 @@ -SET UTF-8 -TRY abcdefghijklmopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ - -SFX A Y 2 -SFX A 0 e n -SFX A 0 e t - -SFX C Y 2 -SFX C 0 d/C c -SFX C 0 c b - -PFX B Y 1 -PFX B 0 s o \ No newline at end of file diff --git a/solr/core/src/test-files/solr/conf/hunspell-test.dic b/solr/core/src/test-files/solr/conf/hunspell-test.dic deleted file mode 100644 index 92c35d2..0000000 --- a/solr/core/src/test-files/solr/conf/hunspell-test.dic +++ /dev/null @@ -1,6 +0,0 @@ -5 -lucen/A -lucene -mahout/A -olr/B -ab/C \ No newline at end of file diff --git a/solr/core/src/test-files/solr/conf/hyphenation.dtd b/solr/core/src/test-files/solr/conf/hyphenation.dtd deleted file mode 100644 index 083c2bd..0000000 --- a/solr/core/src/test-files/solr/conf/hyphenation.dtd +++ /dev/null @@ -1,68 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/keep-1.txt b/solr/core/src/test-files/solr/conf/keep-1.txt deleted file mode 100644 index 8dfe809..0000000 --- a/solr/core/src/test-files/solr/conf/keep-1.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -foo -bar \ No newline at end of file diff --git a/solr/core/src/test-files/solr/conf/keep-2.txt b/solr/core/src/test-files/solr/conf/keep-2.txt deleted file mode 100644 index 646b7ff..0000000 --- a/solr/core/src/test-files/solr/conf/keep-2.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -junk -more \ No newline at end of file diff --git a/solr/core/src/test-files/solr/conf/mapping-ISOLatin1Accent.txt b/solr/core/src/test-files/solr/conf/mapping-ISOLatin1Accent.txt deleted file mode 100644 index ede7742..0000000 --- a/solr/core/src/test-files/solr/conf/mapping-ISOLatin1Accent.txt +++ /dev/null @@ -1,246 +0,0 @@ -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Syntax: -# "source" => "target" -# "source".length() > 0 (source cannot be empty.) -# "target".length() >= 0 (target can be empty.) - -# example: -# "À" => "A" -# "\u00C0" => "A" -# "\u00C0" => "\u0041" -# "ß" => "ss" -# "\t" => " " -# "\n" => "" - -# À => A -"\u00C0" => "A" - -# à => A -"\u00C1" => "A" - -#  => A -"\u00C2" => "A" - -# à => A -"\u00C3" => "A" - -# Ä => A -"\u00C4" => "A" - -# Ã… => A -"\u00C5" => "A" - -# Æ => AE -"\u00C6" => "AE" - -# Ç => C -"\u00C7" => "C" - -# È => E -"\u00C8" => "E" - -# É => E -"\u00C9" => "E" - -# Ê => E -"\u00CA" => "E" - -# Ë => E -"\u00CB" => "E" - -# ÃŒ => I -"\u00CC" => "I" - -# à => I -"\u00CD" => "I" - -# ÃŽ => I -"\u00CE" => "I" - -# à => I -"\u00CF" => "I" - -# IJ => IJ -"\u0132" => "IJ" - -# à => D -"\u00D0" => "D" - -# Ñ => N -"\u00D1" => "N" - -# Ã’ => O -"\u00D2" => "O" - -# Ó => O -"\u00D3" => "O" - -# Ô => O -"\u00D4" => "O" - -# Õ => O -"\u00D5" => "O" - -# Ö => O -"\u00D6" => "O" - -# Ø => O -"\u00D8" => "O" - -# Å’ => OE -"\u0152" => "OE" - -# Þ -"\u00DE" => "TH" - -# Ù => U -"\u00D9" => "U" - -# Ú => U -"\u00DA" => "U" - -# Û => U -"\u00DB" => "U" - -# Ãœ => U -"\u00DC" => "U" - -# à => Y -"\u00DD" => "Y" - -# Ÿ => Y -"\u0178" => "Y" - -# à => a -"\u00E0" => "a" - -# á => a -"\u00E1" => "a" - -# â => a -"\u00E2" => "a" - -# ã => a -"\u00E3" => "a" - -# ä => a -"\u00E4" => "a" - -# Ã¥ => a -"\u00E5" => "a" - -# æ => ae -"\u00E6" => "ae" - -# ç => c -"\u00E7" => "c" - -# è => e -"\u00E8" => "e" - -# é => e -"\u00E9" => "e" - -# ê => e -"\u00EA" => "e" - -# ë => e -"\u00EB" => "e" - -# ì => i -"\u00EC" => "i" - -# í => i -"\u00ED" => "i" - -# î => i -"\u00EE" => "i" - -# ï => i -"\u00EF" => "i" - -# ij => ij -"\u0133" => "ij" - -# ð => d -"\u00F0" => "d" - -# ñ => n -"\u00F1" => "n" - -# ò => o -"\u00F2" => "o" - -# ó => o -"\u00F3" => "o" - -# ô => o -"\u00F4" => "o" - -# õ => o -"\u00F5" => "o" - -# ö => o -"\u00F6" => "o" - -# ø => o -"\u00F8" => "o" - -# Å“ => oe -"\u0153" => "oe" - -# ß => ss -"\u00DF" => "ss" - -# þ => th -"\u00FE" => "th" - -# ù => u -"\u00F9" => "u" - -# ú => u -"\u00FA" => "u" - -# û => u -"\u00FB" => "u" - -# ü => u -"\u00FC" => "u" - -# ý => y -"\u00FD" => "y" - -# ÿ => y -"\u00FF" => "y" - -# ff => ff -"\uFB00" => "ff" - -# ï¬ => fi -"\uFB01" => "fi" - -# fl => fl -"\uFB02" => "fl" - -# ffi => ffi -"\uFB03" => "ffi" - -# ffl => ffl -"\uFB04" => "ffl" - -# ſt => ft -"\uFB05" => "ft" - -# st => st -"\uFB06" => "st" diff --git a/solr/core/src/test-files/solr/conf/old_synonyms.txt b/solr/core/src/test-files/solr/conf/old_synonyms.txt deleted file mode 100644 index a7624f0..0000000 --- a/solr/core/src/test-files/solr/conf/old_synonyms.txt +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -a => aa -b => b1 b2 -c => c1,c2 -a\=>a => b\=>b -a\,a => b\,b -foo,bar,baz - -Television,TV,Televisions diff --git a/solr/core/src/test-files/solr/conf/open-exchange-rates.json b/solr/core/src/test-files/solr/conf/open-exchange-rates.json deleted file mode 100644 index c7bf1ed..0000000 --- a/solr/core/src/test-files/solr/conf/open-exchange-rates.json +++ /dev/null @@ -1,167 +0,0 @@ -{ - "disclaimer": "This data is collected from various providers and provided free of charge for informational purposes only, with no guarantee whatsoever of accuracy, validity, availability or fitness for any purpose; use at your own risk. Other than that - have fun, and please share/watch/fork if you think data like this should be free!", - "license": "Data collected from various providers with public-facing APIs; copyright may apply; not for resale; no warranties given.", - "timestamp": 1332070464, - "base": "USD", - "rates": { - "AED": 3.6732, - "AFN": 48.299999, - "ALL": 105.919998, - "AMD": 388.890015, - "ANG": 1.79, - "AOA": 94.769997, - "ARS": 4.35, - "AUD": 0.943931, - "AWG": 1.7899, - "AZN": 0.7863, - "BAM": 1.48775, - "BBD": 2, - "BDT": 82, - "BGN": 1.4962, - "BHD": 0.37703, - "BIF": 1304.170044, - "BMD": 1, - "BND": 1.2575, - "BOB": 6.91, - "BRL": 1.8003, - "BSD": 1, - "BTN": 50.185001, - "BWP": 7.2307, - "BYR": 8150, - "BZD": 1.9135, - "CAD": 0.9921, - "CDF": 917.276917, - "CHF": 0.9164, - "CLF": 0.02146, - "CLP": 482.75, - "CNY": 6.3239, - "COP": 1760, - "CRC": 507.600006, - "CUP": 1, - "CVE": 84.190002, - "CZK": 18.606001, - "DJF": 179.490005, - "DKK": 5.64424, - "DOP": 39.025002, - "DZD": 74.544998, - "EGP": 6.0385, - "ETB": 17.720449, - "EUR": 0.758956, - "FJD": 1.7734, - "FKP": 0.6316, - "GBP": 0.631373, - "GEL": 1.6469, - "GHS": 1.7455, - "GIP": 0.63165, - "GMD": 31.5, - "GNF": 7100, - "GTQ": 7.6975, - "GYD": 203.699997, - "HKD": 7.76306, - "HNL": 19.055, - "HRK": 5.7333, - "HTG": 41, - "HUF": 219.850006, - "IDR": 9118, - "IEP": 0.5978, - "ILS": 3.7542, - "INR": 50.165001, - "IQD": 1165.5, - "IRR": 12308, - "ISK": 127.440002, - "JMD": 86.699997, - "JOD": 0.7095, - "JPY": 83.445, - "KES": 83.18, - "KGS": 46.699402, - "KHR": 4010.300049, - "KMF": 373.424255, - "KPW": 900, - "KRW": 1125.849976, - "KWD": 0.27925, - "KZT": 147.690002, - "LAK": 7993.799805, - "LBP": 1504, - "LKR": 125.224998, - "LRD": 73.459999, - "LSL": 7.5768, - "LTL": 2.6219, - "LVL": 0.5291, - "LYD": 1.2572, - "MAD": 8.4611, - "MDL": 11.89, - "MGA": 2155, - "MKD": 46.705002, - "MMK": 6.51, - "MNT": 1322.5, - "MOP": 7.9958, - "MRO": 293, - "MUR": 29.110001, - "MVR": 15.36, - "MWK": 165.206207, - "MXN": 12.6745, - "MYR": 3.0575, - "MZN": 27.200001, - "NAD": 7.58, - "NGN": 157.600006, - "NIO": 23.215, - "NOK": 5.73163, - "NPR": 80.620003, - "NZD": 1.212269, - "OMR": 0.38485, - "PAB": 1, - "PEN": 2.674, - "PGK": 2.0627, - "PHP": 43.02, - "PKR": 90.800003, - "PLN": 3.1285, - "PYG": 4245, - "QAR": 3.6415, - "RON": 3.3256, - "RSD": 84.100502, - "RUB": 29.2342, - "RWF": 606.717468, - "SAR": 3.7505, - "SBD": 7.075973, - "SCR": 14.0447, - "SDG": 2.6765, - "SEK": 6.74525, - "SGD": 1.258, - "SHP": 0.63165, - "SLL": 4364.5, - "SOS": 1629, - "SRD": 3.2875, - "STD": 18650, - "SVC": 8.7475, - "SYP": 57.450001, - "SZL": 7.5752, - "THB": 30.700001, - "TJS": 4.7588, - "TMT": 2.85, - "TND": 1.5178, - "TOP": 1.693601, - "TRY": 1.796, - "TTD": 6.40015, - "TWD": 29.532, - "TZS": 1595, - "UAH": 8.029, - "UGX": 2481.699951, - "USD": 1, - "UYU": 19.469999, - "UZS": 1835.75, - "VEF": 4.295, - "VND": 20820, - "VUV": 90.199997, - "WST": 2.247475, - "XAF": 497.898987, - "XCD": 2.7, - "XDR": 0.652794, - "XOF": 498.399994, - "XPF": 90.639999, - "YER": 216.005005, - "ZAR": 7.5688, - "ZMK": 5271.5, - "ZWD": 378.700012, - "ZWL": 322.355011 - } -} \ No newline at end of file diff --git a/solr/core/src/test-files/solr/conf/phrasesuggest.txt b/solr/core/src/test-files/solr/conf/phrasesuggest.txt deleted file mode 100644 index fd4984d..0000000 --- a/solr/core/src/test-files/solr/conf/phrasesuggest.txt +++ /dev/null @@ -1,8 +0,0 @@ -# simple auto-suggest phrase dictionary for testing -# note this uses tabs as separator! -the first phrase 1.0 -the second phrase 2.0 -testing 1234 3.0 -foo 5.0 -the fifth phrase 2.0 -the final phrase 4.0 diff --git a/solr/core/src/test-files/solr/conf/protwords.txt b/solr/core/src/test-files/solr/conf/protwords.txt deleted file mode 100644 index ab7e3e2..0000000 --- a/solr/core/src/test-files/solr/conf/protwords.txt +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#use a protected word file to avoid stemming two -#unrelated words to the same base word. -#to test, we will use words that would normally obviously be stemmed. -cats -ridding -c# -c++ -.net diff --git a/solr/core/src/test-files/solr/conf/schema-binaryfield.xml b/solr/core/src/test-files/solr/conf/schema-binaryfield.xml deleted file mode 100644 index bf0e3e9..0000000 --- a/solr/core/src/test-files/solr/conf/schema-binaryfield.xml +++ /dev/null @@ -1,100 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - diff --git a/solr/core/src/test-files/solr/conf/schema-bm25.xml b/solr/core/src/test-files/solr/conf/schema-bm25.xml deleted file mode 100644 index 54bdc05..0000000 --- a/solr/core/src/test-files/solr/conf/schema-bm25.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - - - - - - - - - - - - - - - - - - 1.2 - 0.76 - - - - - - - - - - - - text - id - - - diff --git a/solr/core/src/test-files/solr/conf/schema-collate.xml b/solr/core/src/test-files/solr/conf/schema-collate.xml deleted file mode 100644 index 7feb73a..0000000 --- a/solr/core/src/test-files/solr/conf/schema-collate.xml +++ /dev/null @@ -1,62 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text - id - - - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/schema-collatefilter.xml b/solr/core/src/test-files/solr/conf/schema-collatefilter.xml deleted file mode 100644 index 19e3eb3..0000000 --- a/solr/core/src/test-files/solr/conf/schema-collatefilter.xml +++ /dev/null @@ -1,61 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text - id - - - - - diff --git a/solr/core/src/test-files/solr/conf/schema-copyfield-test.xml b/solr/core/src/test-files/solr/conf/schema-copyfield-test.xml deleted file mode 100644 index d3e09d9..0000000 --- a/solr/core/src/test-files/solr/conf/schema-copyfield-test.xml +++ /dev/null @@ -1,473 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text - id - - - - - - - - - - - - - - - - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/schema-dfr.xml b/solr/core/src/test-files/solr/conf/schema-dfr.xml deleted file mode 100644 index c4f7d83..0000000 --- a/solr/core/src/test-files/solr/conf/schema-dfr.xml +++ /dev/null @@ -1,70 +0,0 @@ - - - - - - - - - - - - - - I(F) - B - H2 - - - - - - - - I(F) - B - H3 - 900 - - - - - - - - P - L - H2 - 7 - - - - - - - - - - - - - text - id - - - diff --git a/solr/core/src/test-files/solr/conf/schema-folding.xml b/solr/core/src/test-files/solr/conf/schema-folding.xml deleted file mode 100644 index 95b37c3..0000000 --- a/solr/core/src/test-files/solr/conf/schema-folding.xml +++ /dev/null @@ -1,259 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - content - id - - diff --git a/solr/core/src/test-files/solr/conf/schema-ib.xml b/solr/core/src/test-files/solr/conf/schema-ib.xml deleted file mode 100644 index 3d55b2a..0000000 --- a/solr/core/src/test-files/solr/conf/schema-ib.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - - - - - - - - - - - SPL - DF - H2 - - - - - - - - LL - TTF - H3 - 900 - - - - - - - - - - - - text - id - - - diff --git a/solr/core/src/test-files/solr/conf/schema-lmdirichlet.xml b/solr/core/src/test-files/solr/conf/schema-lmdirichlet.xml deleted file mode 100644 index f39922f..0000000 --- a/solr/core/src/test-files/solr/conf/schema-lmdirichlet.xml +++ /dev/null @@ -1,51 +0,0 @@ - - - - - - - - - - - - - - - - - - - - 1000 - - - - - - - - - - - - text - id - - - diff --git a/solr/core/src/test-files/solr/conf/schema-lmjelinekmercer.xml b/solr/core/src/test-files/solr/conf/schema-lmjelinekmercer.xml deleted file mode 100644 index 49b692e..0000000 --- a/solr/core/src/test-files/solr/conf/schema-lmjelinekmercer.xml +++ /dev/null @@ -1,51 +0,0 @@ - - - - - - - - - - - - - - - - - - - - 0.4 - - - - - - - - - - - - text - id - - - diff --git a/solr/core/src/test-files/solr/conf/schema-luceneMatchVersion.xml b/solr/core/src/test-files/solr/conf/schema-luceneMatchVersion.xml deleted file mode 100644 index 7c439d7..0000000 --- a/solr/core/src/test-files/solr/conf/schema-luceneMatchVersion.xml +++ /dev/null @@ -1,56 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - diff --git a/solr/core/src/test-files/solr/conf/schema-minimal.xml b/solr/core/src/test-files/solr/conf/schema-minimal.xml deleted file mode 100644 index 9e2f947..0000000 --- a/solr/core/src/test-files/solr/conf/schema-minimal.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/schema-not-required-unique-key.xml b/solr/core/src/test-files/solr/conf/schema-not-required-unique-key.xml deleted file mode 100644 index f3e5a1a..0000000 --- a/solr/core/src/test-files/solr/conf/schema-not-required-unique-key.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - subject - id - diff --git a/solr/core/src/test-files/solr/conf/schema-numeric.xml b/solr/core/src/test-files/solr/conf/schema-numeric.xml deleted file mode 100644 index 0a12736..0000000 --- a/solr/core/src/test-files/solr/conf/schema-numeric.xml +++ /dev/null @@ -1,78 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/schema-phrasesuggest.xml b/solr/core/src/test-files/solr/conf/schema-phrasesuggest.xml deleted file mode 100644 index da11caf..0000000 --- a/solr/core/src/test-files/solr/conf/schema-phrasesuggest.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text - id - diff --git a/solr/core/src/test-files/solr/conf/schema-replication1.xml b/solr/core/src/test-files/solr/conf/schema-replication1.xml deleted file mode 100644 index 48ecd9f..0000000 --- a/solr/core/src/test-files/solr/conf/schema-replication1.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - id - - diff --git a/solr/core/src/test-files/solr/conf/schema-replication2.xml b/solr/core/src/test-files/solr/conf/schema-replication2.xml deleted file mode 100644 index 65ad6f3..0000000 --- a/solr/core/src/test-files/solr/conf/schema-replication2.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - id - - diff --git a/solr/core/src/test-files/solr/conf/schema-required-fields.xml b/solr/core/src/test-files/solr/conf/schema-required-fields.xml deleted file mode 100644 index 0945629..0000000 --- a/solr/core/src/test-files/solr/conf/schema-required-fields.xml +++ /dev/null @@ -1,439 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text - id - - - - - - - - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/schema-reversed.xml b/solr/core/src/test-files/solr/conf/schema-reversed.xml deleted file mode 100644 index 22844af..0000000 --- a/solr/core/src/test-files/solr/conf/schema-reversed.xml +++ /dev/null @@ -1,85 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - one - id - - - diff --git a/solr/core/src/test-files/solr/conf/schema-sim.xml b/solr/core/src/test-files/solr/conf/schema-sim.xml deleted file mode 100644 index ca2bd78..0000000 --- a/solr/core/src/test-files/solr/conf/schema-sim.xml +++ /dev/null @@ -1,68 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - is there an echo? - - - - - - - - - - - - - - - - - - - - - - - - sim1text - id - - - - diff --git a/solr/core/src/test-files/solr/conf/schema-spellchecker.xml b/solr/core/src/test-files/solr/conf/schema-spellchecker.xml deleted file mode 100644 index 7124065..0000000 --- a/solr/core/src/test-files/solr/conf/schema-spellchecker.xml +++ /dev/null @@ -1,87 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - text - - - - - diff --git a/solr/core/src/test-files/solr/conf/schema-stop-keep.xml b/solr/core/src/test-files/solr/conf/schema-stop-keep.xml deleted file mode 100644 index e3d7839..0000000 --- a/solr/core/src/test-files/solr/conf/schema-stop-keep.xml +++ /dev/null @@ -1,67 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - one - id - - - diff --git a/solr/core/src/test-files/solr/conf/schema-tfidf.xml b/solr/core/src/test-files/solr/conf/schema-tfidf.xml deleted file mode 100644 index 8d2ed78..0000000 --- a/solr/core/src/test-files/solr/conf/schema-tfidf.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - text - id - diff --git a/solr/core/src/test-files/solr/conf/schema-trie.xml b/solr/core/src/test-files/solr/conf/schema-trie.xml deleted file mode 100644 index 5c7b4cc..0000000 --- a/solr/core/src/test-files/solr/conf/schema-trie.xml +++ /dev/null @@ -1,333 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - text - - diff --git a/solr/core/src/test-files/solr/conf/schema.xml b/solr/core/src/test-files/solr/conf/schema.xml deleted file mode 100644 index 20e3bcb..0000000 --- a/solr/core/src/test-files/solr/conf/schema.xml +++ /dev/null @@ -1,713 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - is there an echo? - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - I am your default sim - - diff --git a/solr/core/src/test-files/solr/conf/schema11.xml b/solr/core/src/test-files/solr/conf/schema11.xml deleted file mode 100755 index 6d5fd1a..0000000 --- a/solr/core/src/test-files/solr/conf/schema11.xml +++ /dev/null @@ -1,385 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - text - - diff --git a/solr/core/src/test-files/solr/conf/schema12.xml b/solr/core/src/test-files/solr/conf/schema12.xml deleted file mode 100755 index b6cc3c0..0000000 --- a/solr/core/src/test-files/solr/conf/schema12.xml +++ /dev/null @@ -1,605 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text - id - - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/schema15.xml b/solr/core/src/test-files/solr/conf/schema15.xml deleted file mode 100755 index 48c2c9e..0000000 --- a/solr/core/src/test-files/solr/conf/schema15.xml +++ /dev/null @@ -1,591 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text - id - - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/schema_codec.xml b/solr/core/src/test-files/solr/conf/schema_codec.xml deleted file mode 100644 index e08ab8d..0000000 --- a/solr/core/src/test-files/solr/conf/schema_codec.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - string_f - string_f - diff --git a/solr/core/src/test-files/solr/conf/schemasurround.xml b/solr/core/src/test-files/solr/conf/schemasurround.xml deleted file mode 100644 index 6020cf1..0000000 --- a/solr/core/src/test-files/solr/conf/schemasurround.xml +++ /dev/null @@ -1,613 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text - id - - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-SOLR-749.xml b/solr/core/src/test-files/solr/conf/solrconfig-SOLR-749.xml deleted file mode 100644 index 3a0a0cc..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-SOLR-749.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-altdirectory.xml b/solr/core/src/test-files/solr/conf/solrconfig-altdirectory.xml deleted file mode 100755 index 96cfd2d..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-altdirectory.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-basic-luceneVersion31.xml b/solr/core/src/test-files/solr/conf/solrconfig-basic-luceneVersion31.xml deleted file mode 100644 index 50ac5f1..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-basic-luceneVersion31.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - LUCENE_31 - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-basic.xml b/solr/core/src/test-files/solr/conf/solrconfig-basic.xml deleted file mode 100644 index b23c612..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-basic.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - ${solr.data.dir:} - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-caching.xml b/solr/core/src/test-files/solr/conf/solrconfig-caching.xml deleted file mode 100644 index c6f7cbf..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-caching.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - - - - - - \ No newline at end of file diff --git a/solr/core/src/test-files/solr/conf/solrconfig-delpolicy1.xml b/solr/core/src/test-files/solr/conf/solrconfig-delpolicy1.xml deleted file mode 100644 index 751663c..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-delpolicy1.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - - - - - true - 3 - 100MILLISECONDS - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-delpolicy2.xml b/solr/core/src/test-files/solr/conf/solrconfig-delpolicy2.xml deleted file mode 100644 index 585a1cd..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-delpolicy2.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - value1 - value2 - - - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-elevate.xml b/solr/core/src/test-files/solr/conf/solrconfig-elevate.xml deleted file mode 100644 index b32572a..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-elevate.xml +++ /dev/null @@ -1,191 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - ${solr.data.dir:} - - - - - - single - - - - - - - - - - - - - - - - - 1024 - - - - - - - - - - - true - - 10 - - - - - - - - - - - - - - - - - string - ${elevate.file:elevate.xml} - - - - - - string - ${elevate.data.file:elevate-data.xml} - - - - - explicit - - - elevate - - - - - - explicit - - - dataElevate - - - - - - - - - - max-age=30, public - - - - - solr - solrconfig.xml scheam.xml admin-extra.html - - - - prefix-${solr.test.sys.prop2}-suffix - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-enableplugin.xml b/solr/core/src/test-files/solr/conf/solrconfig-enableplugin.xml deleted file mode 100644 index 5d97ab1..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-enableplugin.xml +++ /dev/null @@ -1,68 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - ${solr.data.dir:} - - - - - - single - - - - - - - true - - - - - - - - - - - - - - - - - - - - - - max-age=30, public - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-functionquery.xml b/solr/core/src/test-files/solr/conf/solrconfig-functionquery.xml deleted file mode 100755 index 12bf3a2..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-functionquery.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - - - - - - - - - - - 0.0 - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-highlight.xml b/solr/core/src/test-files/solr/conf/solrconfig-highlight.xml deleted file mode 100644 index 9da3245..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-highlight.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - - - - - - - - - - - - 100 - - - - - - 70 - - - - - - - ]]> - ]]> - - - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-lazywriter.xml b/solr/core/src/test-files/solr/conf/solrconfig-lazywriter.xml deleted file mode 100644 index 1e2939f..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-lazywriter.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-master.xml b/solr/core/src/test-files/solr/conf/solrconfig-master.xml deleted file mode 100644 index cece0ab..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-master.xml +++ /dev/null @@ -1,76 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - ${solr.data.dir:} - - - single - - - - - - - true - - - - - commit - schema.xml - - - - - - - 4 - true - text,name,subject,title,whitetok - - - - - - - 4 - true - text,name,subject,title,whitetok - - - - - - - - - - max-age=30, public - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-master1-keepOneBackup.xml b/solr/core/src/test-files/solr/conf/solrconfig-master1-keepOneBackup.xml deleted file mode 100644 index 5347975..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-master1-keepOneBackup.xml +++ /dev/null @@ -1,51 +0,0 @@ - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - ${solr.data.dir:} - - - - single - - - - - - - - - - commit - schema-replication2.xml:schema.xml - - 1 - - - - - - - - - max-age=30, public - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-master1.xml b/solr/core/src/test-files/solr/conf/solrconfig-master1.xml deleted file mode 100644 index 019f47e..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-master1.xml +++ /dev/null @@ -1,76 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - ${solr.data.dir:} - - - - single - - - - - - - true - - - - - commit - schema-replication2.xml:schema.xml - - - - - - - 4 - true - text,name,subject,title,whitetok - - - - - - - 4 - true - text,name,subject,title,whitetok - - - - - - - - - - max-age=30, public - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-master2.xml b/solr/core/src/test-files/solr/conf/solrconfig-master2.xml deleted file mode 100644 index b088fb9..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-master2.xml +++ /dev/null @@ -1,76 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - ${solr.data.dir:} - - - - single - - - - - - - true - - - - - startup - schema.xml - - - - - - - 4 - true - text,name,subject,title,whitetok - - - - - - - 4 - true - text,name,subject,title,whitetok - - - - - - - - - - max-age=30, public - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-master3.xml b/solr/core/src/test-files/solr/conf/solrconfig-master3.xml deleted file mode 100644 index cf3ee59..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-master3.xml +++ /dev/null @@ -1,77 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - ${solr.data.dir:} - - - - single - - - - - - - true - - - - - commit - startup - schema.xml - - - - - - - 4 - true - text,name,subject,title,whitetok - - - - - - - 4 - true - text,name,subject,title,whitetok - - - - - - - - - - max-age=30, public - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-mergepolicy.xml b/solr/core/src/test-files/solr/conf/solrconfig-mergepolicy.xml deleted file mode 100644 index 8fa12a7..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-mergepolicy.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - 7 - - 19 - 9 - 1.0 - - - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-nativelock.xml b/solr/core/src/test-files/solr/conf/solrconfig-nativelock.xml deleted file mode 100644 index ac0f70d..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-nativelock.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - - - ${solr.data.dir:} - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - native - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-nocache.xml b/solr/core/src/test-files/solr/conf/solrconfig-nocache.xml deleted file mode 100644 index cb6c92d..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-nocache.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - - ${solr.data.dir:} - - - - - - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-phrasesuggest.xml b/solr/core/src/test-files/solr/conf/solrconfig-phrasesuggest.xml deleted file mode 100644 index 4cdd038..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-phrasesuggest.xml +++ /dev/null @@ -1,63 +0,0 @@ - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - ${solr.data.dir:} - - - - - - - suggest_wfst - org.apache.solr.spelling.suggest.Suggester - org.apache.solr.spelling.suggest.fst.WFSTLookupFactory - suggest_wfst - false - - - true - - phrasesuggest.txt - - - - phrase_suggest - - - - - - - - - true - suggest_wfst - false - - true - - - suggest_wfst - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-propinject-indexdefault.xml b/solr/core/src/test-files/solr/conf/solrconfig-propinject-indexdefault.xml deleted file mode 100644 index efa1e6c..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-propinject-indexdefault.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - - - - 32.0 - - - 4 - 4 - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-propinject.xml b/solr/core/src/test-files/solr/conf/solrconfig-propinject.xml deleted file mode 100644 index f3e5391..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-propinject.xml +++ /dev/null @@ -1,61 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - - - - - 64.0 - - - - 2 - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-querysender-noquery.xml b/solr/core/src/test-files/solr/conf/solrconfig-querysender-noquery.xml deleted file mode 100644 index 206350d..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-querysender-noquery.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-querysender.xml b/solr/core/src/test-files/solr/conf/solrconfig-querysender.xml deleted file mode 100644 index a23c734..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-querysender.xml +++ /dev/null @@ -1,75 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - - - - - - - - - - - - - solr 0 10 mock - rocks 0 10 mock - - - - - - - - - fast_warm 0 10 - mock - - - - - - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-repeater.xml b/solr/core/src/test-files/solr/conf/solrconfig-repeater.xml deleted file mode 100644 index 14d540c..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-repeater.xml +++ /dev/null @@ -1,72 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - ${solr.data.dir:} - - - single - - - - - - - true - - - - - - - - - - - - - - - - commit - schema.xml - - - http://localhost:TEST_PORT/solr/replication - 00:00:01 - - - - - - - - - max-age=30, public - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-reqHandler.incl b/solr/core/src/test-files/solr/conf/solrconfig-reqHandler.incl deleted file mode 100644 index 03f236f..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-reqHandler.incl +++ /dev/null @@ -1,5 +0,0 @@ - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-simplelock.xml b/solr/core/src/test-files/solr/conf/solrconfig-simplelock.xml deleted file mode 100644 index 1760267..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-simplelock.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - - - ${solr.data.dir:} - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - simple - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-slave.xml b/solr/core/src/test-files/solr/conf/solrconfig-slave.xml deleted file mode 100644 index 67b3f1d..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-slave.xml +++ /dev/null @@ -1,68 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - ${solr.data.dir:} - - - single - - - - - - - true - - - - - - - - - - - - - - - - http://localhost:TEST_PORT/solr/replication - 00:00:01 - - - - - - - - - max-age=30, public - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-slave1.xml b/solr/core/src/test-files/solr/conf/solrconfig-slave1.xml deleted file mode 100644 index afcd2ef..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-slave1.xml +++ /dev/null @@ -1,65 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - ${solr.data.dir:} - - - - single - - - - - - - true - - - - - - - - - - - - - - - - - - - - - - - max-age=30, public - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-solcoreproperties.xml b/solr/core/src/test-files/solr/conf/solrconfig-solcoreproperties.xml deleted file mode 100644 index 39398ba..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-solcoreproperties.xml +++ /dev/null @@ -1,66 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - ${solr.data.dir:} - - - - - - single - - - - - - - true - - - - - - - ${foo.foo1} - ${foo.foo2} - - - - - - - - - - - - max-age=30, public - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-spellcheckcomponent.xml b/solr/core/src/test-files/solr/conf/solrconfig-spellcheckcomponent.xml deleted file mode 100644 index 597f89c..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-spellcheckcomponent.xml +++ /dev/null @@ -1,165 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - ${solr.data.dir:} - - - - - - - lowerpunctfilt - - - default - lowerfilt - spellchecker1 - true - - - default_teststop - default_teststop - true - teststop - - - direct - solr.DirectSolrSpellChecker - 3 - 100 - teststop - - - direct_lowerfilt - solr.DirectSolrSpellChecker - 3 - 100 - lowerfilt - - - threshold - lowerfilt - spellcheckerThreshold - true - .29 - - - threshold_direct - solr.DirectSolrSpellChecker - lowerfilt - spellcheckerThreshold - true - .29 - - - multipleFields - lowerfilt1and2 - spellcheckerMultipleFields - true - - - - jarowinkler - lowerfilt - - org.apache.lucene.search.spell.JaroWinklerDistance - spellchecker2 - - - - solr.FileBasedSpellChecker - external - spellings.txt - UTF-8 - spellchecker3 - - - - freq - lowerfilt - spellcheckerFreq - - freq - true - - - fqcn - lowerfilt - spellcheckerFQCN - org.apache.solr.spelling.SampleComparator - true - - - perDict - org.apache.solr.handler.component.DummyCustomParamSpellChecker - lowerfilt - - - - - - - - - - false - - false - - 1 - - - spellcheck - - - - - dismax - lowerfilt1^1 - - - spellcheck - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-spellchecker.xml b/solr/core/src/test-files/solr/conf/solrconfig-spellchecker.xml deleted file mode 100644 index 72fc0f6..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-spellchecker.xml +++ /dev/null @@ -1,142 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - - - - - - - - suggest - org.apache.solr.spelling.suggest.Suggester - org.apache.solr.spelling.suggest.jaspell.JaspellLookup - suggest - suggest - true - - - 0.0 - - - - - - - suggest_tst - org.apache.solr.spelling.suggest.Suggester - org.apache.solr.spelling.suggest.tst.TSTLookup - suggest - suggest_tst - true - - - 0.0 - - - - - - - suggest_fst - org.apache.solr.spelling.suggest.Suggester - org.apache.solr.spelling.suggest.fst.FSTLookup - suggest - suggest_fst - true - - - 5 - true - - - - - - - suggest_wfst - org.apache.solr.spelling.suggest.Suggester - org.apache.solr.spelling.suggest.fst.WFSTLookupFactory - suggest - suggest_wfst - true - - - true - - - - - - - true - suggest - true - - - suggest_jaspell - - - - - - - true - suggest_tst - true - - - suggest_tst - - - - - - - true - suggest_fst - false - - - suggest_fst - - - - - - - true - suggest_wfst - false - - - suggest_wfst - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-termindex.xml b/solr/core/src/test-files/solr/conf/solrconfig-termindex.xml deleted file mode 100644 index d5fcdad..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-termindex.xml +++ /dev/null @@ -1,62 +0,0 @@ - - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - - - - - - - - 256 - - - - - - - - - 12 - - - - - - solr - solrconfig.xml scheam.xml admin-extra.html - - - - prefix-${solr.test.sys.prop2}-suffix - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-tlog.xml b/solr/core/src/test-files/solr/conf/solrconfig-tlog.xml deleted file mode 100644 index 9933392..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-tlog.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - ${solr.data.dir:} - - - - - - - true - - - - - - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-transformers.xml b/solr/core/src/test-files/solr/conf/solrconfig-transformers.xml deleted file mode 100644 index 776d49a..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-transformers.xml +++ /dev/null @@ -1,77 +0,0 @@ - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - - - - 100 - - - - x1 - x2 - - - - - xA - xA - - - - - - - - 88 - 99 - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-update-processor-chains.xml b/solr/core/src/test-files/solr/conf/solrconfig-update-processor-chains.xml deleted file mode 100644 index ea35d56..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-update-processor-chains.xml +++ /dev/null @@ -1,236 +0,0 @@ - - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - - - - solr.TrieIntField - solr.TrieLongField - - - - min_foo_l - - - max_foo_l - - - ; - - primary_author_s1 - - - - primary_author_s1 - first_foo_l - - - - - - - - - - - - - - foo_t - - - - - - - - - foo_t - - - - - - foo.* - bar.* - - .*HOSS.* - - - - - - foo.* - bar.* - - - solr.DateField - - - .*HOSS.* - - - - - - foo.* - bar.* - - - solr.DateField - .*HOSS.* - - - - - - - name - foo_t - - - - - - name - foo_t - - - - - - - foo.* - bar.*_s - - - - - - nametext - text_sw - - - - - - solr.DateField - solr.StrField - - - - - - solr.DateField - solr.StrField - - foo.* - - - - - - - - - - - - - - - - - - foo.* - yak.* - - - - - - - - - - - foo_s - - - - - string - ; - - - - - - foo_s - bar_s - - - - - foo_s - bar_s - - - - - foo_i - foo_s - bar_s - - - - - foo_i - foo_s - bar_s - - - - - - html_s - - - - - - - content - title - \s+ - X - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig-xinclude.xml b/solr/core/src/test-files/solr/conf/solrconfig-xinclude.xml deleted file mode 100644 index 5135a7e..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig-xinclude.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig.xml b/solr/core/src/test-files/solr/conf/solrconfig.xml deleted file mode 100644 index 653357c..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig.xml +++ /dev/null @@ -1,541 +0,0 @@ - - - - - - - - - - - - ${solr.data.dir:} - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - - 1000 - 8 - - - 10 - - single - true - - - - - - - - - - - - ${solr.data.dir:} - - - - - - - - 1024 - - - - - - - - - - - true - - - - - - - 10 - - - - - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - true - - - - - - - dismax - *:* - 0.01 - - text^0.5 features_t^1.0 subject^1.4 title_stemmed^2.0 - - - text^0.2 features_t^1.1 subject^1.4 title_stemmed^2.0 title^1.5 - - - ord(weight)^0.5 recip(rord(iind),1,1000,1000)^0.3 - - - 3<-1 5<-2 6<90% - - 100 - - - - - - dismax - - - - - - - - - - - 4 - true - text,name,subject,title,whitetok - - - - - - - 4 - true - text,name,subject,title,whitetok - - - - - - - - - lowerpunctfilt - - - default - lowerfilt - spellchecker1 - false - - - direct - DirectSolrSpellChecker - lowerfilt - 3 - - - multipleFields - lowerfilt1and2 - spellcheckerMultipleFields - false - - - - jarowinkler - lowerfilt - - org.apache.lucene.search.spell.JaroWinklerDistance - spellchecker2 - - - - solr.FileBasedSpellChecker - external - spellings.txt - UTF-8 - spellchecker3 - - - - freq - lowerfilt - spellcheckerFreq - - freq - false - - - fqcn - lowerfilt - spellcheckerFQCN - org.apache.solr.spelling.SampleComparator - false - - - perDict - org.apache.solr.handler.component.DummyCustomParamSpellChecker - lowerfilt - - - - - - - - termsComp - - - - - - - - - false - - false - - 1 - - - spellcheck - - - - - direct - false - false - 1 - - - spellcheck - - - - - dismax - lowerfilt1^1 - - - spellcheck - - - - - - - - - - - tvComponent - - - - - - - - - - - - 100 - - - - - - 70 - - - - - - - ]]> - ]]> - - - - - - - - - - - - - 10 - .,!? - - - - - - WORD - en - US - - - - - - - - - - - max-age=30, public - - - - - - - explicit - true - - - - - solr - solrconfig.xml scheam.xml admin-extra.html - - - - prefix-${solr.test.sys.prop2}-suffix - - - - - - false - true - v_t,t_field - org.apache.solr.update.processor.TextProfileSignature - - - - - - false - false - id - - org.apache.solr.update.processor.Lookup3Signature - - - - - - - true - non_indexed_signature_sS - false - v_t,t_field - org.apache.solr.update.processor.TextProfileSignature - - - - - - - uniq - uniq2 - uniq3 - - - - - - - - - regex_dup_A_s - x - x_x - - - - regex_dup_B_s - x - x_x - - - - - - - - regex_dup_A_s - x - x_x - - - regex_dup_B_s - x - x_x - - - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig_codec.xml b/solr/core/src/test-files/solr/conf/solrconfig_codec.xml deleted file mode 100644 index 28b4608..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig_codec.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - diff --git a/solr/core/src/test-files/solr/conf/solrconfig_perf.xml b/solr/core/src/test-files/solr/conf/solrconfig_perf.xml deleted file mode 100755 index 0c5e205..0000000 --- a/solr/core/src/test-files/solr/conf/solrconfig_perf.xml +++ /dev/null @@ -1,75 +0,0 @@ - - - - - ${tests.luceneMatchVersion:LUCENE_CURRENT} - - ${solr.data.dir:} - - - - - - - - - - - - - - - true - 20 - 200 - false - 2 - - - - - - - - - - - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/stemdict.txt b/solr/core/src/test-files/solr/conf/stemdict.txt deleted file mode 100644 index f57a4ad..0000000 --- a/solr/core/src/test-files/solr/conf/stemdict.txt +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#----------------------------------------------------------------------- -# test that we can override the stemming algorithm with our own mappings -# these must be tab-separated -monkeys monkey -otters otter -# some crazy ones that a stemmer would never do -dogs cat diff --git a/solr/core/src/test-files/solr/conf/stop-1.txt b/solr/core/src/test-files/solr/conf/stop-1.txt deleted file mode 100644 index 8dfe809..0000000 --- a/solr/core/src/test-files/solr/conf/stop-1.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -foo -bar \ No newline at end of file diff --git a/solr/core/src/test-files/solr/conf/stop-2.txt b/solr/core/src/test-files/solr/conf/stop-2.txt deleted file mode 100644 index 646b7ff..0000000 --- a/solr/core/src/test-files/solr/conf/stop-2.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -junk -more \ No newline at end of file diff --git a/solr/core/src/test-files/solr/conf/stop-snowball.txt b/solr/core/src/test-files/solr/conf/stop-snowball.txt deleted file mode 100644 index 1c0c6f5..0000000 --- a/solr/core/src/test-files/solr/conf/stop-snowball.txt +++ /dev/null @@ -1,10 +0,0 @@ - | This is a file in snowball format, empty lines are ignored, '|' is a comment - | Additionally, multiple words can be on the same line, allowing stopwords to be - | arranged in tables (useful in some languages where they might inflect) - - | fictitious table below - -|third person singular -|Subject Object Possessive Reflexive -he him his himself| masculine -she her hers herself| feminine diff --git a/solr/core/src/test-files/solr/conf/stoptypes-1.txt b/solr/core/src/test-files/solr/conf/stoptypes-1.txt deleted file mode 100644 index 456348e..0000000 --- a/solr/core/src/test-files/solr/conf/stoptypes-1.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - - diff --git a/solr/core/src/test-files/solr/conf/stoptypes-2.txt b/solr/core/src/test-files/solr/conf/stoptypes-2.txt deleted file mode 100644 index d8a3810..0000000 --- a/solr/core/src/test-files/solr/conf/stoptypes-2.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - - diff --git a/solr/core/src/test-files/solr/conf/stopwithbom.txt b/solr/core/src/test-files/solr/conf/stopwithbom.txt deleted file mode 100644 index eb5f6e1..0000000 --- a/solr/core/src/test-files/solr/conf/stopwithbom.txt +++ /dev/null @@ -1 +0,0 @@ -BOMsAreEvil diff --git a/solr/core/src/test-files/solr/conf/stopwords.txt b/solr/core/src/test-files/solr/conf/stopwords.txt deleted file mode 100644 index b5824da..0000000 --- a/solr/core/src/test-files/solr/conf/stopwords.txt +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#----------------------------------------------------------------------- -# a couple of test stopwords to test that the words are really being -# configured from this file: -stopworda -stopwordb - -#Standard english stop words taken from Lucene's StopAnalyzer -a -an -and -are -as -at -be -but -by -for -if -in -into -is -it -no -not -of -on -or -s -such -t -that -the -their -then -there -these -they -this -to -was -will -with - diff --git a/solr/core/src/test-files/solr/conf/stopwordsWrongEncoding.txt b/solr/core/src/test-files/solr/conf/stopwordsWrongEncoding.txt deleted file mode 100644 index 0d305c8..0000000 --- a/solr/core/src/test-files/solr/conf/stopwordsWrongEncoding.txt +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# stopwords in the wrong encoding (ISO-8859-1). -# tests resourceloader's ability to report wrongly encoded files. -bañadores diff --git a/solr/core/src/test-files/solr/conf/synonyms.txt b/solr/core/src/test-files/solr/conf/synonyms.txt deleted file mode 100644 index b0e31cb..0000000 --- a/solr/core/src/test-files/solr/conf/synonyms.txt +++ /dev/null @@ -1,31 +0,0 @@ -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#----------------------------------------------------------------------- -#some test synonym mappings unlikely to appear in real input text -aaa => aaaa -bbb => bbbb1 bbbb2 -ccc => cccc1,cccc2 -a\=>a => b\=>b -a\,a => b\,b -fooaaa,baraaa,bazaaa - -# Some synonym groups specific to this example -GB,gib,gigabyte,gigabytes -MB,mib,megabyte,megabytes -Television, Televisions, TV, TVs -#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming -#after us won't split it into two words. - -# Synonym mappings can be used for spelling correction too -pixima => pixma - diff --git a/solr/core/src/test-files/solr/conf/wdftypes.txt b/solr/core/src/test-files/solr/conf/wdftypes.txt deleted file mode 100644 index 7378b08..0000000 --- a/solr/core/src/test-files/solr/conf/wdftypes.txt +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# A customized type mapping for WordDelimiterFilterFactory -# the allowable types are: LOWER, UPPER, ALPHA, DIGIT, ALPHANUM, SUBWORD_DELIM -# -# the default for any character without a mapping is always computed from -# Unicode character properties - -# Map the $, %, '.', and ',' characters to DIGIT -# This might be useful for financial data. -$ => DIGIT -% => DIGIT -. => DIGIT -\u002C => DIGIT - -# in some cases you might not want to split on ZWJ -# this also tests the case where we need a bigger byte[] -# see http://en.wikipedia.org/wiki/Zero-width_joiner -\u200D => ALPHANUM diff --git a/solr/core/src/test-files/solr/conf/xslt/dummy-using-include.xsl b/solr/core/src/test-files/solr/conf/xslt/dummy-using-include.xsl deleted file mode 100644 index f10cfbf..0000000 --- a/solr/core/src/test-files/solr/conf/xslt/dummy-using-include.xsl +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/xslt/dummy.xsl b/solr/core/src/test-files/solr/conf/xslt/dummy.xsl deleted file mode 100644 index fbbd8f7..0000000 --- a/solr/core/src/test-files/solr/conf/xslt/dummy.xsl +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - - - - - - - - - - diff --git a/solr/core/src/test-files/solr/conf/xslt/xsl-update-handler-test.xsl b/solr/core/src/test-files/solr/conf/xslt/xsl-update-handler-test.xsl deleted file mode 100644 index 2e7359a..0000000 --- a/solr/core/src/test-files/solr/conf/xslt/xsl-update-handler-test.xsl +++ /dev/null @@ -1,49 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/solr/core/src/test-files/solr/lib/README b/solr/core/src/test-files/solr/lib/README deleted file mode 100644 index b7ca5b8..0000000 --- a/solr/core/src/test-files/solr/lib/README +++ /dev/null @@ -1,18 +0,0 @@ - - -Items under this directory are used by TestConfig.testLibs() diff --git a/solr/core/src/test-files/solr/lib/classes/empty-file-main-lib.txt b/solr/core/src/test-files/solr/lib/classes/empty-file-main-lib.txt deleted file mode 100644 index 8b13789..0000000 --- a/solr/core/src/test-files/solr/lib/classes/empty-file-main-lib.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/solr/core/src/test-files/solr/solr.xml b/solr/core/src/test-files/solr/solr.xml index e99cbf9..ccfb224 100644 --- a/solr/core/src/test-files/solr/solr.xml +++ b/solr/core/src/test-files/solr/solr.xml @@ -29,6 +29,6 @@ If 'null' (or absent), cores will not be manageable via request handler --> - + diff --git a/solr/core/src/test/org/apache/solr/AnalysisAfterCoreReloadTest.java b/solr/core/src/test/org/apache/solr/AnalysisAfterCoreReloadTest.java new file mode 100644 index 0000000..9ba6918 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/AnalysisAfterCoreReloadTest.java @@ -0,0 +1,174 @@ +package org.apache.solr; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.File; +import java.io.IOException; + +import org.apache.commons.io.FileUtils; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServer; +import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.apache.solr.client.solrj.impl.HttpSolrServer; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.request.AbstractUpdateRequest.ACTION; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.core.SolrCore; +import org.apache.solr.util.AbstractSolrTestCase; +import org.junit.After; + +public class AnalysisAfterCoreReloadTest extends AbstractSolrTestCase { + private File homeDir; + int port = 0; + static final String context = "/solr"; + JettySolrRunner jetty; + static final String collection = "collection1"; + + @After + public void cleanUp() throws Exception { + jetty.stop(); + if (homeDir != null && homeDir.isDirectory() && homeDir.exists()) + recurseDelete(homeDir); + } + + @Override + public String getSolrHome() { + return homeDir.getAbsolutePath(); + } + + @Override + public void setUp() throws Exception { + homeDir = new File(TEMP_DIR + File.separator + "solr-test-home-" + System.nanoTime()); + homeDir.mkdirs(); + FileUtils.copyDirectory(new File(getFile("solr/" + collection).getParent()), homeDir, false); + + super.setUp(); + + jetty = new JettySolrRunner(getSolrHome(), context, 0 ); + jetty.start(false); + port = jetty.getLocalPort(); + } + + public void testStopwordsAfterCoreReload() throws Exception { + SolrInputDocument doc = new SolrInputDocument(); + doc.setField( "id", "42" ); + doc.setField( "teststop", "terma stopworda stopwordb stopwordc" ); + + // default stopwords - stopworda and stopwordb + + UpdateRequest up = new UpdateRequest(); + up.setAction(ACTION.COMMIT, true, true); + up.add( doc ); + up.process( getSolrCore() ); + + SolrQuery q = new SolrQuery(); + QueryRequest r = new QueryRequest( q ); + q.setQuery( "teststop:terma" ); + assertEquals( 1, r.process( getSolrCore() ).getResults().size() ); + + q = new SolrQuery(); + r = new QueryRequest( q ); + q.setQuery( "teststop:stopworda" ); + assertEquals( 0, r.process( getSolrCore() ).getResults().size() ); + + q = new SolrQuery(); + r = new QueryRequest( q ); + q.setQuery( "teststop:stopwordb" ); + assertEquals( 0, r.process( getSolrCore() ).getResults().size() ); + + q = new SolrQuery(); + r = new QueryRequest( q ); + q.setQuery( "teststop:stopwordc" ); + assertEquals( 1, r.process( getSolrCore() ).getResults().size() ); + + // overwrite stopwords file with stopword list ["stopwordc"] and reload the core + overwriteStopwords("stopwordc\n"); + SolrServer coreadmin = getSolrAdmin(); + CoreAdminRequest.reloadCore(collection, coreadmin); + + up.process( getSolrCore() ); + + q = new SolrQuery(); + r = new QueryRequest( q ); + q.setQuery( "teststop:terma" ); + assertEquals( 1, r.process( getSolrCore() ).getResults().size() ); + + q = new SolrQuery(); + r = new QueryRequest( q ); + q.setQuery( "teststop:stopworda" ); + // stopworda is no longer a stopword + assertEquals( 1, r.process( getSolrCore() ).getResults().size() ); + + q = new SolrQuery(); + r = new QueryRequest( q ); + q.setQuery( "teststop:stopwordb" ); + // stopwordb is no longer a stopword + assertEquals( 1, r.process( getSolrCore() ).getResults().size() ); + + q = new SolrQuery(); + r = new QueryRequest( q ); + q.setQuery( "teststop:stopwordc" ); + // stopwordc should be a stopword + assertEquals( 0, r.process( getSolrCore() ).getResults().size() ); + } + + private void overwriteStopwords(String stopwords) throws IOException { + SolrCore core = h.getCoreContainer().getCore(collection); + try { + String configDir = core.getResourceLoader().getConfigDir(); + File file = new File(configDir, "stopwords.txt"); + FileUtils.writeStringToFile(file, stopwords); + } finally { + core.close(); + } + } + + protected SolrServer getSolrAdmin() { + return createServer(""); + } + protected SolrServer getSolrCore() { + return createServer(collection); + } + private SolrServer createServer( String name ) { + try { + // setup the server... + String url = "http://localhost:"+port+context+"/"+name; + HttpSolrServer s = new HttpSolrServer( url ); + s.setConnectionTimeout(SolrTestCaseJ4.DEFAULT_CONNECTION_TIMEOUT); + s.setDefaultMaxConnectionsPerHost(100); + s.setMaxTotalConnections(100); + return s; + } + catch( Exception ex ) { + throw new RuntimeException( ex ); + } + } + + @Override + public String getSchemaFile() { + return "schema.xml"; + } + + @Override + public String getSolrConfigFile() { + return "solrconfig.xml"; + } + +} \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java index acb3ed1..9f68288 100644 --- a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java +++ b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -128,6 +128,11 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 { IndexWriter writer = ((DirectUpdateHandler2)core.getUpdateHandler()).getSolrCoreState().getIndexWriter(core); assertEquals("Mergefactor was not picked up", 8, ((LogMergePolicy)writer.getConfig().getMergePolicy()).getMergeFactor()); + // test stats call + NamedList stats = core.getStatistics(); + assertEquals("collection1", stats.get("coreName")); + assertTrue(stats.get("refCount") != null); + lrf.args.put(CommonParams.VERSION,"2.2"); assertQ("test query on empty index", req("qlkciyopsbgzyvkylsjhchghjrdf") diff --git a/solr/core/src/test/org/apache/solr/ConvertedLegacyTest.java b/solr/core/src/test/org/apache/solr/ConvertedLegacyTest.java index ef7851a..9150af4 100644 --- a/solr/core/src/test/org/apache/solr/ConvertedLegacyTest.java +++ b/solr/core/src/test/org/apache/solr/ConvertedLegacyTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/DisMaxRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/DisMaxRequestHandlerTest.java index ab1859f..54d5336 100644 --- a/solr/core/src/test/org/apache/solr/DisMaxRequestHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/DisMaxRequestHandlerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -194,16 +194,5 @@ public class DisMaxRequestHandlerTest extends SolrTestCaseJ4 { assertTrue(p_bool.matcher(resp).find()); } - - @Test - public void testSimplestParams() throws Exception { - - assertQ("match w/o only q param", - req("qt", "dismaxNoDefaults", - "q","guide") - ,"//*[@numFound='2']" - ); - - } } diff --git a/solr/core/src/test/org/apache/solr/EchoParamsTest.java b/solr/core/src/test/org/apache/solr/EchoParamsTest.java index dded78d..d41ab5d 100644 --- a/solr/core/src/test/org/apache/solr/EchoParamsTest.java +++ b/solr/core/src/test/org/apache/solr/EchoParamsTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/MinimalSchemaTest.java b/solr/core/src/test/org/apache/solr/MinimalSchemaTest.java index a22763a..7098269 100644 --- a/solr/core/src/test/org/apache/solr/MinimalSchemaTest.java +++ b/solr/core/src/test/org/apache/solr/MinimalSchemaTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -37,7 +37,7 @@ public class MinimalSchemaTest extends SolrTestCaseJ4 { */ @BeforeClass public static void beforeClass() throws Exception { - initCore("solr/conf/solrconfig.xml","solr/conf/schema-minimal.xml"); + initCore("solr/collection1/conf/solrconfig.xml","solr/collection1/conf/schema-minimal.xml"); /* make sure some misguided soul doesn't inadvertently give us a uniqueKey field and defeat the point of the tests diff --git a/solr/core/src/test/org/apache/solr/OutputWriterTest.java b/solr/core/src/test/org/apache/solr/OutputWriterTest.java index 48daa61..864aaef 100644 --- a/solr/core/src/test/org/apache/solr/OutputWriterTest.java +++ b/solr/core/src/test/org/apache/solr/OutputWriterTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/SampleTest.java b/solr/core/src/test/org/apache/solr/SampleTest.java index a9ffa9a..8cfa171 100644 --- a/solr/core/src/test/org/apache/solr/SampleTest.java +++ b/solr/core/src/test/org/apache/solr/SampleTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/SolrInfoMBeanTest.java b/solr/core/src/test/org/apache/solr/SolrInfoMBeanTest.java index 70e7d72..8d51177 100644 --- a/solr/core/src/test/org/apache/solr/SolrInfoMBeanTest.java +++ b/solr/core/src/test/org/apache/solr/SolrInfoMBeanTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java b/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java index fc9132f..a681b19 100755 --- a/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java +++ b/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java @@ -17,6 +17,7 @@ package org.apache.solr; * limitations under the License. */ +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.response.QueryResponse; @@ -31,6 +32,7 @@ import org.apache.solr.common.util.NamedList; * * @since solr 4.0 */ +@Slow public class TestDistributedGrouping extends BaseDistributedSearchTestCase { String t1="a_t"; diff --git a/solr/core/src/test/org/apache/solr/TestDistributedSearch.java b/solr/core/src/test/org/apache/solr/TestDistributedSearch.java index a5c14bc..2a3948b 100755 --- a/solr/core/src/test/org/apache/solr/TestDistributedSearch.java +++ b/solr/core/src/test/org/apache/solr/TestDistributedSearch.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -24,6 +24,7 @@ import java.util.Map; import org.apache.commons.lang.StringUtils; import org.apache.lucene.search.FieldCache; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.embedded.JettySolrRunner; @@ -43,6 +44,7 @@ import org.apache.solr.common.util.NamedList; * * @since solr 1.3 */ +@Slow public class TestDistributedSearch extends BaseDistributedSearchTestCase { String t1="a_t"; @@ -348,7 +350,13 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase { downJettys.add(downJetty); } - queryPartialResults(upShards, upClients, "q","*:*",ShardParams.SHARDS_INFO,"true",ShardParams.SHARDS_TOLERANT,"true"); + queryPartialResults(upShards, upClients, + "q","*:*", + "facet","true", + "facet.field",t1, + "facet.limit",5, + ShardParams.SHARDS_INFO,"true", + ShardParams.SHARDS_TOLERANT,"true"); // restart the jettys for (JettySolrRunner downJetty : downJettys) { diff --git a/solr/core/src/test/org/apache/solr/TestGroupingSearch.java b/solr/core/src/test/org/apache/solr/TestGroupingSearch.java index 72a1de3..d6c8573 100644 --- a/solr/core/src/test/org/apache/solr/TestGroupingSearch.java +++ b/solr/core/src/test/org/apache/solr/TestGroupingSearch.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/TestJoin.java b/solr/core/src/test/org/apache/solr/TestJoin.java index 60b6e8f..1eb94dc 100644 --- a/solr/core/src/test/org/apache/solr/TestJoin.java +++ b/solr/core/src/test/org/apache/solr/TestJoin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/TestPluginEnable.java b/solr/core/src/test/org/apache/solr/TestPluginEnable.java index 454302f..d2505c1 100644 --- a/solr/core/src/test/org/apache/solr/TestPluginEnable.java +++ b/solr/core/src/test/org/apache/solr/TestPluginEnable.java @@ -1,6 +1,6 @@ package org.apache.solr; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/TestRandomFaceting.java b/solr/core/src/test/org/apache/solr/TestRandomFaceting.java index fc7e271..efa5153 100644 --- a/solr/core/src/test/org/apache/solr/TestRandomFaceting.java +++ b/solr/core/src/test/org/apache/solr/TestRandomFaceting.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,19 +19,16 @@ package org.apache.solr; import org.apache.lucene.search.FieldCache; import org.apache.lucene.util._TestUtil; -import org.apache.noggit.JSONUtil; -import org.apache.noggit.ObjectBuilder; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; -import org.apache.solr.util.TestUtils; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import java.util.*; +@Slow public class TestRandomFaceting extends SolrTestCaseJ4 { public static final String FOO_STRING_FIELD = "foo_s1"; @@ -80,7 +77,7 @@ public class TestRandomFaceting extends SolrTestCaseJ4 { model = indexDocs(types, model, ndocs); } - void deleteSomeDocs() throws Exception { + void deleteSomeDocs() { Random rand = random(); int percent = rand.nextInt(100); if (model == null) return; diff --git a/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java b/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java index 518fe17..c9b739d 100644 --- a/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java +++ b/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -37,7 +37,7 @@ import java.util.Properties; * @since solr 1.4 */ public class TestSolrCoreProperties extends LuceneTestCase { - private static final String CONF_DIR = "." + File.separator + "solr" + File.separator + "conf" + File.separator; + private static final String CONF_DIR = "." + File.separator + "solr" + File.separator + "collection1" + File.separator + "conf" + File.separator; JettySolrRunner solrJetty; SolrServer client; @@ -106,8 +106,8 @@ public class TestSolrCoreProperties extends LuceneTestCase { getClass().getName() + "-" + System.currentTimeMillis()); - dataDir = new File(homeDir, "data"); - confDir = new File(homeDir, "conf"); + dataDir = new File(homeDir + "/collection1", "data"); + confDir = new File(homeDir + "/collection1", "conf"); homeDir.mkdirs(); diff --git a/solr/core/src/test/org/apache/solr/TestTrie.java b/solr/core/src/test/org/apache/solr/TestTrie.java index 4637e03..271501f 100644 --- a/solr/core/src/test/org/apache/solr/TestTrie.java +++ b/solr/core/src/test/org/apache/solr/TestTrie.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/CommonGramsFilterFactoryTest.java b/solr/core/src/test/org/apache/solr/analysis/CommonGramsFilterFactoryTest.java index 25ca68c..27841e7 100644 --- a/solr/core/src/test/org/apache/solr/analysis/CommonGramsFilterFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/analysis/CommonGramsFilterFactoryTest.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -38,7 +38,7 @@ import java.util.HashMap; public class CommonGramsFilterFactoryTest extends BaseTokenStreamTestCase { public void testInform() throws Exception { - ResourceLoader loader = new SolrResourceLoader(null, null); + ResourceLoader loader = new SolrResourceLoader("solr/collection1"); assertTrue("loader is null and it shouldn't be", loader != null); CommonGramsFilterFactory factory = new CommonGramsFilterFactory(); Map args = new HashMap(); @@ -88,7 +88,7 @@ public class CommonGramsFilterFactoryTest extends BaseTokenStreamTestCase { * If no words are provided, then a set of english default stopwords is used. */ public void testDefaults() throws Exception { - ResourceLoader loader = new SolrResourceLoader(null, null); + ResourceLoader loader = new SolrResourceLoader("solr/collection1"); assertTrue("loader is null and it shouldn't be", loader != null); CommonGramsFilterFactory factory = new CommonGramsFilterFactory(); factory.setLuceneMatchVersion(TEST_VERSION_CURRENT); diff --git a/solr/core/src/test/org/apache/solr/analysis/CommonGramsQueryFilterFactoryTest.java b/solr/core/src/test/org/apache/solr/analysis/CommonGramsQueryFilterFactoryTest.java index 4b0213a..f648432 100644 --- a/solr/core/src/test/org/apache/solr/analysis/CommonGramsQueryFilterFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/analysis/CommonGramsQueryFilterFactoryTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -37,7 +37,7 @@ import java.util.HashMap; public class CommonGramsQueryFilterFactoryTest extends BaseTokenStreamTestCase { public void testInform() throws Exception { - ResourceLoader loader = new SolrResourceLoader(null, null); + ResourceLoader loader = new SolrResourceLoader("solr/collection1"); assertTrue("loader is null and it shouldn't be", loader != null); CommonGramsQueryFilterFactory factory = new CommonGramsQueryFilterFactory(); Map args = new HashMap(); @@ -87,7 +87,7 @@ public class CommonGramsQueryFilterFactoryTest extends BaseTokenStreamTestCase { * If no words are provided, then a set of english default stopwords is used. */ public void testDefaults() throws Exception { - ResourceLoader loader = new SolrResourceLoader(null, null); + ResourceLoader loader = new SolrResourceLoader("solr/collection1"); assertTrue("loader is null and it shouldn't be", loader != null); CommonGramsQueryFilterFactory factory = new CommonGramsQueryFilterFactory(); factory.setLuceneMatchVersion(TEST_VERSION_CURRENT); diff --git a/solr/core/src/test/org/apache/solr/analysis/DoubleMetaphoneFilterFactoryTest.java b/solr/core/src/test/org/apache/solr/analysis/DoubleMetaphoneFilterFactoryTest.java index c9c59c3..7b08dbb 100644 --- a/solr/core/src/test/org/apache/solr/analysis/DoubleMetaphoneFilterFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/analysis/DoubleMetaphoneFilterFactoryTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java b/solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java index a4a9a57..68d2ace 100644 --- a/solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java +++ b/solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -267,7 +267,7 @@ public class LegacyHTMLStripCharFilterTest extends BaseTokenStreamTestCase { } @Override - protected Reader initReader(Reader reader) { + protected Reader initReader(String fieldName, Reader reader) { return new LegacyHTMLStripCharFilter(CharReader.get(new BufferedReader(reader))); } }; diff --git a/solr/core/src/test/org/apache/solr/analysis/SnowballPorterFilterFactoryTest.java b/solr/core/src/test/org/apache/solr/analysis/SnowballPorterFilterFactoryTest.java index db84862..c906551 100644 --- a/solr/core/src/test/org/apache/solr/analysis/SnowballPorterFilterFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/analysis/SnowballPorterFilterFactoryTest.java @@ -85,7 +85,7 @@ public class SnowballPorterFilterFactoryTest extends BaseTokenStreamTestCase { */ public void testProtected() throws Exception { SnowballPorterFilterFactory factory = new SnowballPorterFilterFactory(); - ResourceLoader loader = new SolrResourceLoader(null, null); + ResourceLoader loader = new SolrResourceLoader("solr/collection1"); Map args = new HashMap(); args.put("protected", "protwords.txt"); args.put("language", "English"); diff --git a/solr/core/src/test/org/apache/solr/analysis/TestArabicFilters.java b/solr/core/src/test/org/apache/solr/analysis/TestArabicFilters.java index c0e45ec..9e06570 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestArabicFilters.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestArabicFilters.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -31,20 +31,6 @@ import org.apache.lucene.analysis.Tokenizer; * Simple tests to ensure the Arabic filter Factories are working. */ public class TestArabicFilters extends BaseTokenStreamTestCase { - /** - * Test ArabicLetterTokenizerFactory - * @deprecated (3.1) Remove in Lucene 5.0 - */ - @Deprecated - public void testTokenizer() throws Exception { - Reader reader = new StringReader("الذين مَلكت أيمانكم"); - ArabicLetterTokenizerFactory factory = new ArabicLetterTokenizerFactory(); - factory.setLuceneMatchVersion(TEST_VERSION_CURRENT); - Map args = Collections.emptyMap(); - factory.init(args); - Tokenizer stream = factory.create(reader); - assertTokenStreamContents(stream, new String[] {"الذين", "مَلكت", "أيمانكم"}); - } /** * Test ArabicNormalizationFilterFactory diff --git a/solr/core/src/test/org/apache/solr/analysis/TestBeiderMorseFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestBeiderMorseFilterFactory.java index fd85a61..eaff03d 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestBeiderMorseFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestBeiderMorseFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestBrazilianStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestBrazilianStemFilterFactory.java index 83f2145..0511265 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestBrazilianStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestBrazilianStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestBulgarianStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestBulgarianStemFilterFactory.java index 2222954..7be02eb 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestBulgarianStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestBulgarianStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestCJKBigramFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestCJKBigramFilterFactory.java index 0a988db..bf48876 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestCJKBigramFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestCJKBigramFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestCJKTokenizerFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestCJKTokenizerFactory.java deleted file mode 100644 index 9369b97..0000000 --- a/solr/core/src/test/org/apache/solr/analysis/TestCJKTokenizerFactory.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.apache.solr.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.Reader; -import java.io.StringReader; - -import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.TokenStream; - -/** - * Simple tests to ensure the CJK tokenizer factory is working. - * @deprecated - */ -@Deprecated -public class TestCJKTokenizerFactory extends BaseTokenStreamTestCase { - /** - * Ensure the tokenizer actually tokenizes CJK text correctly - */ - public void testTokenizer() throws Exception { - Reader reader = new StringReader("我是中国人"); - CJKTokenizerFactory factory = new CJKTokenizerFactory(); - TokenStream stream = factory.create(reader); - assertTokenStreamContents(stream, new String[] {"我是", "是中", "中国", "国人"}); - } -} diff --git a/solr/core/src/test/org/apache/solr/analysis/TestCJKWidthFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestCJKWidthFilterFactory.java index c6b1df5..0de6588 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestCJKWidthFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestCJKWidthFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestCapitalizationFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestCapitalizationFilterFactory.java index 810902b..55f3c9f 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestCapitalizationFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestCapitalizationFilterFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestChineseFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestChineseFilterFactory.java deleted file mode 100644 index fd5b227..0000000 --- a/solr/core/src/test/org/apache/solr/analysis/TestChineseFilterFactory.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.apache.solr.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.Reader; -import java.io.StringReader; - -import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.MockTokenizer; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.Tokenizer; - -/** - * Simple tests to ensure the Chinese filter factory is working. - */ -public class TestChineseFilterFactory extends BaseTokenStreamTestCase { - /** - * Ensure the filter actually normalizes text (numerics, stopwords) - */ - public void testFiltering() throws Exception { - Reader reader = new StringReader("this 1234 Is such a silly filter"); - Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); - ChineseFilterFactory factory = new ChineseFilterFactory(); - TokenStream stream = factory.create(tokenizer); - assertTokenStreamContents(stream, new String[] { "Is", "silly", "filter" }); - } -} diff --git a/solr/core/src/test/org/apache/solr/analysis/TestChineseTokenizerFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestChineseTokenizerFactory.java deleted file mode 100644 index fce6114..0000000 --- a/solr/core/src/test/org/apache/solr/analysis/TestChineseTokenizerFactory.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.apache.solr.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.Reader; -import java.io.StringReader; - -import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.TokenStream; - -/** - * Simple tests to ensure the Chinese tokenizer factory is working. - */ -public class TestChineseTokenizerFactory extends BaseTokenStreamTestCase { - /** - * Ensure the tokenizer actually tokenizes chinese text correctly - */ - public void testTokenizer() throws Exception { - Reader reader = new StringReader("我是中国人"); - ChineseTokenizerFactory factory = new ChineseTokenizerFactory(); - TokenStream stream = factory.create(reader); - assertTokenStreamContents(stream, new String[] {"我", "是", "中", "国", "人"}); - } -} diff --git a/solr/core/src/test/org/apache/solr/analysis/TestCollationKeyFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestCollationKeyFilterFactory.java deleted file mode 100644 index 494c9c7..0000000 --- a/solr/core/src/test/org/apache/solr/analysis/TestCollationKeyFilterFactory.java +++ /dev/null @@ -1,197 +0,0 @@ -package org.apache.solr.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.StringReader; -import java.text.Collator; -import java.text.RuleBasedCollator; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; - -import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.MockTokenizer; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.analysis.util.ResourceLoader; - -public class TestCollationKeyFilterFactory extends BaseTokenStreamTestCase { - - /* - * Turkish has some funny casing. - * This test shows how you can solve this kind of thing easily with collation. - * Instead of using LowerCaseFilter, use a turkish collator with primary strength. - * Then things will sort and match correctly. - */ - public void testBasicUsage() throws IOException { - String turkishUpperCase = "I WÄ°LL USE TURKÄ°SH CASING"; - String turkishLowerCase = "ı will use turkish casıng"; - CollationKeyFilterFactory factory = new CollationKeyFilterFactory(); - Map args = new HashMap(); - args.put("language", "tr"); - args.put("strength", "primary"); - factory.init(args); - factory.inform(new StringMockSolrResourceLoader("")); - TokenStream tsUpper = factory.create( - new MockTokenizer(new StringReader(turkishUpperCase), MockTokenizer.KEYWORD, false)); - TokenStream tsLower = factory.create( - new MockTokenizer(new StringReader(turkishLowerCase), MockTokenizer.KEYWORD, false)); - assertCollatesToSame(tsUpper, tsLower); - } - - /* - * Test usage of the decomposition option for unicode normalization. - */ - public void testNormalization() throws IOException { - String turkishUpperCase = "I W\u0049\u0307LL USE TURKÄ°SH CASING"; - String turkishLowerCase = "ı will use turkish casıng"; - CollationKeyFilterFactory factory = new CollationKeyFilterFactory(); - Map args = new HashMap(); - args.put("language", "tr"); - args.put("strength", "primary"); - args.put("decomposition", "canonical"); - factory.init(args); - factory.inform(new StringMockSolrResourceLoader("")); - TokenStream tsUpper = factory.create( - new MockTokenizer(new StringReader(turkishUpperCase), MockTokenizer.KEYWORD, false)); - TokenStream tsLower = factory.create( - new MockTokenizer(new StringReader(turkishLowerCase), MockTokenizer.KEYWORD, false)); - assertCollatesToSame(tsUpper, tsLower); - } - - /* - * Test usage of the K decomposition option for unicode normalization. - * This works even with identical strength. - */ - public void testFullDecomposition() throws IOException { - String fullWidth = "Ｔｅｓｔｉｎｇ"; - String halfWidth = "Testing"; - CollationKeyFilterFactory factory = new CollationKeyFilterFactory(); - Map args = new HashMap(); - args.put("language", "zh"); - args.put("strength", "identical"); - args.put("decomposition", "full"); - factory.init(args); - factory.inform(new StringMockSolrResourceLoader("")); - TokenStream tsFull = factory.create( - new MockTokenizer(new StringReader(fullWidth), MockTokenizer.KEYWORD, false)); - TokenStream tsHalf = factory.create( - new MockTokenizer(new StringReader(halfWidth), MockTokenizer.KEYWORD, false)); - assertCollatesToSame(tsFull, tsHalf); - } - - /* - * Test secondary strength, for english case is not significant. - */ - public void testSecondaryStrength() throws IOException { - String upperCase = "TESTING"; - String lowerCase = "testing"; - CollationKeyFilterFactory factory = new CollationKeyFilterFactory(); - Map args = new HashMap(); - args.put("language", "en"); - args.put("strength", "secondary"); - args.put("decomposition", "no"); - factory.init(args); - factory.inform(new StringMockSolrResourceLoader("")); - TokenStream tsUpper = factory.create( - new MockTokenizer(new StringReader(upperCase), MockTokenizer.KEYWORD, false)); - TokenStream tsLower = factory.create( - new MockTokenizer(new StringReader(lowerCase), MockTokenizer.KEYWORD, false)); - assertCollatesToSame(tsUpper, tsLower); - } - - /* - * For german, you might want oe to sort and match with o umlaut. - * This is not the default, but you can make a customized ruleset to do this. - * - * The default is DIN 5007-1, this shows how to tailor a collator to get DIN 5007-2 behavior. - * http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4423383 - */ - public void testCustomRules() throws Exception { - RuleBasedCollator baseCollator = (RuleBasedCollator) Collator.getInstance(new Locale("de", "DE")); - - String DIN5007_2_tailorings = - "& ae , a\u0308 & AE , A\u0308"+ - "& oe , o\u0308 & OE , O\u0308"+ - "& ue , u\u0308 & UE , u\u0308"; - - RuleBasedCollator tailoredCollator = new RuleBasedCollator(baseCollator.getRules() + DIN5007_2_tailorings); - String tailoredRules = tailoredCollator.getRules(); - // - // at this point, you would save these tailoredRules to a file, - // and use the custom parameter. - // - String germanUmlaut = "Töne"; - String germanOE = "Toene"; - CollationKeyFilterFactory factory = new CollationKeyFilterFactory(); - Map args = new HashMap(); - args.put("custom", "rules.txt"); - args.put("strength", "primary"); - factory.init(args); - factory.inform(new StringMockSolrResourceLoader(tailoredRules)); - TokenStream tsUmlaut = factory.create( - new MockTokenizer(new StringReader(germanUmlaut), MockTokenizer.KEYWORD, false)); - TokenStream tsOE = factory.create( - new MockTokenizer(new StringReader(germanOE), MockTokenizer.KEYWORD, false)); - - assertCollatesToSame(tsUmlaut, tsOE); - } - - private class StringMockSolrResourceLoader implements ResourceLoader { - String text; - - StringMockSolrResourceLoader(String text) { - this.text = text; - } - - public List getLines(String resource) throws IOException { - return null; - } - - public T newInstance(String cname, Class expectedType, String... subpackages) { - return null; - } - - public InputStream openResource(String resource) throws IOException { - return new ByteArrayInputStream(text.getBytes("UTF-8")); - } - } - - private void assertCollatesToSame(TokenStream stream1, TokenStream stream2) - throws IOException { - stream1.reset(); - stream2.reset(); - CharTermAttribute term1 = stream1 - .addAttribute(CharTermAttribute.class); - CharTermAttribute term2 = stream2 - .addAttribute(CharTermAttribute.class); - assertTrue(stream1.incrementToken()); - assertTrue(stream2.incrementToken()); - assertEquals(term1.toString(), term2.toString()); - assertFalse(stream1.incrementToken()); - assertFalse(stream2.incrementToken()); - stream1.end(); - stream2.end(); - stream1.close(); - stream2.close(); - } -} diff --git a/solr/core/src/test/org/apache/solr/analysis/TestCollationKeyRangeQueries.java b/solr/core/src/test/org/apache/solr/analysis/TestCollationKeyRangeQueries.java deleted file mode 100644 index 09d0aa4..0000000 --- a/solr/core/src/test/org/apache/solr/analysis/TestCollationKeyRangeQueries.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.analysis; - -import org.apache.solr.SolrTestCaseJ4; -import org.junit.BeforeClass; - -/** - * Tests {@link CollationKeyFilterFactory} with RangeQueries - */ -public class TestCollationKeyRangeQueries extends SolrTestCaseJ4 { - - @BeforeClass - public static void beforeClass() throws Exception { - initCore("solrconfig-basic.xml","schema-collatefilter.xml"); - // add some docs - assertU(adoc("id", "1", "text", "\u0633\u0627\u0628")); - assertU(adoc("id", "2", "text", "I WÄ°LL USE TURKÄ°SH CASING")); - assertU(adoc("id", "3", "text", "ı will use turkish casıng")); - assertU(adoc("id", "4", "text", "Töne")); - assertU(adoc("id", "5", "text", "I W\u0049\u0307LL USE TURKÄ°SH CASING")); - assertU(adoc("id", "6", "text", "Ｔｅｓｔｉｎｇ")); - assertU(adoc("id", "7", "text", "Tone")); - assertU(adoc("id", "8", "text", "Testing")); - assertU(adoc("id", "9", "text", "testing")); - assertU(adoc("id", "10", "text", "toene")); - assertU(adoc("id", "11", "text", "Tzne")); - assertU(adoc("id", "12", "text", "\u0698\u0698")); - assertU(commit()); - } - - /** - * Test termquery with german DIN 5007-1 primary strength. - * In this case, ö is equivalent to o (but not oe) - */ - public void testBasicTermQuery() { - assertQ("Collated TQ: ", - req("fl", "id", "q", "sort_de:tone", "sort", "id asc" ), - "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=4]", - "//result/doc[2]/int[@name='id'][.=7]" - ); - } - - /** - * Test rangequery again with the DIN 5007-1 collator. - * We do a range query of tone .. tp, in binary order this - * would retrieve nothing due to case and accent differences. - */ - public void testBasicRangeQuery() { - assertQ("Collated RangeQ: ", - req("fl", "id", "q", "sort_de:[tone TO tp]", "sort", "id asc" ), - "//*[@numFound='2']", - "//result/doc[1]/int[@name='id'][.=4]", - "//result/doc[2]/int[@name='id'][.=7]" - ); - } - - /** - * Test rangequery again with an Arabic collator. - * Binary order would normally order U+0633 in this range. - */ - public void testNegativeRangeQuery() { - assertQ("Collated RangeQ: ", - req("fl", "id", "q", "sort_ar:[\u062F TO \u0698]", "sort", "id asc" ), - "//*[@numFound='0']" - ); - } -} diff --git a/solr/core/src/test/org/apache/solr/analysis/TestCzechStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestCzechStemFilterFactory.java index 7035187..2893d46 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestCzechStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestCzechStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestDelimitedPayloadTokenFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestDelimitedPayloadTokenFilterFactory.java index 18bb2bf..9644959 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestDelimitedPayloadTokenFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestDelimitedPayloadTokenFilterFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -38,7 +38,7 @@ public class TestDelimitedPayloadTokenFilterFactory extends BaseTokenStreamTestC args.put(DelimitedPayloadTokenFilterFactory.ENCODER_ATTR, "float"); DelimitedPayloadTokenFilterFactory factory = new DelimitedPayloadTokenFilterFactory(); factory.init(args); - ResourceLoader loader = new SolrResourceLoader(null, null); + ResourceLoader loader = new SolrResourceLoader("solr/collection1"); factory.inform(loader); TokenStream input = new MockTokenizer(new StringReader("the|0.1 quick|0.1 red|0.1"), MockTokenizer.WHITESPACE, false); @@ -47,7 +47,7 @@ public class TestDelimitedPayloadTokenFilterFactory extends BaseTokenStreamTestC while (tf.incrementToken()){ PayloadAttribute payAttr = tf.getAttribute(PayloadAttribute.class); assertTrue("payAttr is null and it shouldn't be", payAttr != null); - byte[] payData = payAttr.getPayload().getData(); + byte[] payData = payAttr.getPayload().bytes; assertTrue("payData is null and it shouldn't be", payData != null); assertTrue("payData is null and it shouldn't be", payData != null); float payFloat = PayloadHelper.decodeFloat(payData); @@ -61,7 +61,7 @@ public class TestDelimitedPayloadTokenFilterFactory extends BaseTokenStreamTestC args.put(DelimitedPayloadTokenFilterFactory.DELIMITER_ATTR, "*"); DelimitedPayloadTokenFilterFactory factory = new DelimitedPayloadTokenFilterFactory(); factory.init(args); - ResourceLoader loader = new SolrResourceLoader(null, null); + ResourceLoader loader = new SolrResourceLoader("solr/collection1"); factory.inform(loader); TokenStream input = new MockTokenizer(new StringReader("the*0.1 quick*0.1 red*0.1"), MockTokenizer.WHITESPACE, false); @@ -70,7 +70,7 @@ public class TestDelimitedPayloadTokenFilterFactory extends BaseTokenStreamTestC while (tf.incrementToken()){ PayloadAttribute payAttr = tf.getAttribute(PayloadAttribute.class); assertTrue("payAttr is null and it shouldn't be", payAttr != null); - byte[] payData = payAttr.getPayload().getData(); + byte[] payData = payAttr.getPayload().bytes; assertTrue("payData is null and it shouldn't be", payData != null); float payFloat = PayloadHelper.decodeFloat(payData); assertTrue(payFloat + " does not equal: " + 0.1f, payFloat == 0.1f); diff --git a/solr/core/src/test/org/apache/solr/analysis/TestDictionaryCompoundWordTokenFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestDictionaryCompoundWordTokenFilterFactory.java index cc3c264..8b7832d 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestDictionaryCompoundWordTokenFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestDictionaryCompoundWordTokenFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -40,7 +40,7 @@ public class TestDictionaryCompoundWordTokenFilterFactory extends BaseTokenStrea Reader reader = new StringReader("I like to play softball"); Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); DictionaryCompoundWordTokenFilterFactory factory = new DictionaryCompoundWordTokenFilterFactory(); - ResourceLoader loader = new SolrResourceLoader(null, null); + ResourceLoader loader = new SolrResourceLoader("solr/collection1"); Map args = new HashMap(); args.put("dictionary", "compoundDictionary.txt"); factory.setLuceneMatchVersion(TEST_VERSION_CURRENT); diff --git a/solr/core/src/test/org/apache/solr/analysis/TestElisionFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestElisionFilterFactory.java index a9dc80f..95c061f 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestElisionFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestElisionFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -42,7 +42,7 @@ public class TestElisionFilterFactory extends BaseTokenStreamTestCase { Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); ElisionFilterFactory factory = new ElisionFilterFactory(); factory.setLuceneMatchVersion(TEST_VERSION_CURRENT); - ResourceLoader loader = new SolrResourceLoader(null, null); + ResourceLoader loader = new SolrResourceLoader("solr/collection1"); Map args = new HashMap(); args.put("articles", "frenchArticles.txt"); factory.init(args); @@ -61,7 +61,7 @@ public class TestElisionFilterFactory extends BaseTokenStreamTestCase { factory.setLuceneMatchVersion(TEST_VERSION_CURRENT); Map args = Collections.emptyMap(); factory.init(args); - ResourceLoader loader = new SolrResourceLoader(null, null); + ResourceLoader loader = new SolrResourceLoader("solr/collection1"); factory.inform(loader); TokenStream stream = factory.create(tokenizer); assertTokenStreamContents(stream, new String[] { "avion" }); @@ -75,7 +75,7 @@ public class TestElisionFilterFactory extends BaseTokenStreamTestCase { Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); ElisionFilterFactory factory = new ElisionFilterFactory(); factory.setLuceneMatchVersion(TEST_VERSION_CURRENT); - ResourceLoader loader = new SolrResourceLoader(null, null); + ResourceLoader loader = new SolrResourceLoader("solr/collection1"); Map args = new HashMap(); args.put("articles", "frenchArticles.txt"); args.put("ignoreCase", "true"); diff --git a/solr/core/src/test/org/apache/solr/analysis/TestEnglishMinimalStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestEnglishMinimalStemFilterFactory.java index 2903d95..806d036 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestEnglishMinimalStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestEnglishMinimalStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestFinnishLightStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestFinnishLightStemFilterFactory.java index fc2daec..c345f3f 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestFinnishLightStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestFinnishLightStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestFrenchLightStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestFrenchLightStemFilterFactory.java index 076222a..784bf13 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestFrenchLightStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestFrenchLightStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestFrenchMinimalStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestFrenchMinimalStemFilterFactory.java index 8f6a277..db869c2 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestFrenchMinimalStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestFrenchMinimalStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestGalicianMinimalStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestGalicianMinimalStemFilterFactory.java index 7c108cb..8cd0933 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestGalicianMinimalStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestGalicianMinimalStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestGalicianStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestGalicianStemFilterFactory.java index 1674c76..1eafa38 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestGalicianStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestGalicianStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestGermanLightStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestGermanLightStemFilterFactory.java index 5ebdb63..6524b36 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestGermanLightStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestGermanLightStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestGermanMinimalStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestGermanMinimalStemFilterFactory.java index 1c81b97..62f8a68 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestGermanMinimalStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestGermanMinimalStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestGermanNormalizationFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestGermanNormalizationFilterFactory.java index 644c097..ecfecc4 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestGermanNormalizationFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestGermanNormalizationFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestGermanStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestGermanStemFilterFactory.java index 104bcf6..39b1d78 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestGermanStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestGermanStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestGreekLowerCaseFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestGreekLowerCaseFilterFactory.java index 25d5aa5..b33a6eb 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestGreekLowerCaseFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestGreekLowerCaseFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestGreekStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestGreekStemFilterFactory.java index af3923f..82a7e38 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestGreekStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestGreekStemFilterFactory.java @@ -9,7 +9,7 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.el.GreekLowerCaseFilter; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestHTMLStripCharFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestHTMLStripCharFilterFactory.java index fd4f9b8..cfdf062 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestHTMLStripCharFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestHTMLStripCharFilterFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestHindiFilters.java b/solr/core/src/test/org/apache/solr/analysis/TestHindiFilters.java index 466f0d4..4c40a6f 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestHindiFilters.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestHindiFilters.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestHungarianLightStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestHungarianLightStemFilterFactory.java index 5ee5d25..ae79afc 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestHungarianLightStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestHungarianLightStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestHunspellStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestHunspellStemFilterFactory.java index 93e8e52..b5f33b4 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestHunspellStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestHunspellStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -39,7 +39,7 @@ public class TestHunspellStemFilterFactory extends BaseTokenStreamTestCase { args.put("affix", "hunspell-test.aff"); factory.setLuceneMatchVersion(TEST_VERSION_CURRENT); factory.init(args); - factory.inform(new SolrResourceLoader("solr")); + factory.inform(new SolrResourceLoader("solr/collection1")); Reader reader = new StringReader("abc"); TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false)); diff --git a/solr/core/src/test/org/apache/solr/analysis/TestHyphenationCompoundWordTokenFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestHyphenationCompoundWordTokenFilterFactory.java index d0b05f4..30a282c 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestHyphenationCompoundWordTokenFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestHyphenationCompoundWordTokenFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -40,7 +40,7 @@ public class TestHyphenationCompoundWordTokenFilterFactory extends BaseTokenStre Reader reader = new StringReader("min veninde som er lidt af en læsehest"); Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); HyphenationCompoundWordTokenFilterFactory factory = new HyphenationCompoundWordTokenFilterFactory(); - ResourceLoader loader = new SolrResourceLoader(null, null); + ResourceLoader loader = new SolrResourceLoader("solr/collection1"); Map args = new HashMap(); args.put("hyphenator", "da_UTF8.xml"); args.put("dictionary", "da_compoundDictionary.txt"); @@ -64,7 +64,7 @@ public class TestHyphenationCompoundWordTokenFilterFactory extends BaseTokenStre Reader reader = new StringReader("basketballkurv"); Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); HyphenationCompoundWordTokenFilterFactory factory = new HyphenationCompoundWordTokenFilterFactory(); - ResourceLoader loader = new SolrResourceLoader(null, null); + ResourceLoader loader = new SolrResourceLoader("solr/collection1"); Map args = new HashMap(); args.put("hyphenator", "da_UTF8.xml"); args.put("minSubwordSize", "2"); diff --git a/solr/core/src/test/org/apache/solr/analysis/TestIndonesianStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestIndonesianStemFilterFactory.java index cb19764..1233d6b 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestIndonesianStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestIndonesianStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestIrishLowerCaseFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestIrishLowerCaseFilterFactory.java index 3c839de..e429d91 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestIrishLowerCaseFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestIrishLowerCaseFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestItalianLightStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestItalianLightStemFilterFactory.java index ba1f3b2..7787e0b 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestItalianLightStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestItalianLightStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestJapaneseBaseFormFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestJapaneseBaseFormFilterFactory.java index 67b5874..6bb5c26 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestJapaneseBaseFormFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestJapaneseBaseFormFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestJapaneseKatakanaStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestJapaneseKatakanaStemFilterFactory.java new file mode 100644 index 0000000..020f91f --- /dev/null +++ b/solr/core/src/test/org/apache/solr/analysis/TestJapaneseKatakanaStemFilterFactory.java @@ -0,0 +1,49 @@ +package org.apache.solr.analysis; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.analysis.BaseTokenStreamTestCase; +import org.apache.lucene.analysis.TokenStream; +import org.apache.solr.core.SolrResourceLoader; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Collections; +import java.util.Map; + +/** + * Simple tests for {@link JapaneseKatakanaStemFilterFactory} + */ +public class TestJapaneseKatakanaStemFilterFactory extends BaseTokenStreamTestCase { + public void testKatakanaStemming() throws IOException { + JapaneseTokenizerFactory tokenizerFactory = new JapaneseTokenizerFactory(); + Map tokenizerArgs = Collections.emptyMap(); + tokenizerFactory.init(tokenizerArgs); + tokenizerFactory.inform(new SolrResourceLoader(null, null)); + TokenStream tokenStream = tokenizerFactory.create( + new StringReader("明後日パーティーã«è¡Œã予定ãŒã‚る。図書館ã§è³‡æ–™ã‚’コピーã—ã¾ã—ãŸã€‚") + ); + JapaneseKatakanaStemFilterFactory filterFactory = new JapaneseKatakanaStemFilterFactory(); + Map filterArgs = Collections.emptyMap(); + filterFactory.init(filterArgs); + assertTokenStreamContents(filterFactory.create(tokenStream), + new String[]{ "明後日", "パーティ", "ã«", "è¡Œã", "予定", "ãŒ", "ã‚ã‚‹", // パーティー should be stemmed + "図書館", "ã§", "資料", "ã‚’", "コピー", "ã—", "ã¾ã—", "ãŸ"} // コピー should not be stemmed + ); + } +} diff --git a/solr/core/src/test/org/apache/solr/analysis/TestJapanesePartOfSpeechStopFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestJapanesePartOfSpeechStopFilterFactory.java index 48cd9e1..2338c8c 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestJapanesePartOfSpeechStopFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestJapanesePartOfSpeechStopFilterFactory.java @@ -10,7 +10,7 @@ import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.TokenStream; import org.apache.solr.core.SolrResourceLoader; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestJapaneseReadingFormFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestJapaneseReadingFormFilterFactory.java new file mode 100644 index 0000000..d46f9b1 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/analysis/TestJapaneseReadingFormFilterFactory.java @@ -0,0 +1,44 @@ +package org.apache.solr.analysis; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.analysis.BaseTokenStreamTestCase; +import org.apache.lucene.analysis.TokenStream; +import org.apache.solr.core.SolrResourceLoader; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Collections; +import java.util.Map; + +/** + * Simple tests for {@link JapaneseReadingFormFilterFactory} + */ +public class TestJapaneseReadingFormFilterFactory extends BaseTokenStreamTestCase { + public void testReadings() throws IOException { + JapaneseTokenizerFactory tokenizerFactory = new JapaneseTokenizerFactory(); + Map args = Collections.emptyMap(); + tokenizerFactory.init(args); + tokenizerFactory.inform(new SolrResourceLoader(null, null)); + TokenStream tokenStream = tokenizerFactory.create(new StringReader("å…ˆã»ã©ãƒ™ãƒ«ãƒªãƒ³ã‹ã‚‰æ¥ã¾ã—ãŸã€‚")); + JapaneseReadingFormFilterFactory filterFactory = new JapaneseReadingFormFilterFactory(); + assertTokenStreamContents(filterFactory.create(tokenStream), + new String[] { "サキ", "ホド", "ベルリン", "カラ", "ã‚­", "マシ", "ã‚¿" } + ); + } +} diff --git a/solr/core/src/test/org/apache/solr/analysis/TestJapaneseTokenizerFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestJapaneseTokenizerFactory.java index 44087ca..c4bb811 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestJapaneseTokenizerFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestJapaneseTokenizerFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestKStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestKStemFilterFactory.java index a4029ee..c94fc0f 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestKStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestKStemFilterFactory.java @@ -7,7 +7,7 @@ import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenStream; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestKeepFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestKeepFilterFactory.java index 1f4c79c..1c10041 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestKeepFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestKeepFilterFactory.java @@ -1,5 +1,5 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -31,7 +31,7 @@ import java.util.HashMap; public class TestKeepFilterFactory extends BaseTokenStreamTestCase { public void testInform() throws Exception { - ResourceLoader loader = new SolrResourceLoader(null, null); + ResourceLoader loader = new SolrResourceLoader("solr/collection1"); assertTrue("loader is null and it shouldn't be", loader != null); KeepWordFilterFactory factory = new KeepWordFilterFactory(); Map args = new HashMap(); diff --git a/solr/core/src/test/org/apache/solr/analysis/TestKeywordMarkerFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestKeywordMarkerFilterFactory.java index 73cc42f..7994f68 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestKeywordMarkerFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestKeywordMarkerFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -40,7 +40,7 @@ public class TestKeywordMarkerFilterFactory extends BaseTokenStreamTestCase { Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); KeywordMarkerFilterFactory factory = new KeywordMarkerFilterFactory(); Map args = new HashMap(); - ResourceLoader loader = new SolrResourceLoader(null, null); + ResourceLoader loader = new SolrResourceLoader("solr/collection1"); args.put("protected", "protwords.txt"); factory.setLuceneMatchVersion(TEST_VERSION_CURRENT); factory.init(args); @@ -55,7 +55,7 @@ public class TestKeywordMarkerFilterFactory extends BaseTokenStreamTestCase { Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); KeywordMarkerFilterFactory factory = new KeywordMarkerFilterFactory(); Map args = new HashMap(); - ResourceLoader loader = new SolrResourceLoader(null, null); + ResourceLoader loader = new SolrResourceLoader("solr/collection1"); args.put("protected", "protwords.txt"); args.put("ignoreCase", "true"); factory.setLuceneMatchVersion(TEST_VERSION_CURRENT); diff --git a/solr/core/src/test/org/apache/solr/analysis/TestLatvianStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestLatvianStemFilterFactory.java index 28b102b..4415cde 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestLatvianStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestLatvianStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestLuceneMatchVersion.java b/solr/core/src/test/org/apache/solr/analysis/TestLuceneMatchVersion.java index 0521f7f..b4f4271 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestLuceneMatchVersion.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestLuceneMatchVersion.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -52,10 +52,10 @@ public class TestLuceneMatchVersion extends SolrTestCaseJ4 { assertEquals(DEFAULT_VERSION, (ana.getTokenizerFactory()).getLuceneMatchVersion()); assertEquals(DEFAULT_VERSION, (ana.getTokenFilterFactories()[2]).getLuceneMatchVersion()); - type = schema.getFieldType("text30"); + type = schema.getFieldType("text40"); ana = (TokenizerChain) type.getAnalyzer(); - assertEquals(Version.LUCENE_30, (ana.getTokenizerFactory()).getLuceneMatchVersion()); - assertEquals(Version.LUCENE_31, (ana.getTokenFilterFactories()[2]).getLuceneMatchVersion()); + assertEquals(Version.LUCENE_40, (ana.getTokenizerFactory()).getLuceneMatchVersion()); + assertEquals(Version.LUCENE_50, (ana.getTokenFilterFactories()[2]).getLuceneMatchVersion()); // this is a hack to get the private matchVersion field in StandardAnalyzer's superclass, may break in later lucene versions - we have no getter :( final Field matchVersionField = StandardAnalyzer.class.getSuperclass().getDeclaredField("matchVersion"); @@ -66,9 +66,9 @@ public class TestLuceneMatchVersion extends SolrTestCaseJ4 { assertTrue(ana1 instanceof StandardAnalyzer); assertEquals(DEFAULT_VERSION, matchVersionField.get(ana1)); - type = schema.getFieldType("textStandardAnalyzer30"); + type = schema.getFieldType("textStandardAnalyzer40"); ana1 = type.getAnalyzer(); assertTrue(ana1 instanceof StandardAnalyzer); - assertEquals(Version.LUCENE_30, matchVersionField.get(ana1)); + assertEquals(Version.LUCENE_40, matchVersionField.get(ana1)); } } diff --git a/solr/core/src/test/org/apache/solr/analysis/TestMappingCharFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestMappingCharFilterFactory.java index 1a8fe6d..c9db4aa 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestMappingCharFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestMappingCharFilterFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestMultiWordSynonyms.java b/solr/core/src/test/org/apache/solr/analysis/TestMultiWordSynonyms.java index dcbb160..c9b4392 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestMultiWordSynonyms.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestMultiWordSynonyms.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -26,7 +26,6 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.StringReader; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -35,21 +34,6 @@ import java.util.Map; * @since solr 1.4 */ public class TestMultiWordSynonyms extends BaseTokenStreamTestCase { - - /** - * @deprecated Remove this test in 5.0 - */ - @Deprecated - public void testMultiWordSynonymsOld() throws IOException { - List rules = new ArrayList(); - rules.add("a b c,d"); - SlowSynonymMap synMap = new SlowSynonymMap(true); - SlowSynonymFilterFactory.parseRules(rules, synMap, "=>", ",", true, null); - - SlowSynonymFilter ts = new SlowSynonymFilter(new MockTokenizer(new StringReader("a e"), MockTokenizer.WHITESPACE, false), synMap); - // This fails because ["e","e"] is the value of the token stream - assertTokenStreamContents(ts, new String[] { "a", "e" }); - } public void testMultiWordSynonyms() throws IOException { SynonymFilterFactory factory = new SynonymFilterFactory(); diff --git a/solr/core/src/test/org/apache/solr/analysis/TestNGramFilters.java b/solr/core/src/test/org/apache/solr/analysis/TestNGramFilters.java index f1b0706..3790250 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestNGramFilters.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestNGramFilters.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestNorwegianLightStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestNorwegianLightStemFilterFactory.java index 751c13d..f8cbb53 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestNorwegianLightStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestNorwegianLightStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestNorwegianMinimalStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestNorwegianMinimalStemFilterFactory.java index c76b3cb..fb86cf6 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestNorwegianMinimalStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestNorwegianMinimalStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceCharFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceCharFilterFactory.java index 8161923..26ab438 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceCharFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceCharFilterFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceFilterFactory.java index 60a7ee1..0311043 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceFilterFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestPatternTokenizerFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestPatternTokenizerFactory.java index 5f490f2..afe15d7 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestPatternTokenizerFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestPatternTokenizerFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestPersianNormalizationFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestPersianNormalizationFilterFactory.java index 7f0dc86..a189d55 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestPersianNormalizationFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestPersianNormalizationFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestPhoneticFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestPhoneticFilterFactory.java index e20ae5f..dc7809d 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestPhoneticFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestPhoneticFilterFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -27,11 +27,13 @@ import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.util.LuceneTestCase.Slow; /** * */ +@Slow public class TestPhoneticFilterFactory extends BaseTokenStreamTestCase { private static final int REPEATS = 100000; diff --git a/solr/core/src/test/org/apache/solr/analysis/TestPorterStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestPorterStemFilterFactory.java index 43ae2ec..bb3be89 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestPorterStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestPorterStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestPortugueseLightStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestPortugueseLightStemFilterFactory.java index a2e3c37..aa5c009 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestPortugueseLightStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestPortugueseLightStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestPortugueseMinimalStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestPortugueseMinimalStemFilterFactory.java index 209b657..695b54c 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestPortugueseMinimalStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestPortugueseMinimalStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestPortugueseStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestPortugueseStemFilterFactory.java index 7b42175..13e9719 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestPortugueseStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestPortugueseStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestRemoveDuplicatesTokenFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestRemoveDuplicatesTokenFilterFactory.java index 58e4dda..d098b26 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestRemoveDuplicatesTokenFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestRemoveDuplicatesTokenFilterFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestReverseStringFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestReverseStringFilterFactory.java index 9064a5c..20c6d4d 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestReverseStringFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestReverseStringFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java index 195a203..60db59b 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java @@ -1,5 +1,5 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestRussianFilters.java b/solr/core/src/test/org/apache/solr/analysis/TestRussianFilters.java deleted file mode 100644 index 20be1e5..0000000 --- a/solr/core/src/test/org/apache/solr/analysis/TestRussianFilters.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.apache.solr.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.Reader; -import java.io.StringReader; -import java.util.Collections; -import java.util.Map; - -import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.Tokenizer; - -/** - * Simple tests to ensure the Russian filter factories are working. - */ -public class TestRussianFilters extends BaseTokenStreamTestCase { - /** - * Test RussianLetterTokenizerFactory - */ - public void testTokenizer() throws Exception { - Reader reader = new StringReader("ВмеÑте Ñ Ñ‚ÐµÐ¼ о Ñиле Ñлектромагнитной 100"); - RussianLetterTokenizerFactory factory = new RussianLetterTokenizerFactory(); - factory.setLuceneMatchVersion(TEST_VERSION_CURRENT); - Map args = Collections.emptyMap(); - factory.init(args); - Tokenizer stream = factory.create(reader); - assertTokenStreamContents(stream, new String[] {"ВмеÑте", "Ñ", "тем", "о", - "Ñиле", "Ñлектромагнитной", "100"}); - } -} diff --git a/solr/core/src/test/org/apache/solr/analysis/TestRussianLightStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestRussianLightStemFilterFactory.java index 15ead2b..810ffbd 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestRussianLightStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestRussianLightStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestShingleFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestShingleFilterFactory.java index d5712c9..ca85836 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestShingleFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestShingleFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestSlowSynonymFilter.java b/solr/core/src/test/org/apache/solr/analysis/TestSlowSynonymFilter.java deleted file mode 100644 index 22da7fa..0000000 --- a/solr/core/src/test/org/apache/solr/analysis/TestSlowSynonymFilter.java +++ /dev/null @@ -1,416 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.analysis; - -import java.io.IOException; -import java.io.StringReader; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; - -import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.MockTokenizer; -import org.apache.lucene.analysis.Token; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.tokenattributes.*; - -/** - * @deprecated Remove this test in Lucene 5.0 - */ -@Deprecated -public class TestSlowSynonymFilter extends BaseTokenStreamTestCase { - - static List strings(String str) { - String[] arr = str.split(" "); - return Arrays.asList(arr); - } - - static void assertTokenizesTo(SlowSynonymMap dict, String input, - String expected[]) throws IOException { - Tokenizer tokenizer = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false); - SlowSynonymFilter stream = new SlowSynonymFilter(tokenizer, dict); - assertTokenStreamContents(stream, expected); - } - - static void assertTokenizesTo(SlowSynonymMap dict, String input, - String expected[], int posIncs[]) throws IOException { - Tokenizer tokenizer = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false); - SlowSynonymFilter stream = new SlowSynonymFilter(tokenizer, dict); - assertTokenStreamContents(stream, expected, posIncs); - } - - static void assertTokenizesTo(SlowSynonymMap dict, List input, - String expected[], int posIncs[]) - throws IOException { - TokenStream tokenizer = new IterTokenStream(input); - SlowSynonymFilter stream = new SlowSynonymFilter(tokenizer, dict); - assertTokenStreamContents(stream, expected, posIncs); - } - - static void assertTokenizesTo(SlowSynonymMap dict, List input, - String expected[], int startOffsets[], int endOffsets[], int posIncs[]) - throws IOException { - TokenStream tokenizer = new IterTokenStream(input); - SlowSynonymFilter stream = new SlowSynonymFilter(tokenizer, dict); - assertTokenStreamContents(stream, expected, startOffsets, endOffsets, - posIncs); - } - - public void testMatching() throws IOException { - SlowSynonymMap map = new SlowSynonymMap(); - - boolean orig = false; - boolean merge = true; - map.add(strings("a b"), tokens("ab"), orig, merge); - map.add(strings("a c"), tokens("ac"), orig, merge); - map.add(strings("a"), tokens("aa"), orig, merge); - map.add(strings("b"), tokens("bb"), orig, merge); - map.add(strings("z x c v"), tokens("zxcv"), orig, merge); - map.add(strings("x c"), tokens("xc"), orig, merge); - - assertTokenizesTo(map, "$", new String[] { "$" }); - assertTokenizesTo(map, "a", new String[] { "aa" }); - assertTokenizesTo(map, "a $", new String[] { "aa", "$" }); - assertTokenizesTo(map, "$ a", new String[] { "$", "aa" }); - assertTokenizesTo(map, "a a", new String[] { "aa", "aa" }); - assertTokenizesTo(map, "b", new String[] { "bb" }); - assertTokenizesTo(map, "z x c v", new String[] { "zxcv" }); - assertTokenizesTo(map, "z x c $", new String[] { "z", "xc", "$" }); - - // repeats - map.add(strings("a b"), tokens("ab"), orig, merge); - map.add(strings("a b"), tokens("ab"), orig, merge); - - // FIXME: the below test intended to be { "ab" } - assertTokenizesTo(map, "a b", new String[] { "ab", "ab", "ab" }); - - // check for lack of recursion - map.add(strings("zoo"), tokens("zoo"), orig, merge); - assertTokenizesTo(map, "zoo zoo $ zoo", new String[] { "zoo", "zoo", "$", "zoo" }); - map.add(strings("zoo"), tokens("zoo zoo"), orig, merge); - // FIXME: the below test intended to be { "zoo", "zoo", "zoo", "zoo", "$", "zoo", "zoo" } - // maybe this was just a typo in the old test???? - assertTokenizesTo(map, "zoo zoo $ zoo", new String[] { "zoo", "zoo", "zoo", "zoo", "zoo", "zoo", "$", "zoo", "zoo", "zoo" }); - } - - public void testIncludeOrig() throws IOException { - SlowSynonymMap map = new SlowSynonymMap(); - - boolean orig = true; - boolean merge = true; - map.add(strings("a b"), tokens("ab"), orig, merge); - map.add(strings("a c"), tokens("ac"), orig, merge); - map.add(strings("a"), tokens("aa"), orig, merge); - map.add(strings("b"), tokens("bb"), orig, merge); - map.add(strings("z x c v"), tokens("zxcv"), orig, merge); - map.add(strings("x c"), tokens("xc"), orig, merge); - - assertTokenizesTo(map, "$", - new String[] { "$" }, - new int[] { 1 }); - assertTokenizesTo(map, "a", - new String[] { "a", "aa" }, - new int[] { 1, 0 }); - assertTokenizesTo(map, "a", - new String[] { "a", "aa" }, - new int[] { 1, 0 }); - assertTokenizesTo(map, "$ a", - new String[] { "$", "a", "aa" }, - new int[] { 1, 1, 0 }); - assertTokenizesTo(map, "a $", - new String[] { "a", "aa", "$" }, - new int[] { 1, 0, 1 }); - assertTokenizesTo(map, "$ a !", - new String[] { "$", "a", "aa", "!" }, - new int[] { 1, 1, 0, 1 }); - assertTokenizesTo(map, "a a", - new String[] { "a", "aa", "a", "aa" }, - new int[] { 1, 0, 1, 0 }); - assertTokenizesTo(map, "b", - new String[] { "b", "bb" }, - new int[] { 1, 0 }); - assertTokenizesTo(map, "z x c v", - new String[] { "z", "zxcv", "x", "c", "v" }, - new int[] { 1, 0, 1, 1, 1 }); - assertTokenizesTo(map, "z x c $", - new String[] { "z", "x", "xc", "c", "$" }, - new int[] { 1, 1, 0, 1, 1 }); - - // check for lack of recursion - map.add(strings("zoo zoo"), tokens("zoo"), orig, merge); - // CHECKME: I think the previous test (with 4 zoo's), was just a typo. - assertTokenizesTo(map, "zoo zoo $ zoo", - new String[] { "zoo", "zoo", "zoo", "$", "zoo" }, - new int[] { 1, 0, 1, 1, 1 }); - - map.add(strings("zoo"), tokens("zoo zoo"), orig, merge); - assertTokenizesTo(map, "zoo zoo $ zoo", - new String[] { "zoo", "zoo", "zoo", "$", "zoo", "zoo", "zoo" }, - new int[] { 1, 0, 1, 1, 1, 0, 1 }); - } - - - public void testMapMerge() throws IOException { - SlowSynonymMap map = new SlowSynonymMap(); - - boolean orig = false; - boolean merge = true; - map.add(strings("a"), tokens("a5,5"), orig, merge); - map.add(strings("a"), tokens("a3,3"), orig, merge); - - assertTokenizesTo(map, "a", - new String[] { "a3", "a5" }, - new int[] { 1, 2 }); - - map.add(strings("b"), tokens("b3,3"), orig, merge); - map.add(strings("b"), tokens("b5,5"), orig, merge); - - assertTokenizesTo(map, "b", - new String[] { "b3", "b5" }, - new int[] { 1, 2 }); - - map.add(strings("a"), tokens("A3,3"), orig, merge); - map.add(strings("a"), tokens("A5,5"), orig, merge); - - assertTokenizesTo(map, "a", - new String[] { "a3", "A3", "a5", "A5" }, - new int[] { 1, 0, 2, 0 }); - - map.add(strings("a"), tokens("a1"), orig, merge); - assertTokenizesTo(map, "a", - new String[] { "a1", "a3", "A3", "a5", "A5" }, - new int[] { 1, 2, 0, 2, 0 }); - - map.add(strings("a"), tokens("a2,2"), orig, merge); - map.add(strings("a"), tokens("a4,4 a6,2"), orig, merge); - assertTokenizesTo(map, "a", - new String[] { "a1", "a2", "a3", "A3", "a4", "a5", "A5", "a6" }, - new int[] { 1, 1, 1, 0, 1, 1, 0, 1 }); - } - - - public void testOverlap() throws IOException { - SlowSynonymMap map = new SlowSynonymMap(); - - boolean orig = false; - boolean merge = true; - map.add(strings("qwe"), tokens("qq/ww/ee"), orig, merge); - map.add(strings("qwe"), tokens("xx"), orig, merge); - map.add(strings("qwe"), tokens("yy"), orig, merge); - map.add(strings("qwe"), tokens("zz"), orig, merge); - assertTokenizesTo(map, "$", new String[] { "$" }); - assertTokenizesTo(map, "qwe", - new String[] { "qq", "ww", "ee", "xx", "yy", "zz" }, - new int[] { 1, 0, 0, 0, 0, 0 }); - - // test merging within the map - - map.add(strings("a"), tokens("a5,5 a8,3 a10,2"), orig, merge); - map.add(strings("a"), tokens("a3,3 a7,4 a9,2 a11,2 a111,100"), orig, merge); - assertTokenizesTo(map, "a", - new String[] { "a3", "a5", "a7", "a8", "a9", "a10", "a11", "a111" }, - new int[] { 1, 2, 2, 1, 1, 1, 1, 100 }); - } - - public void testPositionIncrements() throws IOException { - SlowSynonymMap map = new SlowSynonymMap(); - - boolean orig = false; - boolean merge = true; - - // test that generated tokens start at the same posInc as the original - map.add(strings("a"), tokens("aa"), orig, merge); - assertTokenizesTo(map, tokens("a,5"), - new String[] { "aa" }, - new int[] { 5 }); - assertTokenizesTo(map, tokens("b,1 a,0"), - new String[] { "b", "aa" }, - new int[] { 1, 0 }); - - // test that offset of first replacement is ignored (always takes the orig offset) - map.add(strings("b"), tokens("bb,100"), orig, merge); - assertTokenizesTo(map, tokens("b,5"), - new String[] { "bb" }, - new int[] { 5 }); - assertTokenizesTo(map, tokens("c,1 b,0"), - new String[] { "c", "bb" }, - new int[] { 1, 0 }); - - // test that subsequent tokens are adjusted accordingly - map.add(strings("c"), tokens("cc,100 c2,2"), orig, merge); - assertTokenizesTo(map, tokens("c,5"), - new String[] { "cc", "c2" }, - new int[] { 5, 2 }); - assertTokenizesTo(map, tokens("d,1 c,0"), - new String[] { "d", "cc", "c2" }, - new int[] { 1, 0, 2 }); - } - - - public void testPositionIncrementsWithOrig() throws IOException { - SlowSynonymMap map = new SlowSynonymMap(); - - boolean orig = true; - boolean merge = true; - - // test that generated tokens start at the same offset as the original - map.add(strings("a"), tokens("aa"), orig, merge); - assertTokenizesTo(map, tokens("a,5"), - new String[] { "a", "aa" }, - new int[] { 5, 0 }); - assertTokenizesTo(map, tokens("b,1 a,0"), - new String[] { "b", "a", "aa" }, - new int[] { 1, 0, 0 }); - - // test that offset of first replacement is ignored (always takes the orig offset) - map.add(strings("b"), tokens("bb,100"), orig, merge); - assertTokenizesTo(map, tokens("b,5"), - new String[] { "b", "bb" }, - new int[] { 5, 0 }); - assertTokenizesTo(map, tokens("c,1 b,0"), - new String[] { "c", "b", "bb" }, - new int[] { 1, 0, 0 }); - - // test that subsequent tokens are adjusted accordingly - map.add(strings("c"), tokens("cc,100 c2,2"), orig, merge); - assertTokenizesTo(map, tokens("c,5"), - new String[] { "c", "cc", "c2" }, - new int[] { 5, 0, 2 }); - assertTokenizesTo(map, tokens("d,1 c,0"), - new String[] { "d", "c", "cc", "c2" }, - new int[] { 1, 0, 0, 2 }); - } - - - public void testOffsetBug() throws IOException { - // With the following rules: - // a a=>b - // x=>y - // analysing "a x" causes "y" to have a bad offset (end less than start) - // SOLR-167 - SlowSynonymMap map = new SlowSynonymMap(); - - boolean orig = false; - boolean merge = true; - - map.add(strings("a a"), tokens("b"), orig, merge); - map.add(strings("x"), tokens("y"), orig, merge); - - // "a a x" => "b y" - assertTokenizesTo(map, tokens("a,1,0,1 a,1,2,3 x,1,4,5"), - new String[] { "b", "y" }, - new int[] { 0, 4 }, - new int[] { 3, 5 }, - new int[] { 1, 1 }); - } - - - /*** - * Return a list of tokens according to a test string format: - * a b c => returns List [a,b,c] - * a/b => tokens a and b share the same spot (b.positionIncrement=0) - * a,3/b/c => a,b,c all share same position (a.positionIncrement=3, b.positionIncrement=0, c.positionIncrement=0) - * a,1,10,11 => "a" with positionIncrement=1, startOffset=10, endOffset=11 - * @deprecated (3.0) does not support attributes api - */ - @Deprecated - private List tokens(String str) { - String[] arr = str.split(" "); - List result = new ArrayList(); - for (int i=0; i 1) { - posInc = Integer.parseInt(params[1]); - } else { - posInc = 1; - } - - if (params.length > 2) { - start = Integer.parseInt(params[2]); - } else { - start = 0; - } - - if (params.length > 3) { - end = Integer.parseInt(params[3]); - } else { - end = start + params[0].length(); - } - - Token t = new Token(params[0],start,end,"TEST"); - t.setPositionIncrement(posInc); - - result.add(t); - for (int j=1; j tokens) { - this(tokens.toArray(new Token[tokens.size()])); - } - - @Override - public boolean incrementToken() throws IOException { - if (index >= tokens.length) - return false; - else { - clearAttributes(); - Token token = tokens[index++]; - termAtt.setEmpty().append(token); - offsetAtt.setOffset(token.startOffset(), token.endOffset()); - posIncAtt.setPositionIncrement(token.getPositionIncrement()); - flagsAtt.setFlags(token.getFlags()); - typeAtt.setType(token.type()); - payloadAtt.setPayload(token.getPayload()); - return true; - } - } - } -} diff --git a/solr/core/src/test/org/apache/solr/analysis/TestSpanishLightStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestSpanishLightStemFilterFactory.java index 23c8a55..deba80b 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestSpanishLightStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestSpanishLightStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestStandardFactories.java b/solr/core/src/test/org/apache/solr/analysis/TestStandardFactories.java index ef7605e..cd3dcb0 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestStandardFactories.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestStandardFactories.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestStemmerOverrideFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestStemmerOverrideFilterFactory.java index d36d1af..74de226 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestStemmerOverrideFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestStemmerOverrideFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -41,7 +41,7 @@ public class TestStemmerOverrideFilterFactory extends BaseTokenStreamTestCase { Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); StemmerOverrideFilterFactory factory = new StemmerOverrideFilterFactory(); Map args = new HashMap(); - ResourceLoader loader = new SolrResourceLoader(null, null); + ResourceLoader loader = new SolrResourceLoader("solr/collection1"); args.put("dictionary", "stemdict.txt"); factory.setLuceneMatchVersion(TEST_VERSION_CURRENT); factory.init(args); @@ -56,7 +56,7 @@ public class TestStemmerOverrideFilterFactory extends BaseTokenStreamTestCase { Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); StemmerOverrideFilterFactory factory = new StemmerOverrideFilterFactory(); Map args = new HashMap(); - ResourceLoader loader = new SolrResourceLoader(null, null); + ResourceLoader loader = new SolrResourceLoader("solr/collection1"); args.put("dictionary", "stemdict.txt"); args.put("ignoreCase", "true"); factory.setLuceneMatchVersion(TEST_VERSION_CURRENT); diff --git a/solr/core/src/test/org/apache/solr/analysis/TestStopFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestStopFilterFactory.java index 27bd10c..4dc1f9e 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestStopFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestStopFilterFactory.java @@ -1,5 +1,5 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -32,7 +32,7 @@ import java.util.HashMap; public class TestStopFilterFactory extends BaseTokenStreamTestCase { public void testInform() throws Exception { - ResourceLoader loader = new SolrResourceLoader(null, null); + ResourceLoader loader = new SolrResourceLoader("solr/collection1"); assertTrue("loader is null and it shouldn't be", loader != null); StopFilterFactory factory = new StopFilterFactory(); Map args = new HashMap(); diff --git a/solr/core/src/test/org/apache/solr/analysis/TestSwedishLightStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestSwedishLightStemFilterFactory.java index 873b621..b059fc1 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestSwedishLightStemFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestSwedishLightStemFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestSynonymFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestSynonymFilterFactory.java index 7b24b67..d1521ac 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestSynonymFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestSynonymFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -30,7 +30,6 @@ import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.synonym.SynonymFilter; -import org.apache.lucene.util.Version; import org.apache.lucene.analysis.util.ResourceLoader; import org.apache.solr.core.SolrResourceLoader; @@ -42,7 +41,7 @@ public class TestSynonymFilterFactory extends BaseTokenStreamTestCase { args.put("synonyms", "synonyms.txt"); factory.setLuceneMatchVersion(TEST_VERSION_CURRENT); factory.init(args); - factory.inform(new SolrResourceLoader(null, null)); + factory.inform(new SolrResourceLoader("solr/collection1")); TokenStream ts = factory.create(new MockTokenizer(new StringReader("GB"), MockTokenizer.WHITESPACE, false)); assertTrue(ts instanceof SynonymFilter); assertTokenStreamContents(ts, @@ -50,42 +49,6 @@ public class TestSynonymFilterFactory extends BaseTokenStreamTestCase { new int[] { 1, 0, 0, 0 }); } - /** test that we can parse and use the solr syn file, with the old impl - * @deprecated Remove this test in Lucene 5.0 */ - @Deprecated - public void testSynonymsOld() throws Exception { - SynonymFilterFactory factory = new SynonymFilterFactory(); - Map args = new HashMap(); - args.put("synonyms", "synonyms.txt"); - factory.setLuceneMatchVersion(Version.LUCENE_33); - factory.init(args); - factory.inform(new SolrResourceLoader(null, null)); - TokenStream ts = factory.create(new MockTokenizer(new StringReader("GB"), MockTokenizer.WHITESPACE, false)); - assertTrue(ts instanceof SlowSynonymFilter); - assertTokenStreamContents(ts, - new String[] { "GB", "gib", "gigabyte", "gigabytes" }, - new int[] { 1, 0, 0, 0 }); - } - - /** test multiword offsets with the old impl - * @deprecated Remove this test in Lucene 5.0 */ - @Deprecated - public void testMultiwordOffsetsOld() throws Exception { - SynonymFilterFactory factory = new SynonymFilterFactory(); - Map args = new HashMap(); - args.put("synonyms", "synonyms.txt"); - factory.setLuceneMatchVersion(Version.LUCENE_33); - factory.init(args); - factory.inform(new StringMockSolrResourceLoader("national hockey league, nhl")); - TokenStream ts = factory.create(new MockTokenizer(new StringReader("national hockey league"), MockTokenizer.WHITESPACE, false)); - // WTF? - assertTokenStreamContents(ts, - new String[] { "national", "nhl", "hockey", "league" }, - new int[] { 0, 0, 0, 0 }, - new int[] { 22, 22, 22, 22 }, - new int[] { 1, 0, 1, 1 }); - } - /** if the synonyms are completely empty, test that we still analyze correctly */ public void testEmptySynonyms() throws Exception { SynonymFilterFactory factory = new SynonymFilterFactory(); diff --git a/solr/core/src/test/org/apache/solr/analysis/TestSynonymMap.java b/solr/core/src/test/org/apache/solr/analysis/TestSynonymMap.java deleted file mode 100644 index 2449b43..0000000 --- a/solr/core/src/test/org/apache/solr/analysis/TestSynonymMap.java +++ /dev/null @@ -1,319 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.analysis; - -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.lucene.analysis.Token; -import org.apache.lucene.analysis.util.InitializationException; -import org.apache.lucene.analysis.util.TokenizerFactory; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.analysis.util.ResourceLoader; - - -/** - * @deprecated Remove this test in Lucene 5.0 - */ -@Deprecated -public class TestSynonymMap extends LuceneTestCase { - - public void testInvalidMappingRules() throws Exception { - SlowSynonymMap synMap = new SlowSynonymMap( true ); - List rules = new ArrayList( 1 ); - rules.add( "a=>b=>c" ); - try{ - SlowSynonymFilterFactory.parseRules( rules, synMap, "=>", ",", true, null); - fail( "InitializationException must be thrown." ); - } - catch(InitializationException expected) {} - } - - public void testReadMappingRules() throws Exception { - SlowSynonymMap synMap; - - // (a)->[b] - List rules = new ArrayList(); - rules.add( "a=>b" ); - synMap = new SlowSynonymMap( true ); - SlowSynonymFilterFactory.parseRules( rules, synMap, "=>", ",", true, null); - assertEquals( 1, synMap.submap.size() ); - assertTokIncludes( synMap, "a", "b" ); - - // (a)->[c] - // (b)->[c] - rules.clear(); - rules.add( "a,b=>c" ); - synMap = new SlowSynonymMap( true ); - SlowSynonymFilterFactory.parseRules( rules, synMap, "=>", ",", true, null); - assertEquals( 2, synMap.submap.size() ); - assertTokIncludes( synMap, "a", "c" ); - assertTokIncludes( synMap, "b", "c" ); - - // (a)->[b][c] - rules.clear(); - rules.add( "a=>b,c" ); - synMap = new SlowSynonymMap( true ); - SlowSynonymFilterFactory.parseRules( rules, synMap, "=>", ",", true, null); - assertEquals( 1, synMap.submap.size() ); - assertTokIncludes( synMap, "a", "b" ); - assertTokIncludes( synMap, "a", "c" ); - - // (a)->(b)->[a2] - // [a1] - rules.clear(); - rules.add( "a=>a1" ); - rules.add( "a b=>a2" ); - synMap = new SlowSynonymMap( true ); - SlowSynonymFilterFactory.parseRules( rules, synMap, "=>", ",", true, null); - assertEquals( 1, synMap.submap.size() ); - assertTokIncludes( synMap, "a", "a1" ); - assertEquals( 1, getSubSynonymMap( synMap, "a" ).submap.size() ); - assertTokIncludes( getSubSynonymMap( synMap, "a" ), "b", "a2" ); - - // (a)->(b)->[a2] - // (c)->[a3] - // [a1] - rules.clear(); - rules.add( "a=>a1" ); - rules.add( "a b=>a2" ); - rules.add( "a c=>a3" ); - synMap = new SlowSynonymMap( true ); - SlowSynonymFilterFactory.parseRules( rules, synMap, "=>", ",", true, null); - assertEquals( 1, synMap.submap.size() ); - assertTokIncludes( synMap, "a", "a1" ); - assertEquals( 2, getSubSynonymMap( synMap, "a" ).submap.size() ); - assertTokIncludes( getSubSynonymMap( synMap, "a" ), "b", "a2" ); - assertTokIncludes( getSubSynonymMap( synMap, "a" ), "c", "a3" ); - - // (a)->(b)->[a2] - // [a1] - // (b)->(c)->[b2] - // [b1] - rules.clear(); - rules.add( "a=>a1" ); - rules.add( "a b=>a2" ); - rules.add( "b=>b1" ); - rules.add( "b c=>b2" ); - synMap = new SlowSynonymMap( true ); - SlowSynonymFilterFactory.parseRules( rules, synMap, "=>", ",", true, null); - assertEquals( 2, synMap.submap.size() ); - assertTokIncludes( synMap, "a", "a1" ); - assertEquals( 1, getSubSynonymMap( synMap, "a" ).submap.size() ); - assertTokIncludes( getSubSynonymMap( synMap, "a" ), "b", "a2" ); - assertTokIncludes( synMap, "b", "b1" ); - assertEquals( 1, getSubSynonymMap( synMap, "b" ).submap.size() ); - assertTokIncludes( getSubSynonymMap( synMap, "b" ), "c", "b2" ); - } - - public void testRead1waySynonymRules() throws Exception { - SlowSynonymMap synMap; - - // (a)->[a] - // (b)->[a] - List rules = new ArrayList(); - rules.add( "a,b" ); - synMap = new SlowSynonymMap( true ); - SlowSynonymFilterFactory.parseRules( rules, synMap, "=>", ",", false, null); - assertEquals( 2, synMap.submap.size() ); - assertTokIncludes( synMap, "a", "a" ); - assertTokIncludes( synMap, "b", "a" ); - - // (a)->[a] - // (b)->[a] - // (c)->[a] - rules.clear(); - rules.add( "a,b,c" ); - synMap = new SlowSynonymMap( true ); - SlowSynonymFilterFactory.parseRules( rules, synMap, "=>", ",", false, null); - assertEquals( 3, synMap.submap.size() ); - assertTokIncludes( synMap, "a", "a" ); - assertTokIncludes( synMap, "b", "a" ); - assertTokIncludes( synMap, "c", "a" ); - - // (a)->[a] - // (b1)->(b2)->[a] - rules.clear(); - rules.add( "a,b1 b2" ); - synMap = new SlowSynonymMap( true ); - SlowSynonymFilterFactory.parseRules( rules, synMap, "=>", ",", false, null); - assertEquals( 2, synMap.submap.size() ); - assertTokIncludes( synMap, "a", "a" ); - assertEquals( 1, getSubSynonymMap( synMap, "b1" ).submap.size() ); - assertTokIncludes( getSubSynonymMap( synMap, "b1" ), "b2", "a" ); - - // (a1)->(a2)->[a1][a2] - // (b)->[a1][a2] - rules.clear(); - rules.add( "a1 a2,b" ); - synMap = new SlowSynonymMap( true ); - SlowSynonymFilterFactory.parseRules( rules, synMap, "=>", ",", false, null); - assertEquals( 2, synMap.submap.size() ); - assertEquals( 1, getSubSynonymMap( synMap, "a1" ).submap.size() ); - assertTokIncludes( getSubSynonymMap( synMap, "a1" ), "a2", "a1" ); - assertTokIncludes( getSubSynonymMap( synMap, "a1" ), "a2", "a2" ); - assertTokIncludes( synMap, "b", "a1" ); - assertTokIncludes( synMap, "b", "a2" ); - } - - public void testRead2waySynonymRules() throws Exception { - SlowSynonymMap synMap; - - // (a)->[a][b] - // (b)->[a][b] - List rules = new ArrayList(); - rules.add( "a,b" ); - synMap = new SlowSynonymMap( true ); - SlowSynonymFilterFactory.parseRules( rules, synMap, "=>", ",", true, null); - assertEquals( 2, synMap.submap.size() ); - assertTokIncludes( synMap, "a", "a" ); - assertTokIncludes( synMap, "a", "b" ); - assertTokIncludes( synMap, "b", "a" ); - assertTokIncludes( synMap, "b", "b" ); - - // (a)->[a][b][c] - // (b)->[a][b][c] - // (c)->[a][b][c] - rules.clear(); - rules.add( "a,b,c" ); - synMap = new SlowSynonymMap( true ); - SlowSynonymFilterFactory.parseRules( rules, synMap, "=>", ",", true, null); - assertEquals( 3, synMap.submap.size() ); - assertTokIncludes( synMap, "a", "a" ); - assertTokIncludes( synMap, "a", "b" ); - assertTokIncludes( synMap, "a", "c" ); - assertTokIncludes( synMap, "b", "a" ); - assertTokIncludes( synMap, "b", "b" ); - assertTokIncludes( synMap, "b", "c" ); - assertTokIncludes( synMap, "c", "a" ); - assertTokIncludes( synMap, "c", "b" ); - assertTokIncludes( synMap, "c", "c" ); - - // (a)->[a] - // [b1][b2] - // (b1)->(b2)->[a] - // [b1][b2] - rules.clear(); - rules.add( "a,b1 b2" ); - synMap = new SlowSynonymMap( true ); - SlowSynonymFilterFactory.parseRules( rules, synMap, "=>", ",", true, null); - assertEquals( 2, synMap.submap.size() ); - assertTokIncludes( synMap, "a", "a" ); - assertTokIncludes( synMap, "a", "b1" ); - assertTokIncludes( synMap, "a", "b2" ); - assertEquals( 1, getSubSynonymMap( synMap, "b1" ).submap.size() ); - assertTokIncludes( getSubSynonymMap( synMap, "b1" ), "b2", "a" ); - assertTokIncludes( getSubSynonymMap( synMap, "b1" ), "b2", "b1" ); - assertTokIncludes( getSubSynonymMap( synMap, "b1" ), "b2", "b2" ); - - // (a1)->(a2)->[a1][a2] - // [b] - // (b)->[a1][a2] - // [b] - rules.clear(); - rules.add( "a1 a2,b" ); - synMap = new SlowSynonymMap( true ); - SlowSynonymFilterFactory.parseRules( rules, synMap, "=>", ",", true, null); - assertEquals( 2, synMap.submap.size() ); - assertEquals( 1, getSubSynonymMap( synMap, "a1" ).submap.size() ); - assertTokIncludes( getSubSynonymMap( synMap, "a1" ), "a2", "a1" ); - assertTokIncludes( getSubSynonymMap( synMap, "a1" ), "a2", "a2" ); - assertTokIncludes( getSubSynonymMap( synMap, "a1" ), "a2", "b" ); - assertTokIncludes( synMap, "b", "a1" ); - assertTokIncludes( synMap, "b", "a2" ); - assertTokIncludes( synMap, "b", "b" ); - } - - public void testBigramTokenizer() throws Exception { - SlowSynonymMap synMap; - - // prepare bi-gram tokenizer factory - TokenizerFactory tf = new NGramTokenizerFactory(); - Map args = new HashMap(); - args.put("minGramSize","2"); - args.put("maxGramSize","2"); - tf.init( args ); - - // (ab)->(bc)->(cd)->[ef][fg][gh] - List rules = new ArrayList(); - rules.add( "abcd=>efgh" ); - synMap = new SlowSynonymMap( true ); - SlowSynonymFilterFactory.parseRules( rules, synMap, "=>", ",", true, tf); - assertEquals( 1, synMap.submap.size() ); - assertEquals( 1, getSubSynonymMap( synMap, "ab" ).submap.size() ); - assertEquals( 1, getSubSynonymMap( getSubSynonymMap( synMap, "ab" ), "bc" ).submap.size() ); - assertTokIncludes( getSubSynonymMap( getSubSynonymMap( synMap, "ab" ), "bc" ), "cd", "ef" ); - assertTokIncludes( getSubSynonymMap( getSubSynonymMap( synMap, "ab" ), "bc" ), "cd", "fg" ); - assertTokIncludes( getSubSynonymMap( getSubSynonymMap( synMap, "ab" ), "bc" ), "cd", "gh" ); - } - - - public void testLoadRules() throws Exception { - Map args = new HashMap(); - args.put( "synonyms", "something.txt" ); - SlowSynonymFilterFactory ff = new SlowSynonymFilterFactory(); - ff.init(args); - ff.inform( new ResourceLoader() { - @Override - public List getLines(String resource) throws IOException { - if( !"something.txt".equals(resource) ) { - throw new RuntimeException( "should not get a differnt resource" ); - } - List rules = new ArrayList(); - rules.add( "a,b" ); - return rules; - } - - @Override - public T newInstance(String cname, Class expectedType, String... subpackages) { - throw new RuntimeException("stub"); - } - - @Override - public InputStream openResource(String resource) throws IOException { - throw new RuntimeException("stub"); - } - }); - - SlowSynonymMap synMap = ff.getSynonymMap(); - assertEquals( 2, synMap.submap.size() ); - assertTokIncludes( synMap, "a", "a" ); - assertTokIncludes( synMap, "a", "b" ); - assertTokIncludes( synMap, "b", "a" ); - assertTokIncludes( synMap, "b", "b" ); - } - - - private void assertTokIncludes( SlowSynonymMap map, String src, String exp ) throws Exception { - Token[] tokens = map.submap.get( src ).synonyms; - boolean inc = false; - for( Token token : tokens ){ - if( exp.equals( new String(token.buffer(), 0, token.length()) ) ) - inc = true; - } - assertTrue( inc ); - } - - private SlowSynonymMap getSubSynonymMap( SlowSynonymMap map, String src ){ - return map.submap.get( src ); - } -} diff --git a/solr/core/src/test/org/apache/solr/analysis/TestThaiWordFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestThaiWordFilterFactory.java index 532fd51..006d365 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestThaiWordFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestThaiWordFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestTrimFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestTrimFilterFactory.java index a2f3183..9376422 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestTrimFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestTrimFilterFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestTurkishLowerCaseFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestTurkishLowerCaseFilterFactory.java index 6ea51de..b3d9305 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestTurkishLowerCaseFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestTurkishLowerCaseFilterFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestTypeTokenFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestTypeTokenFilterFactory.java index cde4827..e8dee7c 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestTypeTokenFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestTypeTokenFilterFactory.java @@ -1,5 +1,5 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -35,7 +35,7 @@ public class TestTypeTokenFilterFactory extends BaseTokenStreamTestCase { @Test public void testInform() throws Exception { - ResourceLoader loader = new SolrResourceLoader(null, null); + ResourceLoader loader = new SolrResourceLoader("solr/collection1"); TypeTokenFilterFactory factory = new TypeTokenFilterFactory(); Map args = new HashMap(); args.put("types", "stoptypes-1.txt"); diff --git a/solr/core/src/test/org/apache/solr/analysis/TestUAX29URLEmailTokenizerFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestUAX29URLEmailTokenizerFactory.java index bd59fb2..d1a1c1c 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestUAX29URLEmailTokenizerFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestUAX29URLEmailTokenizerFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.analysis; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -190,25 +190,4 @@ public class TestUAX29URLEmailTokenizerFactory extends BaseTokenStreamTestCase { assertTokenStreamContents(stream, new String[] {"one", "two", "three", longWord, "four", "five", "six" }); } - - /** @deprecated nuke this test in lucene 5.0 */ - @Deprecated - public void testMatchVersion() throws Exception { - Reader reader = new StringReader("ã•ã‚™"); - UAX29URLEmailTokenizerFactory factory = new UAX29URLEmailTokenizerFactory(); - factory.setLuceneMatchVersion(TEST_VERSION_CURRENT); - Map args = Collections.emptyMap(); - factory.init(args); - Tokenizer stream = factory.create(reader); - assertTokenStreamContents(stream, - new String[] {"ã•ã‚™"}); - - reader = new StringReader("ã•ã‚™"); - factory = new UAX29URLEmailTokenizerFactory(); - factory.setLuceneMatchVersion(Version.LUCENE_31); - factory.init(args); - stream = factory.create(reader); - assertTokenStreamContents(stream, - new String[] {"ã•"}); // old broken behavior - } } diff --git a/solr/core/src/test/org/apache/solr/analysis/TestWikipediaTokenizerFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestWikipediaTokenizerFactory.java index a2248ed..7dda1e3 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestWikipediaTokenizerFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestWikipediaTokenizerFactory.java @@ -8,7 +8,7 @@ import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.wikipedia.WikipediaTokenizer; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java index f1339cc..ed4d782 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -197,7 +197,7 @@ public class TestWordDelimiterFilterFactory extends SolrTestCaseJ4 { public void testCustomTypes() throws Exception { String testText = "I borrowed $5,400.00 at 25% interest-rate"; WordDelimiterFilterFactory factoryDefault = new WordDelimiterFilterFactory(); - ResourceLoader loader = new SolrResourceLoader(null, null); + ResourceLoader loader = new SolrResourceLoader("solr/collection1"); Map args = new HashMap(); args.put("generateWordParts", "1"); args.put("generateNumberParts", "1"); diff --git a/solr/core/src/test/org/apache/solr/cloud/AbstractDistributedZkTestCase.java b/solr/core/src/test/org/apache/solr/cloud/AbstractDistributedZkTestCase.java index 8191d72..59ca99a 100644 --- a/solr/core/src/test/org/apache/solr/cloud/AbstractDistributedZkTestCase.java +++ b/solr/core/src/test/org/apache/solr/cloud/AbstractDistributedZkTestCase.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,7 +19,9 @@ package org.apache.solr.cloud; import java.io.File; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.commons.io.FileUtils; import org.apache.solr.BaseDistributedSearchTestCase; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.common.cloud.CloudState; @@ -37,6 +39,7 @@ public abstract class AbstractDistributedZkTestCase extends BaseDistributedSearc protected static final String DEFAULT_COLLECTION = "collection1"; private static final boolean DEBUG = false; protected ZkTestServer zkServer; + private AtomicInteger homeCount = new AtomicInteger(); @Before @Override @@ -64,15 +67,22 @@ public abstract class AbstractDistributedZkTestCase extends BaseDistributedSearc @Override protected void createServers(int numShards) throws Exception { + // give everyone there own solrhome + File controlHome = new File(new File(getSolrHome()).getParentFile(), "control" + homeCount.incrementAndGet()); + FileUtils.copyDirectory(new File(getSolrHome()), controlHome); + System.setProperty("collection", "control_collection"); - controlJetty = createJetty(testDir, testDir + "/control/data", "control_shard"); + controlJetty = createJetty(controlHome, null, "control_shard"); System.clearProperty("collection"); controlClient = createNewSolrServer(controlJetty.getLocalPort()); StringBuilder sb = new StringBuilder(); for (int i = 1; i <= numShards; i++) { if (sb.length() > 0) sb.append(','); - JettySolrRunner j = createJetty(testDir, testDir + "/jetty" + i, "shard" + (i + 2)); + // give everyone there own solrhome + File jettyHome = new File(new File(getSolrHome()).getParentFile(), "jetty" + homeCount.incrementAndGet()); + FileUtils.copyDirectory(new File(getSolrHome()), jettyHome); + JettySolrRunner j = createJetty(jettyHome, null, "shard" + (i + 2)); jettys.add(j); clients.add(createNewSolrServer(j.getLocalPort())); sb.append("localhost:").append(j.getLocalPort()).append(context); diff --git a/solr/core/src/test/org/apache/solr/cloud/AbstractZkTestCase.java b/solr/core/src/test/org/apache/solr/cloud/AbstractZkTestCase.java index 0850b4e..150441f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/AbstractZkTestCase.java +++ b/solr/core/src/test/org/apache/solr/cloud/AbstractZkTestCase.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -101,7 +101,7 @@ public abstract class AbstractZkTestCase extends SolrTestCaseJ4 { private static void putConfig(SolrZkClient zkClient, final String name) throws Exception { - zkClient.makePath("/configs/conf1/" + name, getFile("solr" + zkClient.makePath("/configs/conf1/" + name, getFile("solr" + File.separator + "collection1" + File.separator + "conf" + File.separator + name), false, true); } @@ -147,11 +147,7 @@ public abstract class AbstractZkTestCase extends SolrTestCaseJ4 { static void tryCleanPath(String zkHost, String path) throws Exception { SolrZkClient zkClient = new SolrZkClient(zkHost, TIMEOUT); if (zkClient.exists(path, true)) { - List children = zkClient.getChildren(path, null, true); - for (String string : children) { - tryCleanPath(zkHost, path+"/"+string); - } - zkClient.delete(path, -1, true); + zkClient.clean(path); } zkClient.close(); } diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java index bbfd0fd..6c36712 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,8 +23,10 @@ import java.net.MalformedURLException; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.CompletionService; @@ -34,9 +36,7 @@ import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.noggit.JSONUtil; -import org.apache.noggit.ObjectBuilder; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.JSONTestUtil; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrRequest; @@ -48,17 +48,18 @@ import org.apache.solr.client.solrj.request.AbstractUpdateRequest; import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest; import org.apache.solr.client.solrj.request.CoreAdminRequest.Create; import org.apache.solr.client.solrj.request.QueryRequest; -import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.cloud.CloudState; import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CollectionParams.CollectionAction; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.UpdateParams; -import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.update.SolrCmdDistributor.Request; @@ -67,7 +68,7 @@ import org.apache.solr.util.DefaultSolrThreadFactory; /** * */ -// @LuceneTestCase.AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-2161") +@Slow public class BasicDistributedZkTest extends AbstractDistributedZkTestCase { private static final String DEFAULT_COLLECTION = "collection1"; @@ -288,7 +289,8 @@ public class BasicDistributedZkTest extends AbstractDistributedZkTestCase { // on shards with matches. // query("q","matchesnothing","fl","*,score", "debugQuery", "true"); - + // would be better if these where all separate tests - but much, much + // slower doOptimisticLockingAndUpdating(); testMultipleCollections(); testANewCollectionInOneInstance(); @@ -298,6 +300,8 @@ public class BasicDistributedZkTest extends AbstractDistributedZkTestCase { testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-explicit"); testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-implicit"); + + testCollectionsAPI(); // Thread.sleep(10000000000L); if (DEBUG) { @@ -305,6 +309,136 @@ public class BasicDistributedZkTest extends AbstractDistributedZkTestCase { } } + private void testCollectionsAPI() throws Exception { + + // TODO: fragile - because we dont pass collection.confName, it will only + // find a default if a conf set with a name matching the collection name is found, or + // if there is only one conf set. That and the fact that other tests run first in this + // env make this pretty fragile + + // create 2 new collections rapid fire + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("action", CollectionAction.CREATE.toString()); + params.set("numReplicas", 1); + params.set("numShards", 3); + String collectionName = "awholynewcollection"; + params.set("name", collectionName); + SolrRequest request = new QueryRequest(params); + request.setPath("/admin/collections"); + + clients.get(0).request(request); + + String collectionName2 = "awholynewcollection2"; + params.set("name", collectionName2); + params.set("numShards", 2); + clients.get(1).request(request); + + checkForCollection(collectionName, 3); + checkForCollection(collectionName2, 2); + + // lets try and use the solrj client to index and retrieve a couple documents + SolrInputDocument doc = getDoc(id, 6, i1, -600, tlong, 600, t1, + "humpty dumpy sat on a wall"); + int which = (doc.getField(id).toString().hashCode() & 0x7fffffff) + % clients.size(); + SolrServer client = clients.get(which); + client.add(doc); + + doc = getDoc(id, 7, i1, 123, tlong, 123, t1, + "humpty dumpy had a great fall"); + which = (doc.getField(id).toString().hashCode() & 0x7fffffff) + % clients.size(); + client = clients.get(which); + client.add(doc); + doc = getDoc(id, 8, i1, 876, tlong, 876, t1, + "all the kings horses and all the kings men"); + which = (doc.getField(id).toString().hashCode() & 0x7fffffff) + % clients.size(); + client = clients.get(which); + client.add(doc); + + commit(); + + // remove a collection + params = new ModifiableSolrParams(); + params.set("action", CollectionAction.DELETE.toString()); + params.set("name", collectionName2); + request = new QueryRequest(params); + request.setPath("/admin/collections"); + + clients.get(0).request(request); + + // ensure its out of the state + checkForMissingCollection(collectionName2); + + printLayout(); + } + + private void checkForCollection(String collectionName, int expectedSlices) + throws Exception { + // check for an expectedSlices new collection - we poll the state + long timeoutAt = System.currentTimeMillis() + 30000; + boolean found = false; + while (System.currentTimeMillis() < timeoutAt) { + solrj.getZkStateReader().updateCloudState(true); + CloudState cloudState = solrj.getZkStateReader().getCloudState(); + Map> collections = cloudState + .getCollectionStates(); + if (collections.containsKey(collectionName)) { + Map slices = collections.get(collectionName); + // did we find expectedSlices slices/shards? + if (slices.size() == expectedSlices) { + found = true; + // also make sure each are active + Iterator> it = slices.entrySet().iterator(); + while (it.hasNext()) { + Entry sliceEntry = it.next(); + Map sliceShards = sliceEntry.getValue() + .getShards(); + Iterator> shardIt = sliceShards + .entrySet().iterator(); + while (shardIt.hasNext()) { + Entry shardEntry = shardIt.next(); + if (!shardEntry.getValue().get(ZkStateReader.STATE_PROP) + .equals(ZkStateReader.ACTIVE)) { + found = false; + break; + } + } + + } + if (found) break; + } + } + Thread.sleep(50); + } + if (!found) { + fail("Could not find new " + expectedSlices + " slice collection"); + } + } + + private void checkForMissingCollection(String collectionName) + throws Exception { + // check for a collection - we poll the state + long timeoutAt = System.currentTimeMillis() + 15000; + boolean found = true; + while (System.currentTimeMillis() < timeoutAt) { + solrj.getZkStateReader().updateCloudState(true); + CloudState cloudState = solrj.getZkStateReader().getCloudState(); + Map> collections = cloudState + .getCollectionStates(); + if (!collections.containsKey(collectionName)) { + found = false; + break; + } + Thread.sleep(50); + } + if (found) { + fail("Found collection that should be gone " + collectionName); + } + } + /** * Expects a RegexReplaceProcessorFactories in the chain which will * "double up" the values in two (stored) string fields. @@ -392,7 +526,7 @@ public class BasicDistributedZkTest extends AbstractDistributedZkTestCase { } private void testNumberOfCommitsWithCommitAfterAdd() - throws MalformedURLException, SolrServerException, IOException { + throws SolrServerException, IOException { long startCommits = getNumCommits((HttpSolrServer) clients.get(0)); ContentStreamUpdateRequest up = new ContentStreamUpdateRequest("/update"); @@ -406,7 +540,7 @@ public class BasicDistributedZkTest extends AbstractDistributedZkTestCase { assertEquals(startCommits + 1L, endCommits); } - private Long getNumCommits(HttpSolrServer solrServer) throws MalformedURLException, + private Long getNumCommits(HttpSolrServer solrServer) throws SolrServerException, IOException { HttpSolrServer server = new HttpSolrServer(solrServer.getBaseURL()); ModifiableSolrParams params = new ModifiableSolrParams(); @@ -450,7 +584,7 @@ public class BasicDistributedZkTest extends AbstractDistributedZkTestCase { assertAllActive(oneInstanceCollection2, solrj.getZkStateReader()); - // TODO: enable when we don't falsly get slice1... + // TODO: enable when we don't falsely get slice1... // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice1", 30000); // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice2", 30000); client2.add(getDoc(id, "1")); @@ -552,15 +686,13 @@ public class BasicDistributedZkTest extends AbstractDistributedZkTestCase { } private void createCollection(String collection, - List collectionClients, String baseUrl, int num) - throws MalformedURLException, SolrServerException, IOException, InterruptedException { + List collectionClients, String baseUrl, int num) { createCollection(collection, collectionClients, baseUrl, num, null); } private void createCollection(final String collection, List collectionClients, final String baseUrl, final int num, - final String shardId) throws MalformedURLException, SolrServerException, - IOException, InterruptedException { + final String shardId) { Callable call = new Callable() { public Object call() { HttpSolrServer server; @@ -594,8 +726,7 @@ public class BasicDistributedZkTest extends AbstractDistributedZkTestCase { collectionClients.add(createNewSolrServer(collection, baseUrl)); } - private void testMultipleCollections() throws MalformedURLException, - SolrServerException, IOException, Exception { + private void testMultipleCollections() throws Exception { // create another 2 collections and search across them createNewCollection("collection2"); createNewCollection("collection3"); @@ -665,8 +796,7 @@ public class BasicDistributedZkTest extends AbstractDistributedZkTestCase { client.add(doc); } - private void createNewCollection(final String collection) - throws MalformedURLException, SolrServerException, IOException, InterruptedException { + private void createNewCollection(final String collection) throws InterruptedException { final List collectionClients = new ArrayList(); otherCollectionClients.put(collection, collectionClients); int unique = 0; @@ -752,7 +882,7 @@ public class BasicDistributedZkTest extends AbstractDistributedZkTestCase { public void tearDown() throws Exception { super.tearDown(); if (solrj != null) { - solrj.close(); + solrj.shutdown(); } System.clearProperty("zkHost"); } diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java index dd2db62..ea82db4 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,9 +19,11 @@ package org.apache.solr.cloud; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.LogMergePolicy; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; @@ -37,6 +39,7 @@ import org.xml.sax.SAXParseException; * detect if a node is trying to do an update to itself with http - it shouldn't * do that. */ +@Slow public class BasicZkTest extends AbstractZkTestCase { @BeforeClass @@ -158,6 +161,13 @@ public class BasicZkTest extends AbstractZkTestCase { } + // test stats call + NamedList stats = core.getStatistics(); + assertEquals("collection1", stats.get("coreName")); + assertEquals("collection1", stats.get("collection")); + assertEquals("shard1", stats.get("shard")); + assertTrue(stats.get("refCount") != null); + //zkController.getZkClient().printLayoutToStdOut(); } diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java index 2a90fd4..446e9d8 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java index 3a65bed..9e3ef14 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,22 +17,18 @@ package org.apache.solr.cloud; * limitations under the License. */ -import java.io.IOException; import java.net.ConnectException; -import java.net.MalformedURLException; -import java.net.URISyntaxException; import java.util.ArrayList; import java.util.List; import java.util.Set; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; +import org.apache.http.client.HttpClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrServer; +import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.common.SolrInputDocument; -import org.apache.zookeeper.KeeperException; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -43,11 +39,11 @@ import org.junit.Ignore; public class ChaosMonkeyNothingIsSafeTest extends FullSolrCloudTest { @BeforeClass - public static void beforeSuperClass() throws Exception { + public static void beforeSuperClass() { } @AfterClass - public static void afterSuperClass() throws Exception { + public static void afterSuperClass() { } @Before @@ -159,8 +155,7 @@ public class ChaosMonkeyNothingIsSafeTest extends FullSolrCloudTest { } } - private void waitForThingsToLevelOut() throws KeeperException, - InterruptedException, Exception, IOException, URISyntaxException { + private void waitForThingsToLevelOut() throws Exception { int cnt = 0; boolean retry = false; do { @@ -203,15 +198,14 @@ public class ChaosMonkeyNothingIsSafeTest extends FullSolrCloudTest { } class FullThrottleStopableIndexingThread extends StopableIndexingThread { - ThreadSafeClientConnManager cm = new ThreadSafeClientConnManager(); - private DefaultHttpClient httpClient = new DefaultHttpClient(cm) ; + private HttpClient httpClient = HttpClientUtil.createClient(null); private volatile boolean stop = false; int clientIndex = 0; private ConcurrentUpdateSolrServer suss; private List clients; public FullThrottleStopableIndexingThread(List clients, - int startI, boolean doDeletes) throws MalformedURLException { + int startI, boolean doDeletes) { super(startI, doDeletes); setName("FullThrottleStopableIndexingThread"); setDaemon(true); @@ -283,25 +277,21 @@ public class ChaosMonkeyNothingIsSafeTest extends FullSolrCloudTest { if (clientIndex > clients.size() - 1) { clientIndex = 0; } - try { - suss.shutdownNow(); - suss = new ConcurrentUpdateSolrServer( - ((HttpSolrServer) clients.get(clientIndex)).getBaseURL(), - httpClient, 30, 3) { - public void handleError(Throwable ex) { - log.warn("suss error", ex); - } - }; - } catch (MalformedURLException e1) { - e1.printStackTrace(); - } + suss.shutdownNow(); + suss = new ConcurrentUpdateSolrServer( + ((HttpSolrServer) clients.get(clientIndex)).getBaseURL(), + httpClient, 30, 3) { + public void handleError(Throwable ex) { + log.warn("suss error", ex); + } + }; } } public void safeStop() { stop = true; suss.shutdownNow(); - cm.shutdown(); + httpClient.getConnectionManager().shutdown(); } public int getFails() { diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java index 29402a1..f112db6 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,15 +17,12 @@ package org.apache.solr.cloud; * limitations under the License. */ -import java.io.IOException; -import java.net.URISyntaxException; import java.util.ArrayList; import java.util.List; import java.util.Set; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.common.SolrInputDocument; -import org.apache.zookeeper.KeeperException; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -36,12 +33,12 @@ import org.junit.Ignore; public class ChaosMonkeySafeLeaderTest extends FullSolrCloudTest { @BeforeClass - public static void beforeSuperClass() throws Exception { + public static void beforeSuperClass() { } @AfterClass - public static void afterSuperClass() throws Exception { + public static void afterSuperClass() { } @@ -119,8 +116,7 @@ public class ChaosMonkeySafeLeaderTest extends FullSolrCloudTest { if (VERBOSE) System.out.println("control docs:" + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() + "\n\n"); } - private void waitForThingsToLevelOut() throws KeeperException, - InterruptedException, Exception, IOException, URISyntaxException { + private void waitForThingsToLevelOut() throws Exception { int cnt = 0; boolean retry = false; do { diff --git a/solr/core/src/test/org/apache/solr/cloud/CloudStateTest.java b/solr/core/src/test/org/apache/solr/cloud/CloudStateTest.java index 27492b5..ef413c3 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CloudStateTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CloudStateTest.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/solr/core/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java b/solr/core/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java index 57f1b51..02476c7 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.cloud.CloudState; import org.apache.solr.common.cloud.Slice; @@ -39,6 +40,7 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@Slow public class CloudStateUpdateTest extends SolrTestCaseJ4 { protected static Logger log = LoggerFactory .getLogger(AbstractZkTestCase.class); @@ -66,7 +68,7 @@ public class CloudStateUpdateTest extends SolrTestCaseJ4 { private Initializer init2; @BeforeClass - public static void beforeClass() throws Exception { + public static void beforeClass() { System.setProperty("solrcloud.skip.autorecovery", "true"); } diff --git a/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java b/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java index f401c0d..aec1c5a 100644 --- a/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,9 +18,8 @@ package org.apache.solr.cloud; */ import java.io.IOException; -import java.net.MalformedURLException; -import java.util.concurrent.TimeoutException; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; @@ -44,11 +43,12 @@ import org.junit.BeforeClass; /** * Super basic testing, no shard restarting or anything. */ +@Slow public class FullSolrCloudDistribCmdsTest extends FullSolrCloudTest { @BeforeClass - public static void beforeSuperClass() throws Exception { + public static void beforeSuperClass() { } public FullSolrCloudDistribCmdsTest() { @@ -131,8 +131,7 @@ public class FullSolrCloudDistribCmdsTest extends FullSolrCloudTest { testThatCantForwardToLeaderFails(); } - private void testThatCantForwardToLeaderFails() throws InterruptedException, - Exception, TimeoutException, IOException, KeeperException { + private void testThatCantForwardToLeaderFails() throws Exception { ZkNodeProps props = zkStateReader.getLeaderProps(DEFAULT_COLLECTION, "shard1"); chaosMonkey.stopShard("shard1"); @@ -164,8 +163,8 @@ public class FullSolrCloudDistribCmdsTest extends FullSolrCloudTest { assertTrue("A whole shard is down - some of these should fail", fails > 0); } - private long addTwoDocsInOneRequest(long docId) throws SolrServerException, - IOException, Exception { + private long addTwoDocsInOneRequest(long docId) throws + Exception { QueryResponse results; UpdateRequest uReq; uReq = new UpdateRequest(); @@ -192,7 +191,7 @@ public class FullSolrCloudDistribCmdsTest extends FullSolrCloudTest { return docId; } - private long addUpdateDelete() throws Exception, SolrServerException, + private long addUpdateDelete() throws Exception, IOException { long docId = 99999999L; indexr("id", docId, t1, "originalcontent"); @@ -230,13 +229,13 @@ public class FullSolrCloudDistribCmdsTest extends FullSolrCloudTest { return docId; } - private void testDeleteByQueryDistrib() throws Exception, SolrServerException { + private void testDeleteByQueryDistrib() throws Exception { del("*:*"); commit(); assertEquals(0, query(cloudClient).getResults().getNumFound()); } - private void testIndexingWithSuss() throws MalformedURLException, Exception { + private void testIndexingWithSuss() throws Exception { ConcurrentUpdateSolrServer suss = new ConcurrentUpdateSolrServer( ((HttpSolrServer) clients.get(0)).getBaseURL(), 3, 1); diff --git a/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java b/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java index ab249a8..74f05a1 100644 --- a/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,15 +17,20 @@ package org.apache.solr.cloud; * limitations under the License. */ +import java.io.File; import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; -import java.net.URISyntaxException; -import java.util.*; -import java.util.concurrent.TimeoutException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import org.apache.http.params.CoreConnectionPNames; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; @@ -46,12 +51,10 @@ import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.servlet.SolrDispatchFilter; -import org.apache.zookeeper.KeeperException; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Ignore; /** * @@ -59,9 +62,10 @@ import org.junit.Ignore; * what we test now - the default update chain * */ +@Slow public class FullSolrCloudTest extends AbstractDistributedZkTestCase { @BeforeClass - public static void beforeFullSolrCloudTest() throws Exception { + public static void beforeFullSolrCloudTest() { // shorten the log output more for this test type if (formatter != null) formatter.setShorterFormat(); } @@ -148,7 +152,7 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase { } @BeforeClass - public static void beforeClass() throws Exception { + public static void beforeClass() { System .setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory"); System.setProperty("solrcloud.update.delay", "0"); @@ -223,7 +227,7 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase { System.setProperty("collection", "control_collection"); String numShards = System.getProperty(ZkStateReader.NUM_SHARDS_PROP); System.clearProperty(ZkStateReader.NUM_SHARDS_PROP); - controlJetty = createJetty(testDir, testDir + "/control/data", + controlJetty = createJetty(new File(getSolrHome()), testDir + "/control/data", "control_shard"); System.clearProperty("collection"); if(numShards != null) { @@ -255,7 +259,7 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase { StringBuilder sb = new StringBuilder(); for (int i = 1; i <= numJettys; i++) { if (sb.length() > 0) sb.append(','); - JettySolrRunner j = createJetty(testDir, testDir + "/jetty" + JettySolrRunner j = createJetty(new File(getSolrHome()), testDir + "/jetty" + this.jettyIntCntr.incrementAndGet(), null, "solrconfig.xml", null); jettys.add(j); SolrServer client = createNewSolrServer(j.getLocalPort()); @@ -327,8 +331,7 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase { } protected void updateMappingsFromZk(List jettys, - List clients) throws Exception, IOException, KeeperException, - URISyntaxException { + List clients) throws Exception { zkStateReader.updateCloudState(true); shardToClient.clear(); shardToJetty.clear(); @@ -596,8 +599,7 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase { } - private long testUpdateAndDelete() throws Exception, SolrServerException, - IOException { + private long testUpdateAndDelete() throws Exception { long docId = 99999999L; indexr("id", docId, t1, "originalcontent"); @@ -632,9 +634,7 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase { return docId; } - private void addNewReplica() throws Exception, InterruptedException, - TimeoutException, IOException, KeeperException, URISyntaxException, - SolrServerException { + private void addNewReplica() throws Exception { JettySolrRunner newReplica = createJettys(1).get(0); waitForRecoveriesToFinish(false); @@ -656,8 +656,7 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase { super.waitForRecoveriesToFinish(DEFAULT_COLLECTION, zkStateReader, verbose); } - private void brindDownShardIndexSomeDocsAndRecover() throws Exception, - SolrServerException, IOException, InterruptedException { + private void brindDownShardIndexSomeDocsAndRecover() throws Exception { SolrQuery query = new SolrQuery("*:*"); query.set("distrib", false); @@ -697,7 +696,7 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase { // try to index to a living shard at shard2 - // we are careful to make sure the downed node is not longer in the state, + // we are careful to make sure the downed node is no longer in the state, // because on some systems (especially freebsd w/ blackhole enabled), trying // to talk to a downed node causes grief tries = 0; @@ -1320,7 +1319,7 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase { } ((HttpSolrServer) controlClient).shutdown(); if (cloudClient != null) { - cloudClient.close(); + cloudClient.shutdown(); } if (zkStateReader != null) { zkStateReader.close(); diff --git a/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java index 4d54dc3..887c8f2 100644 --- a/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -29,13 +29,13 @@ import java.util.regex.Pattern; import javax.xml.parsers.ParserConfigurationException; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreContainer.Initializer; -import org.apache.zookeeper.KeeperException; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -43,6 +43,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.xml.sax.SAXException; +@Slow public class LeaderElectionIntegrationTest extends SolrTestCaseJ4 { protected static Logger log = LoggerFactory .getLogger(AbstractZkTestCase.class); @@ -67,7 +68,7 @@ public class LeaderElectionIntegrationTest extends SolrTestCaseJ4 { private ZkStateReader reader; @BeforeClass - public static void beforeClass() throws Exception { + public static void beforeClass() { System.setProperty("solrcloud.skip.autorecovery", "true"); } @@ -245,7 +246,7 @@ public class LeaderElectionIntegrationTest extends SolrTestCaseJ4 { //Thread.sleep(100000); } - private String getLeader() throws InterruptedException, KeeperException { + private String getLeader() throws InterruptedException { ZkNodeProps props = reader.getLeaderProps("collection1", "shard1", 30000); String leader = props.get(ZkStateReader.NODE_NAME_PROP); diff --git a/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java index c935357..246a0f2 100644 --- a/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF @@ -28,6 +28,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.cloud.OnReconnect; import org.apache.solr.common.cloud.SolrZkClient; @@ -40,6 +41,7 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +@Slow public class LeaderElectionTest extends SolrTestCaseJ4 { static final int TIMEOUT = 30000; @@ -51,12 +53,12 @@ public class LeaderElectionTest extends SolrTestCaseJ4 { private volatile boolean stopStress = false; @BeforeClass - public static void beforeClass() throws Exception { + public static void beforeClass() { createTempDir(); } @AfterClass - public static void afterClass() throws InterruptedException { + public static void afterClass() { } diff --git a/solr/core/src/test/org/apache/solr/cloud/NodeStateWatcherTest.java b/solr/core/src/test/org/apache/solr/cloud/NodeStateWatcherTest.java deleted file mode 100644 index 79e58fb..0000000 --- a/solr/core/src/test/org/apache/solr/cloud/NodeStateWatcherTest.java +++ /dev/null @@ -1,124 +0,0 @@ -package org.apache.solr.cloud; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -import java.io.File; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.cloud.NodeStateWatcher.NodeStateChangeListener; -import org.apache.solr.cloud.OverseerTest.MockZKController; -import org.apache.solr.common.cloud.CoreState; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.zookeeper.KeeperException; -import org.junit.BeforeClass; -import java.util.Collection; - -public class NodeStateWatcherTest extends SolrTestCaseJ4 { - - private int TIMEOUT = 10000; - - @BeforeClass - public static void beforeClass() throws Exception { - initCore(); - } - - public void testCoreAddDelete() throws Exception { - String zkDir = dataDir.getAbsolutePath() + File.separator - + "zookeeper/server1/data"; - - ZkTestServer server = new ZkTestServer(zkDir); - - SolrZkClient zkClient = null; - ZkStateReader reader = null; - SolrZkClient overseerClient = null; - MockZKController controller = null; - - try { - final String NODE_NAME = "node1"; - server.run(); - zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT); - - AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost()); - AbstractZkTestCase.makeSolrZkNode(server.getZkHost()); - zkClient.makePath("/live_nodes", true); - - System.setProperty(ZkStateReader.NUM_SHARDS_PROP, "2"); - - reader = new ZkStateReader(zkClient); - reader.createClusterStateWatchersAndUpdate(); - - controller = new MockZKController(server.getZkAddress(), NODE_NAME, "collection1"); - - final String path = Overseer.STATES_NODE + "/" + NODE_NAME; - - final AtomicInteger callCounter = new AtomicInteger(); - NodeStateWatcher watcher = new NodeStateWatcher(zkClient, NODE_NAME, path, new NodeStateChangeListener() { - - @Override - public void coreChanged(String nodeName, Set states) - throws KeeperException, InterruptedException { - callCounter.incrementAndGet(); - } - - @Override - public void coreDeleted(String nodeName, Collection states) - throws KeeperException, InterruptedException { - callCounter.incrementAndGet(); - } - }); - - controller.publishState("core1", "state1", 2); - waitForCall(1, callCounter); - assertEquals(1, watcher.getCurrentState().size()); - controller.publishState("core2", "state1", 2); - waitForCall(2, callCounter); - assertEquals(2, watcher.getCurrentState().size()); - controller.publishState("core1", null, 2); - waitForCall(3, callCounter); - assertEquals(1, watcher.getCurrentState().size()); - controller.publishState("core2", null, 2); - waitForCall(4, callCounter); - assertEquals(0, watcher.getCurrentState().size()); - } finally { - System.clearProperty(ZkStateReader.NUM_SHARDS_PROP); - if (zkClient != null) { - zkClient.close(); - } - if (controller != null) { - controller.close(); - } - if (overseerClient != null) { - overseerClient.close(); - } - if (reader != null) { - reader.close(); - } - - } - - } - - private void waitForCall(int i, AtomicInteger callCounter) throws InterruptedException { - while (i > callCounter.get()) { - Thread.sleep(10); - } - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java index 2b4212e..2b460f7 100644 --- a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -32,23 +32,27 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; +import javax.xml.parsers.ParserConfigurationException; + +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.cloud.CloudState; -import org.apache.solr.common.cloud.CoreState; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.core.CoreDescriptor; +import org.apache.solr.handler.component.HttpShardHandlerFactory; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.KeeperException.Code; import org.apache.zookeeper.KeeperException.NodeExistsException; import org.apache.zookeeper.data.Stat; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +import org.xml.sax.SAXException; +@Slow public class OverseerTest extends SolrTestCaseJ4 { static final int TIMEOUT = 10000; @@ -62,7 +66,6 @@ public class OverseerTest extends SolrTestCaseJ4 { private final String nodeName; private final String collection; private final LeaderElector elector; - private final Map coreStates = Collections.synchronizedMap(new HashMap()); private final Map electionContext = Collections.synchronizedMap(new HashMap()); public MockZKController(String zkAddress, String nodeName, String collection) throws InterruptedException, TimeoutException, IOException, KeeperException { @@ -71,7 +74,6 @@ public class OverseerTest extends SolrTestCaseJ4 { zkClient = new SolrZkClient(zkAddress, TIMEOUT); zkStateReader = new ZkStateReader(zkClient); zkStateReader.createClusterStateWatchersAndUpdate(); - Overseer.createClientNodes(zkClient, nodeName); // live node final String nodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeName; @@ -104,29 +106,29 @@ public class OverseerTest extends SolrTestCaseJ4 { public void publishState(String coreName, String stateName, int numShards) throws KeeperException, InterruptedException, IOException { if (stateName == null) { - coreStates.remove(coreName); ElectionContext ec = electionContext.remove(coreName); if (ec != null) { ec.cancelElection(); } + ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, "deletecore", + ZkStateReader.NODE_NAME_PROP, nodeName, + ZkStateReader.CORE_NAME_PROP, coreName, + ZkStateReader.COLLECTION_PROP, collection); + DistributedQueue q = Overseer.getInQueue(zkClient); + q.offer(ZkStateReader.toJSON(m)); + } else { - HashMap coreProps = new HashMap(); - coreProps.put(ZkStateReader.STATE_PROP, stateName); - coreProps.put(ZkStateReader.NODE_NAME_PROP, nodeName); - coreProps.put(ZkStateReader.CORE_NAME_PROP, coreName); - coreProps.put(ZkStateReader.COLLECTION_PROP, collection); - coreProps.put(ZkStateReader.BASE_URL_PROP, "http://" + nodeName + ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, "state", + ZkStateReader.STATE_PROP, stateName, + ZkStateReader.NODE_NAME_PROP, nodeName, + ZkStateReader.CORE_NAME_PROP, coreName, + ZkStateReader.COLLECTION_PROP, collection, + ZkStateReader.NUM_SHARDS_PROP, Integer.toString(numShards), + ZkStateReader.BASE_URL_PROP, "http://" + nodeName + "/solr/"); - CoreState state = new CoreState(coreName, collection, coreProps, - numShards); - coreStates.remove(coreName); - coreStates.put(coreName, state); + DistributedQueue q = Overseer.getInQueue(zkClient); + q.offer(ZkStateReader.toJSON(m)); } - final String statePath = Overseer.STATES_NODE + "/" + nodeName; - zkClient.setData( - statePath, - ZkStateReader.toJSON(coreStates.values().toArray( - new CoreState[coreStates.size()])), true); for (int i = 0; i < 30; i++) { String shardId = getShardId(coreName); @@ -196,8 +198,6 @@ public class OverseerTest extends SolrTestCaseJ4 { ZkStateReader reader = new ZkStateReader(zkClient); reader.createClusterStateWatchersAndUpdate(); - System.setProperty(ZkStateReader.NUM_SHARDS_PROP, "3"); - zkController = new ZkController(null, server.getZkAddress(), TIMEOUT, 10000, "localhost", "8983", "solr", new CurrentCoreDescriptorProvider() { @@ -208,7 +208,7 @@ public class OverseerTest extends SolrTestCaseJ4 { } }); - System.setProperty("bootstrap_confdir", getFile("solr/conf") + System.setProperty("bootstrap_confdir", getFile("solr/collection1/conf") .getAbsolutePath()); final int numShards=6; @@ -216,6 +216,7 @@ public class OverseerTest extends SolrTestCaseJ4 { for (int i = 0; i < numShards; i++) { CloudDescriptor collection1Desc = new CloudDescriptor(); + collection1Desc.setNumShards(3); collection1Desc.setCollectionName("collection1"); CoreDescriptor desc1 = new CoreDescriptor(null, "core" + (i + 1), ""); desc1.setCloudDescriptor(collection1Desc); @@ -238,16 +239,13 @@ public class OverseerTest extends SolrTestCaseJ4 { assertNotNull(reader.getLeaderUrl("collection1", "shard3", 15000)); } finally { - System.clearProperty(ZkStateReader.NUM_SHARDS_PROP); System.clearProperty("bootstrap_confdir"); if (DEBUG) { if (zkController != null) { zkClient.printLayoutToStdOut(); } } - if (zkClient != null) { - zkClient.close(); - } + close(zkClient); if (zkController != null) { zkController.close(); } @@ -266,6 +264,7 @@ public class OverseerTest extends SolrTestCaseJ4 { ZkTestServer server = new ZkTestServer(zkDir); + System.setProperty(ZkStateReader.NUM_SHARDS_PROP, Integer.toString(sliceCount)); SolrZkClient zkClient = null; ZkStateReader reader = null; final ZkController[] controllers = new ZkController[nodeCount]; @@ -281,8 +280,6 @@ public class OverseerTest extends SolrTestCaseJ4 { reader = new ZkStateReader(zkClient); reader.createClusterStateWatchersAndUpdate(); - System.setProperty(ZkStateReader.NUM_SHARDS_PROP, Integer.valueOf(sliceCount).toString()); - for (int i = 0; i < nodeCount; i++) { controllers[i] = new ZkController(null, server.getZkAddress(), TIMEOUT, 10000, @@ -296,7 +293,7 @@ public class OverseerTest extends SolrTestCaseJ4 { }); } - System.setProperty("bootstrap_confdir", getFile("solr/conf") + System.setProperty("bootstrap_confdir", getFile("solr/collection1/conf") .getAbsolutePath()); @@ -313,6 +310,7 @@ public class OverseerTest extends SolrTestCaseJ4 { public void run() { final CloudDescriptor collection1Desc = new CloudDescriptor(); collection1Desc.setCollectionName("collection1"); + collection1Desc.setNumShards(sliceCount); final String coreName = "core" + slot; @@ -402,19 +400,17 @@ public class OverseerTest extends SolrTestCaseJ4 { zkClient.printLayoutToStdOut(); } } - if (zkClient != null) { - zkClient.close(); - } - if (reader != null) { - reader.close(); - } + close(zkClient); + close(reader); for (int i = 0; i < controllers.length; i++) if (controllers[i] != null) { controllers[i].close(); } server.shutdown(); for (int i = 0; i < nodeCount; i++) { - nodeExecutors[i].shutdownNow(); + if (nodeExecutors[i] != null) { + nodeExecutors[i].shutdownNow(); + } } } } @@ -457,36 +453,23 @@ public class OverseerTest extends SolrTestCaseJ4 { AbstractZkTestCase.makeSolrZkNode(server.getZkHost()); zkClient.makePath("/live_nodes", true); - //live node - String nodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + "node1"; - zkClient.makePath(nodePath,CreateMode.EPHEMERAL, true); - reader = new ZkStateReader(zkClient); reader.createClusterStateWatchersAndUpdate(); - Overseer.createClientNodes(zkClient, "node1"); - overseerClient = electNewOverseer(server.getZkAddress()); - HashMap coreProps = new HashMap(); - coreProps.put(ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr"); - coreProps.put(ZkStateReader.NODE_NAME_PROP, "node1"); - coreProps.put(ZkStateReader.CORE_NAME_PROP, "core1"); - coreProps.put(ZkStateReader.ROLES_PROP, ""); - coreProps.put(ZkStateReader.STATE_PROP, ZkStateReader.RECOVERING); - CoreState state = new CoreState("core1", "collection1", coreProps, 2); + DistributedQueue q = Overseer.getInQueue(zkClient); - nodePath = "/node_states/node1"; - - try { - zkClient.makePath(nodePath, CreateMode.EPHEMERAL, true); - } catch (KeeperException ke) { - if(ke.code()!=Code.NODEEXISTS) { - throw ke; - } - } + ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, "state", + ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr", + ZkStateReader.NODE_NAME_PROP, "node1", + ZkStateReader.COLLECTION_PROP, "collection1", + ZkStateReader.CORE_NAME_PROP, "core1", + ZkStateReader.ROLES_PROP, "", + ZkStateReader.STATE_PROP, ZkStateReader.RECOVERING); + + q.offer(ZkStateReader.toJSON(m)); - zkClient.setData(nodePath, ZkStateReader.toJSON(new CoreState[]{state}), true); waitForCollections(reader, "collection1"); assertEquals(reader.getCloudState().toString(), ZkStateReader.RECOVERING, @@ -494,27 +477,24 @@ public class OverseerTest extends SolrTestCaseJ4 { .get("node1_core1").get(ZkStateReader.STATE_PROP)); //publish node state (active) - coreProps.put(ZkStateReader.STATE_PROP, ZkStateReader.ACTIVE); - - coreProps.put(ZkStateReader.SHARD_ID_PROP, "shard1"); - state = new CoreState("core1", "collection1", coreProps, 2); + m = new ZkNodeProps(Overseer.QUEUE_OPERATION, "state", + ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr", + ZkStateReader.NODE_NAME_PROP, "node1", + ZkStateReader.COLLECTION_PROP, "collection1", + ZkStateReader.CORE_NAME_PROP, "core1", + ZkStateReader.ROLES_PROP, "", + ZkStateReader.STATE_PROP, ZkStateReader.ACTIVE); - zkClient.setData(nodePath, ZkStateReader.toJSON(new CoreState[]{state}), true); + q.offer(ZkStateReader.toJSON(m)); verifyStatus(reader, ZkStateReader.ACTIVE); } finally { - if (zkClient != null) { - zkClient.close(); - } - if (overseerClient != null) { - overseerClient.close(); - } + close(zkClient); + close(overseerClient); - if (reader != null) { - reader.close(); - } + close(reader); server.shutdown(); } } @@ -538,6 +518,7 @@ public class OverseerTest extends SolrTestCaseJ4 { private void verifyShardLeader(ZkStateReader reader, String collection, String shard, String expectedCore) throws InterruptedException, KeeperException { int maxIterations = 100; while(maxIterations-->0) { + reader.updateCloudState(true); // poll state ZkNodeProps props = reader.getCloudState().getLeader(collection, shard); if(props!=null) { if(expectedCore.equals(props.get(ZkStateReader.CORE_NAME_PROP))) { @@ -608,27 +589,20 @@ public class OverseerTest extends SolrTestCaseJ4 { version = getCloudStateVersion(controllerClient); mockController.publishState("core1", null,1); while(version == getCloudStateVersion(controllerClient)); - Thread.sleep(100); - assertEquals("Shard count does not match", 0, reader.getCloudState() - .getSlice("collection1", "shard1").getShards().size()); + Thread.sleep(500); + assertFalse("collection1 should be gone after publishing the null state", reader.getCloudState().getCollections().contains("collection1")); } finally { - if (mockController != null) { - mockController.close(); - } + close(mockController); - if (overseerClient != null) { - overseerClient.close(); - } - if (controllerClient != null) { - controllerClient.close(); - } - if (reader != null) { - reader.close(); - } + close(overseerClient); + close(controllerClient); + close(reader); server.shutdown(); } } + + private AtomicInteger killCounter = new AtomicInteger(); private class OverseerRestarter implements Runnable{ SolrZkClient overseerClient = null; @@ -643,33 +617,37 @@ public class OverseerTest extends SolrTestCaseJ4 { public void run() { try { overseerClient = electNewOverseer(zkAddress); - } catch (Throwable t) { - //t.printStackTrace(); - } - Random rnd = random(); - while (run) { - if(rnd.nextInt(20)==1){ + Random rnd = random(); + while (run) { + if (killCounter.get()>0) { + try { + killCounter.decrementAndGet(); + log.info("Killing overseer."); + overseerClient.close(); + overseerClient = electNewOverseer(zkAddress); + } catch (Throwable e) { + // e.printStackTrace(); + } + } try { - overseerClient.close(); - overseerClient = electNewOverseer(zkAddress); + Thread.sleep(100); } catch (Throwable e) { - //e.printStackTrace(); + // e.printStackTrace(); } } - try { - Thread.sleep(100); - } catch (Throwable e) { - //e.printStackTrace(); + } catch (Throwable t) { + // ignore + } finally { + if (overseerClient != null) { + try { + overseerClient.close(); + } catch (Throwable t) { + // ignore + } } } - try { - overseerClient.close(); - } catch (Throwable e) { - //e.printStackTrace(); - } } } - @Test public void testShardLeaderChange() throws Exception { @@ -693,10 +671,10 @@ public class OverseerTest extends SolrTestCaseJ4 { killerThread = new Thread(killer); killerThread.start(); - reader = new ZkStateReader(controllerClient); - reader.createClusterStateWatchersAndUpdate(); + reader = new ZkStateReader(controllerClient); //no watches, we'll poll for (int i = 0; i < atLeast(4); i++) { + killCounter.incrementAndGet(); //for each round allow 1 kill mockController = new MockZKController(server.getZkAddress(), "node1", "collection1"); mockController.publishState("core1", "state1",1); if(mockController2!=null) { @@ -719,18 +697,10 @@ public class OverseerTest extends SolrTestCaseJ4 { killerThread.join(); } } - if (mockController != null) { - mockController.close(); - } - if (mockController2 != null) { - mockController2.close(); - } - if (controllerClient != null) { - controllerClient.close(); - } - if (reader != null) { - reader.close(); - } + close(mockController); + close(mockController2); + close(controllerClient); + close(reader); server.shutdown(); } } @@ -791,19 +761,10 @@ public class OverseerTest extends SolrTestCaseJ4 { assertEquals("Shard was found in more than 1 times in CloudState", 1, numFound); } finally { - if (overseerClient != null) { - overseerClient.close(); - } - if (mockController != null) { - mockController.close(); - } - - if (controllerClient != null) { - controllerClient.close(); - } - if (reader != null) { - reader.close(); - } + close(overseerClient); + close(mockController); + close(controllerClient); + close(reader); server.shutdown(); } } @@ -842,23 +803,101 @@ public class OverseerTest extends SolrTestCaseJ4 { assertEquals("Slicecount does not match", 12, reader.getCloudState().getSlices("collection1").size()); } finally { - if (overseerClient != null) { - overseerClient.close(); - } - if (mockController != null) { - mockController.close(); - } + close(overseerClient); + close(mockController); + close(controllerClient); + close(reader); + server.shutdown(); + } + } - if (controllerClient != null) { - controllerClient.close(); - } - if (reader != null) { - reader.close(); - } + private void close(MockZKController mockController) { + if (mockController != null) { + mockController.close(); + } + } + + + @Test + public void testReplay() throws Exception{ + String zkDir = dataDir.getAbsolutePath() + File.separator + + "zookeeper/server1/data"; + ZkTestServer server = new ZkTestServer(zkDir); + SolrZkClient zkClient = null; + SolrZkClient overseerClient = null; + ZkStateReader reader = null; + + try { + server.run(); + zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT); + AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost()); + AbstractZkTestCase.makeSolrZkNode(server.getZkHost()); + zkClient.makePath(ZkStateReader.LIVE_NODES_ZKNODE, true); + + reader = new ZkStateReader(zkClient); + reader.createClusterStateWatchersAndUpdate(); + //prepopulate work queue with some items to emulate previous overseer died before persisting state + DistributedQueue queue = Overseer.getInternalQueue(zkClient); + ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, "state", + ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr", + ZkStateReader.NODE_NAME_PROP, "node1", + ZkStateReader.SHARD_ID_PROP, "s1", + ZkStateReader.COLLECTION_PROP, "collection1", + ZkStateReader.CORE_NAME_PROP, "core1", + ZkStateReader.ROLES_PROP, "", + ZkStateReader.STATE_PROP, ZkStateReader.RECOVERING); + queue.offer(ZkStateReader.toJSON(m)); + m = new ZkNodeProps(Overseer.QUEUE_OPERATION, "state", + ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr", + ZkStateReader.NODE_NAME_PROP, "node1", + ZkStateReader.SHARD_ID_PROP, "s1", + ZkStateReader.COLLECTION_PROP, "collection1", + ZkStateReader.CORE_NAME_PROP, "core2", + ZkStateReader.ROLES_PROP, "", + ZkStateReader.STATE_PROP, ZkStateReader.RECOVERING); + queue.offer(ZkStateReader.toJSON(m)); + + overseerClient = electNewOverseer(server.getZkAddress()); + + //submit to proper queue + queue = Overseer.getInQueue(zkClient); + m = new ZkNodeProps(Overseer.QUEUE_OPERATION, "state", + ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr", + ZkStateReader.NODE_NAME_PROP, "node1", + ZkStateReader.SHARD_ID_PROP, "s1", + ZkStateReader.COLLECTION_PROP, "collection1", + ZkStateReader.CORE_NAME_PROP, "core3", + ZkStateReader.ROLES_PROP, "", + ZkStateReader.STATE_PROP, ZkStateReader.RECOVERING); + queue.offer(ZkStateReader.toJSON(m)); + + for(int i=0;i<100;i++) { + Slice s = reader.getCloudState().getSlice("collection1", "s1"); + if(s!=null && s.getShards().size()==3) break; + Thread.sleep(100); + } + assertNotNull(reader.getCloudState().getSlice("collection1", "s1")); + assertEquals(3, reader.getCloudState().getSlice("collection1", "s1").getShards().size()); + } finally { + close(overseerClient); + close(zkClient); + close(reader); server.shutdown(); } } + private void close(ZkStateReader reader) { + if (reader != null) { + reader.close(); + } + } + + private void close(SolrZkClient overseerClient) throws InterruptedException { + if (overseerClient != null) { + overseerClient.close(); + } + } + private int getCloudStateVersion(SolrZkClient controllerClient) throws KeeperException, InterruptedException { return controllerClient.exists(ZkStateReader.CLUSTER_STATE, null, false).getVersion(); @@ -866,13 +905,14 @@ public class OverseerTest extends SolrTestCaseJ4 { private SolrZkClient electNewOverseer(String address) throws InterruptedException, - TimeoutException, IOException, KeeperException { + TimeoutException, IOException, KeeperException, ParserConfigurationException, SAXException { SolrZkClient zkClient = new SolrZkClient(address, TIMEOUT); ZkStateReader reader = new ZkStateReader(zkClient); LeaderElector overseerElector = new LeaderElector(zkClient); - ElectionContext ec = new OverseerElectionContext(address.replaceAll("/", "_"), zkClient, reader); + ElectionContext ec = new OverseerElectionContext(new HttpShardHandlerFactory().getShardHandler(), "/admin/cores", address.replaceAll("/", "_"), reader); overseerElector.setup(ec); overseerElector.joinElection(ec); return zkClient; } + } \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java b/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java index dd8a8d3..2962697 100644 --- a/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,6 +19,7 @@ package org.apache.solr.cloud; import java.io.IOException; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.embedded.JettySolrRunner; @@ -28,6 +29,7 @@ import org.junit.BeforeClass; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@Slow public class RecoveryZkTest extends FullSolrCloudTest { //private static final String DISTRIB_UPDATE_CHAIN = "distrib-update-chain"; @@ -35,12 +37,12 @@ public class RecoveryZkTest extends FullSolrCloudTest { private StopableIndexingThread indexThread; private StopableIndexingThread indexThread2; @BeforeClass - public static void beforeSuperClass() throws Exception { + public static void beforeSuperClass() { } @AfterClass - public static void afterSuperClass() throws Exception { + public static void afterSuperClass() { } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java b/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java index 6c8d06f..71a5e11 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/solr/core/src/test/org/apache/solr/cloud/TestMultiCoreConfBootstrap.java b/solr/core/src/test/org/apache/solr/cloud/TestMultiCoreConfBootstrap.java index aca579c..3d9ea8a 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestMultiCoreConfBootstrap.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestMultiCoreConfBootstrap.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,7 +18,6 @@ package org.apache.solr.cloud; import java.io.File; -import java.io.IOException; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.cloud.SolrZkClient; @@ -43,12 +42,12 @@ public class TestMultiCoreConfBootstrap extends SolrTestCaseJ4 { protected static String zkDir; @BeforeClass - public static void beforeClass() throws Exception { + public static void beforeClass() { createTempDir(); } @AfterClass - public static void afterClass() throws IOException { + public static void afterClass() { } diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java index 3920e3f..2129c0b 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF @@ -24,16 +24,19 @@ import java.util.Map; import junit.framework.Assert; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.core.CoreDescriptor; +import org.apache.solr.util.ExternalPaths; import org.apache.zookeeper.CreateMode; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +@Slow public class ZkControllerTest extends SolrTestCaseJ4 { private static final String COLLECTION_NAME = "collection1"; @@ -120,11 +123,11 @@ public class ZkControllerTest extends SolrTestCaseJ4 { } }); - zkController.uploadToZK(getFile("solr/conf"), + zkController.uploadToZK(new File(ExternalPaths.EXAMPLE_HOME + "/collection1/conf"), ZkController.CONFIGS_ZKNODE + "/config1"); // uploading again should overwrite, not error... - zkController.uploadToZK(getFile("solr/conf"), + zkController.uploadToZK(new File(ExternalPaths.EXAMPLE_HOME + "/collection1/conf"), ZkController.CONFIGS_ZKNODE + "/config1"); if (DEBUG) { @@ -179,7 +182,7 @@ public class ZkControllerTest extends SolrTestCaseJ4 { } }); - System.setProperty("bootstrap_confdir", getFile("solr/conf") + System.setProperty("bootstrap_confdir", getFile("solr/collection1/conf") .getAbsolutePath()); final int numShards = 2; @@ -220,7 +223,10 @@ public class ZkControllerTest extends SolrTestCaseJ4 { assertNotNull("New leader was null.", reader.getLeaderUrl("collection1", "shard1", 15000)); - Thread.sleep(2000); + for(int i=0;i<30;i++) { + if(zkController.getZkStateReader().getCloudState().getSlice("collection1", "shard1").getShards().size()==1) break; + Thread.sleep(500); + } assertEquals("shard was not unregistered", 1, zkController.getZkStateReader().getCloudState().getSlice("collection1", "shard1").getShards().size()); } finally { System.clearProperty("solrcloud.skip.autorecovery"); diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkNodePropsTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkNodePropsTest.java index 04c3ce1..e01e900 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkNodePropsTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkNodePropsTest.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java index 5d9309f..7a8f4de 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkTestServer.java b/solr/core/src/test/org/apache/solr/cloud/ZkTestServer.java index e857254..0e35e7a 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkTestServer.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkTestServer.java @@ -1,6 +1,6 @@ package org.apache.solr.cloud; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF diff --git a/solr/core/src/test/org/apache/solr/core/AlternateDirectoryTest.java b/solr/core/src/test/org/apache/solr/core/AlternateDirectoryTest.java index 0578d40..5cfa9bc 100755 --- a/solr/core/src/test/org/apache/solr/core/AlternateDirectoryTest.java +++ b/solr/core/src/test/org/apache/solr/core/AlternateDirectoryTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/core/DummyValueSourceParser.java b/solr/core/src/test/org/apache/solr/core/DummyValueSourceParser.java index 113d0fb..4351f0d 100644 --- a/solr/core/src/test/org/apache/solr/core/DummyValueSourceParser.java +++ b/solr/core/src/test/org/apache/solr/core/DummyValueSourceParser.java @@ -1,5 +1,5 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/core/FakeDeletionPolicy.java b/solr/core/src/test/org/apache/solr/core/FakeDeletionPolicy.java index 721b55f..0fa1bab 100644 --- a/solr/core/src/test/org/apache/solr/core/FakeDeletionPolicy.java +++ b/solr/core/src/test/org/apache/solr/core/FakeDeletionPolicy.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/core/IndexReaderFactoryTest.java b/solr/core/src/test/org/apache/solr/core/IndexReaderFactoryTest.java index 1fbec3f..c4daa5d 100644 --- a/solr/core/src/test/org/apache/solr/core/IndexReaderFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/core/IndexReaderFactoryTest.java @@ -1,5 +1,5 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/core/MockEventListener.java b/solr/core/src/test/org/apache/solr/core/MockEventListener.java index 639bee8..d480721 100644 --- a/solr/core/src/test/org/apache/solr/core/MockEventListener.java +++ b/solr/core/src/test/org/apache/solr/core/MockEventListener.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/core/MockQuerySenderListenerReqHandler.java b/solr/core/src/test/org/apache/solr/core/MockQuerySenderListenerReqHandler.java index 5561831..fe73a93 100644 --- a/solr/core/src/test/org/apache/solr/core/MockQuerySenderListenerReqHandler.java +++ b/solr/core/src/test/org/apache/solr/core/MockQuerySenderListenerReqHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/core/RAMDirectoryFactoryTest.java b/solr/core/src/test/org/apache/solr/core/RAMDirectoryFactoryTest.java index 7f59ee5..ed0e41d 100644 --- a/solr/core/src/test/org/apache/solr/core/RAMDirectoryFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/core/RAMDirectoryFactoryTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -37,7 +37,7 @@ public class RAMDirectoryFactoryTest extends LuceneTestCase { final Directory directory = new RAMDirectory(); RAMDirectoryFactory factory = new RAMDirectoryFactory() { @Override - protected Directory create(String path) throws IOException { + protected Directory create(String path) { return directory; } }; diff --git a/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java b/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java index d10440e..b9e9bf7 100755 --- a/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java +++ b/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java b/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java index 24819ab..8fafab4 100644 --- a/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java +++ b/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -95,7 +95,7 @@ public class ResourceLoaderTest extends LuceneTestCase public void testBOMMarkers() throws Exception { final String fileWithBom = "stopwithbom.txt"; - SolrResourceLoader loader = new SolrResourceLoader(null); + SolrResourceLoader loader = new SolrResourceLoader("solr/collection1"); // preliminary sanity check InputStream bomStream = loader.openResource(fileWithBom); @@ -121,7 +121,7 @@ public class ResourceLoaderTest extends LuceneTestCase public void testWrongEncoding() throws Exception { String wrongEncoding = "stopwordsWrongEncoding.txt"; - SolrResourceLoader loader = new SolrResourceLoader(null); + SolrResourceLoader loader = new SolrResourceLoader("solr/collection1"); // ensure we get our exception try { List lines = loader.getLines(wrongEncoding); diff --git a/solr/core/src/test/org/apache/solr/core/SOLR749Test.java b/solr/core/src/test/org/apache/solr/core/SOLR749Test.java index 38dcdd2..243e591 100644 --- a/solr/core/src/test/org/apache/solr/core/SOLR749Test.java +++ b/solr/core/src/test/org/apache/solr/core/SOLR749Test.java @@ -1,5 +1,5 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java b/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java index 9ac8394..47aada8 100755 --- a/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java +++ b/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -27,9 +27,12 @@ import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.util.plugin.SolrCoreAware; import org.junit.Test; +import java.io.File; import java.util.concurrent.*; import java.util.*; public class SolrCoreTest extends SolrTestCaseJ4 { + private static final String COLLECTION1 = "collection1"; + @Override public void setUp() throws Exception { super.setUp(); @@ -41,6 +44,37 @@ public class SolrCoreTest extends SolrTestCaseJ4 { deleteCore(); super.tearDown(); } + + @Test + public void testRemoveThenAddDefaultCore() throws Exception { + final CoreContainer cores = h.getCoreContainer(); + SolrCore core = cores.getCore(""); + assertEquals(COLLECTION1, cores.getDefaultCoreName()); + + cores.remove(""); + core.close(); + core.close(); + + + SolrCore newCore = new SolrCore(COLLECTION1, dataDir + File.separator + + "datadir2", new SolrConfig("solr/collection1", "solrconfig.xml", null), h.getCore().getSchema(), + new CoreDescriptor(cores, COLLECTION1, "solr/collection1")); + + cores.register(newCore, false); + + assertEquals(COLLECTION1, cores.getDefaultCoreName()); + + // so we should be able to get a core with collection1 + core = cores.getCore(COLLECTION1); + assertNotNull(core); + core.close(); + + // and with "" + core = cores.getCore(""); + assertNotNull(core); + + core.close(); + } @Test public void testRequestHandlerRegistry() { diff --git a/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java b/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java index aab5df0..2ca5c1b 100644 --- a/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java +++ b/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -49,12 +49,12 @@ public class TestArbitraryIndexDir extends AbstractSolrTestCase{ // TODO: fix this test to not require FSDirectory static String savedFactory; @BeforeClass - public static void beforeClass() throws Exception { + public static void beforeClass() { savedFactory = System.getProperty("solr.DirectoryFactory"); System.setProperty("solr.directoryFactory", "org.apache.solr.core.MockFSDirectoryFactory"); } @AfterClass - public static void afterClass() throws Exception { + public static void afterClass() { if (savedFactory == null) { System.clearProperty("solr.directoryFactory"); } else { @@ -71,7 +71,7 @@ public class TestArbitraryIndexDir extends AbstractSolrTestCase{ + System.getProperty("file.separator") + "data"); dataDir.mkdirs(); - solrConfig = TestHarness.createConfig("solrconfig.xml"); + solrConfig = TestHarness.createConfig(getSolrHome(), "solrconfig.xml"); h = new TestHarness( dataDir.getAbsolutePath(), solrConfig, "schema12.xml"); @@ -124,8 +124,8 @@ public class TestArbitraryIndexDir extends AbstractSolrTestCase{ new IndexWriterConfig(Version.LUCENE_40, new StandardAnalyzer(Version.LUCENE_40)) ); Document doc = new Document(); - doc.add(new Field("id", "2", TextField.TYPE_STORED)); - doc.add(new Field("name", "name2", TextField.TYPE_STORED)); + doc.add(new TextField("id", "2", Field.Store.YES)); + doc.add(new TextField("name", "name2", Field.Store.YES)); iw.addDocument(doc); iw.commit(); iw.close(); diff --git a/solr/core/src/test/org/apache/solr/core/TestBadConfig.java b/solr/core/src/test/org/apache/solr/core/TestBadConfig.java index d7ceb89..ba81242 100644 --- a/solr/core/src/test/org/apache/solr/core/TestBadConfig.java +++ b/solr/core/src/test/org/apache/solr/core/TestBadConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java b/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java index 75860bb..d3555fe 100644 --- a/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java +++ b/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java @@ -1,6 +1,6 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/core/TestConfig.java b/solr/core/src/test/org/apache/solr/core/TestConfig.java index d1556a6..84caf31 100644 --- a/solr/core/src/test/org/apache/solr/core/TestConfig.java +++ b/solr/core/src/test/org/apache/solr/core/TestConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -130,22 +130,13 @@ public class TestConfig extends SolrTestCaseJ4 { // If defaults change, add test methods to cover each version @Test public void testDefaults() throws Exception { - SolrConfig sc = new SolrConfig("solrconfig-basic.xml"); + SolrConfig sc = new SolrConfig(new SolrResourceLoader("solr/collection1"), "solrconfig-basic.xml", null); SolrIndexConfig sic = sc.indexConfig; assertTrue("default ramBufferSizeMB should be 32", sic.ramBufferSizeMB == 32); assertTrue("default useCompoundFile should be false", sic.useCompoundFile == false); assertTrue("default LockType should be native", sic.lockType.equals(SolrIndexConfig.LOCK_TYPE_NATIVE)); } - @Test - public void testDefaults31() throws Exception { - SolrConfig sc = new SolrConfig("solrconfig-basic-luceneVersion31.xml"); - SolrIndexConfig sic = sc.indexConfig; - assertTrue("default ramBufferSizeMB should be 16", sic.ramBufferSizeMB == 16); - assertTrue("default useCompoundFile should be true", sic.useCompoundFile == true); - assertTrue("default LockType should be simple", sic.lockType.equals(SolrIndexConfig.LOCK_TYPE_SIMPLE)); - } - } diff --git a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java index 94bf50c..9e06bc4 100644 --- a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java +++ b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,12 +17,16 @@ package org.apache.solr.core; +import java.io.BufferedWriter; import java.io.File; +import java.io.FileWriter; import java.io.IOException; +import javax.xml.parsers.ParserConfigurationException; import javax.xml.xpath.XPathExpressionException; import org.apache.commons.io.FileUtils; +import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.solr.SolrTestCaseJ4; import org.junit.BeforeClass; import org.junit.Test; @@ -158,4 +162,61 @@ public class TestCoreContainer extends SolrTestCaseJ4 { } } + public void testNoCores() throws IOException, ParserConfigurationException, SAXException { + //create solrHome + File solrHomeDirectory = new File(TEMP_DIR, this.getClass().getName() + + "_noCores"); + if (solrHomeDirectory.exists()) { + FileUtils.deleteDirectory(solrHomeDirectory); + } + assertTrue("Failed to mkdirs workDir", solrHomeDirectory.mkdirs()); + try { + File solrXmlFile = new File(solrHomeDirectory, "solr.xml"); + BufferedWriter out = new BufferedWriter(new FileWriter(solrXmlFile)); + out.write(EMPTY_SOLR_XML); + out.close(); + } catch (IOException e) { + FileUtils.deleteDirectory(solrHomeDirectory); + throw e; + } + + //init + System.setProperty("solr.solr.home", solrHomeDirectory.getAbsolutePath()); + CoreContainer.Initializer init = new CoreContainer.Initializer(); + CoreContainer cores = null; + try { + cores = init.initialize(); + } + catch(Exception e) { + fail("CoreContainer not created" + e.getMessage()); + } + try { + //assert cero cores + assertEquals("There should not be cores", 0, cores.getCores().size()); + + //add a new core + CoreDescriptor coreDescriptor = new CoreDescriptor(cores, "core1", SolrTestCaseJ4.TEST_HOME() + "/collection1"); + SolrCore newCore = cores.create(coreDescriptor); + cores.register(newCore, false); + + //assert one registered core + assertEquals("There core registered", 1, cores.getCores().size()); + + newCore.close(); + cores.remove("core1"); + //assert cero cores + assertEquals("There should not be cores", 0, cores.getCores().size()); + } finally { + cores.shutdown(); + FileUtils.deleteDirectory(solrHomeDirectory); + } + + } + + private static final String EMPTY_SOLR_XML ="\n" + + "\n" + + " \n" + + " \n" + + ""; + } diff --git a/solr/core/src/test/org/apache/solr/core/TestJmxIntegration.java b/solr/core/src/test/org/apache/solr/core/TestJmxIntegration.java index f12b1f7..31bc5fc 100644 --- a/solr/core/src/test/org/apache/solr/core/TestJmxIntegration.java +++ b/solr/core/src/test/org/apache/solr/core/TestJmxIntegration.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/core/TestJmxMonitoredMap.java b/solr/core/src/test/org/apache/solr/core/TestJmxMonitoredMap.java index b8dc63d..24c4666 100644 --- a/solr/core/src/test/org/apache/solr/core/TestJmxMonitoredMap.java +++ b/solr/core/src/test/org/apache/solr/core/TestJmxMonitoredMap.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java b/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java index 06b7f80..dabfd5f 100644 --- a/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java +++ b/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java @@ -1,6 +1,6 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/core/TestPropInject.java b/solr/core/src/test/org/apache/solr/core/TestPropInject.java index b5b9b8b..3450a34 100644 --- a/solr/core/src/test/org/apache/solr/core/TestPropInject.java +++ b/solr/core/src/test/org/apache/solr/core/TestPropInject.java @@ -1,6 +1,6 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/core/TestPropInjectDefaults.java b/solr/core/src/test/org/apache/solr/core/TestPropInjectDefaults.java index f121ec9..e0ce38f 100644 --- a/solr/core/src/test/org/apache/solr/core/TestPropInjectDefaults.java +++ b/solr/core/src/test/org/apache/solr/core/TestPropInjectDefaults.java @@ -1,6 +1,6 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/core/TestQuerySenderListener.java b/solr/core/src/test/org/apache/solr/core/TestQuerySenderListener.java index 59d7ec7..d6d5a24 100644 --- a/solr/core/src/test/org/apache/solr/core/TestQuerySenderListener.java +++ b/solr/core/src/test/org/apache/solr/core/TestQuerySenderListener.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/core/TestQuerySenderNoQuery.java b/solr/core/src/test/org/apache/solr/core/TestQuerySenderNoQuery.java index be73d89..94b91ab 100644 --- a/solr/core/src/test/org/apache/solr/core/TestQuerySenderNoQuery.java +++ b/solr/core/src/test/org/apache/solr/core/TestQuerySenderNoQuery.java @@ -1,6 +1,6 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/core/TestSolrDeletionPolicy1.java b/solr/core/src/test/org/apache/solr/core/TestSolrDeletionPolicy1.java index 53cec48..375a58a 100644 --- a/solr/core/src/test/org/apache/solr/core/TestSolrDeletionPolicy1.java +++ b/solr/core/src/test/org/apache/solr/core/TestSolrDeletionPolicy1.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/core/TestSolrDeletionPolicy2.java b/solr/core/src/test/org/apache/solr/core/TestSolrDeletionPolicy2.java index cadf084..99f3a5b 100644 --- a/solr/core/src/test/org/apache/solr/core/TestSolrDeletionPolicy2.java +++ b/solr/core/src/test/org/apache/solr/core/TestSolrDeletionPolicy2.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/core/TestSolrXMLSerializer.java b/solr/core/src/test/org/apache/solr/core/TestSolrXMLSerializer.java index d8b1877..5e61810 100644 --- a/solr/core/src/test/org/apache/solr/core/TestSolrXMLSerializer.java +++ b/solr/core/src/test/org/apache/solr/core/TestSolrXMLSerializer.java @@ -1,6 +1,6 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,7 +22,6 @@ import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.StringWriter; -import java.io.UnsupportedEncodingException; import java.io.Writer; import java.util.ArrayList; import java.util.HashMap; @@ -90,8 +89,7 @@ public class TestSolrXMLSerializer extends LuceneTestCase { } private void assertResults(byte[] bytes) - throws ParserConfigurationException, UnsupportedEncodingException, - IOException, SAXException, XPathExpressionException { + throws ParserConfigurationException, IOException, SAXException, XPathExpressionException { DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); BufferedInputStream is = new BufferedInputStream(new ByteArrayInputStream(bytes)); Document document; diff --git a/solr/core/src/test/org/apache/solr/core/TestXIncludeConfig.java b/solr/core/src/test/org/apache/solr/core/TestXIncludeConfig.java index bd15627..aaf659c 100644 --- a/solr/core/src/test/org/apache/solr/core/TestXIncludeConfig.java +++ b/solr/core/src/test/org/apache/solr/core/TestXIncludeConfig.java @@ -1,6 +1,6 @@ package org.apache.solr.core; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/AnalysisRequestHandlerTestBase.java b/solr/core/src/test/org/apache/solr/handler/AnalysisRequestHandlerTestBase.java index 5ad0787..071f971 100644 --- a/solr/core/src/test/org/apache/solr/handler/AnalysisRequestHandlerTestBase.java +++ b/solr/core/src/test/org/apache/solr/handler/AnalysisRequestHandlerTestBase.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/BinaryUpdateRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/BinaryUpdateRequestHandlerTest.java index 21d90b4..dfece60 100644 --- a/solr/core/src/test/org/apache/solr/handler/BinaryUpdateRequestHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/BinaryUpdateRequestHandlerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/CSVRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/CSVRequestHandlerTest.java index c7d860b..05d260d 100644 --- a/solr/core/src/test/org/apache/solr/handler/CSVRequestHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/CSVRequestHandlerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java index 5deb533..5164de0 100644 --- a/solr/core/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java index 527f518..aa7ae49 100644 --- a/solr/core/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java b/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java index 42cc9e4..916cf69 100644 --- a/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java +++ b/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -71,9 +71,9 @@ public class JsonLoaderTest extends SolrTestCaseJ4 { "'optimize': { 'waitSearcher':false, 'openSearcher':false },\n" + "\n" + "'delete': { 'id':'ID' },\n" + - "'delete': { 'id':'ID', 'commitWithin':'500' },\n" + + "'delete': { 'id':'ID', 'commitWithin':500 },\n" + "'delete': { 'query':'QUERY' },\n" + - "'delete': { 'query':'QUERY', 'commitWithin':'500' },\n" + + "'delete': { 'query':'QUERY', 'commitWithin':500 },\n" + "'rollback': {}\n" + "\n" + "}\n" + @@ -237,4 +237,66 @@ public class JsonLoaderTest extends SolrTestCaseJ4 { ); } + // The delete syntax was both extended for simplification in 4.0 + @Test + public void testDeleteSyntax() throws Exception { + String str = "{'delete':10" + +"\n ,'delete':'20'" + +"\n ,'delete':['30','40']" + +"\n ,'delete':{'id':50, '_version_':12345}" + +"\n ,'delete':[{'id':60, '_version_':67890}, {'id':70, '_version_':77777}, {'query':'id:80', '_version_':88888}]" + + "\n}\n"; + str = str.replace('\'', '"'); + SolrQueryRequest req = req(); + SolrQueryResponse rsp = new SolrQueryResponse(); + BufferingRequestProcessor p = new BufferingRequestProcessor(null); + JsonLoader loader = new JsonLoader(); + loader.load(req, rsp, new ContentStreamBase.StringStream(str), p); + + // DELETE COMMANDS + assertEquals( 8, p.deleteCommands.size() ); + DeleteUpdateCommand delete = p.deleteCommands.get( 0 ); + assertEquals( delete.id, "10" ); + assertEquals( delete.query, null ); + assertEquals( delete.commitWithin, -1); + + delete = p.deleteCommands.get( 1 ); + assertEquals( delete.id, "20" ); + assertEquals( delete.query, null ); + assertEquals( delete.commitWithin, -1); + + delete = p.deleteCommands.get( 2 ); + assertEquals( delete.id, "30" ); + assertEquals( delete.query, null ); + assertEquals( delete.commitWithin, -1); + + delete = p.deleteCommands.get( 3 ); + assertEquals( delete.id, "40" ); + assertEquals( delete.query, null ); + assertEquals( delete.commitWithin, -1); + + delete = p.deleteCommands.get( 4 ); + assertEquals( delete.id, "50" ); + assertEquals( delete.query, null ); + assertEquals( delete.getVersion(), 12345L); + + delete = p.deleteCommands.get( 5 ); + assertEquals( delete.id, "60" ); + assertEquals( delete.query, null ); + assertEquals( delete.getVersion(), 67890L); + + delete = p.deleteCommands.get( 6 ); + assertEquals( delete.id, "70" ); + assertEquals( delete.query, null ); + assertEquals( delete.getVersion(), 77777L); + + delete = p.deleteCommands.get( 7 ); + assertEquals( delete.id, null ); + assertEquals( delete.query, "id:80" ); + assertEquals( delete.getVersion(), 88888L); + + req.close(); + } + + } \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/handler/MoreLikeThisHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/MoreLikeThisHandlerTest.java index c7d8a39..6c8d25a 100644 --- a/solr/core/src/test/org/apache/solr/handler/MoreLikeThisHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/MoreLikeThisHandlerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/PingRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/PingRequestHandlerTest.java index 5de4a73..c90c8c6 100644 --- a/solr/core/src/test/org/apache/solr/handler/PingRequestHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/PingRequestHandlerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/StandardRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/StandardRequestHandlerTest.java index 1cb930f..070d12a 100644 --- a/solr/core/src/test/org/apache/solr/handler/StandardRequestHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/StandardRequestHandlerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java b/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java index 2712615..a2deb3b 100755 --- a/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java +++ b/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java index 05d79dc..8d837aa 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java +++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -29,12 +29,14 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.BaseDistributedSearchTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.TestDistributedSearch; @@ -61,10 +63,13 @@ import org.junit.BeforeClass; * @since 1.4 */ // TODO: can this test be sped up? it used to not be so slow... +@Slow public class TestReplicationHandler extends SolrTestCaseJ4 { - private static final String CONF_DIR = "." + File.separator + "solr" + File.separator + "conf" + File.separator; + private static final String CONF_DIR = "." + File.separator + "solr" + + File.separator + "collection1" + File.separator + "conf" + + File.separator; static JettySolrRunner masterJetty, slaveJetty; static SolrServer masterClient, slaveClient; @@ -906,7 +911,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 { assertEquals(1, files.length); snapDir[i] = files[0]; Directory dir = new SimpleFSDirectory(snapDir[i].getAbsoluteFile()); - IndexReader reader = IndexReader.open(dir); + IndexReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); TopDocs hits = searcher.search(new MatchAllDocsQuery(), 1); assertEquals(nDocs, hits.totalHits); @@ -999,8 +1004,8 @@ public class TestReplicationHandler extends SolrTestCaseJ4 { homeDir = new File(home, name); - dataDir = new File(homeDir, "data"); - confDir = new File(homeDir, "conf"); + dataDir = new File(homeDir + "/collection1", "data"); + confDir = new File(homeDir + "/collection1", "conf"); homeDir.mkdirs(); dataDir.mkdirs(); diff --git a/solr/core/src/test/org/apache/solr/handler/XmlUpdateRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/XmlUpdateRequestHandlerTest.java index 7f3cb9d..76af46d 100644 --- a/solr/core/src/test/org/apache/solr/handler/XmlUpdateRequestHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/XmlUpdateRequestHandlerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/XsltUpdateRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/XsltUpdateRequestHandlerTest.java index f2e210f..677fb50 100644 --- a/solr/core/src/test/org/apache/solr/handler/XsltUpdateRequestHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/XsltUpdateRequestHandlerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java index 68ef593..62c20de 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/admin/LoggingHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/LoggingHandlerTest.java index 2f7f82a..a81e0f1 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/LoggingHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/LoggingHandlerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/admin/LukeRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/LukeRequestHandlerTest.java index c9eea72..32213a9 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/LukeRequestHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/LukeRequestHandlerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MBeansHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MBeansHandlerTest.java index 8f4749f..ce0a8d5 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/MBeansHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/MBeansHandlerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/admin/SystemInfoHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/SystemInfoHandlerTest.java index d4be9c9..fccc1a0 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/SystemInfoHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/SystemInfoHandlerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/component/DebugComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DebugComponentTest.java index ae35d4a..6f20558 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DebugComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DebugComponentTest.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.component; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryElevationComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryElevationComponentTest.java index 05deb9e..3b396c0 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryElevationComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryElevationComponentTest.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.component; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,13 +18,10 @@ package org.apache.solr.handler.component; */ import java.io.File; -import java.io.IOException; import org.apache.solr.BaseDistributedSearchTestCase; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.CommonParams; -import org.apache.solr.util.FileUtils; -import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -44,13 +41,13 @@ public class DistributedQueryElevationComponentTest extends BaseDistributedSearc } @BeforeClass - public static void beforeClass() throws IOException { + public static void beforeClass() { System.setProperty("elevate.data.file", "elevate.xml"); File parent = new File(TEST_HOME(), "conf"); } @AfterClass - public static void afterClass() throws IOException { + public static void afterClass() { System.clearProperty("elevate.data.file"); } diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedSpellCheckComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedSpellCheckComponentTest.java index 7d19d7d..e7289b9 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DistributedSpellCheckComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedSpellCheckComponentTest.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.component; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,6 +20,7 @@ package org.apache.solr.handler.component; import junit.framework.Assert; import junit.framework.TestCase; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.BaseDistributedSearchTestCase; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.response.QueryResponse; @@ -34,9 +35,11 @@ import org.apache.solr.common.util.NamedList; * * @see org.apache.solr.handler.component.SpellCheckComponent */ +@Slow public class DistributedSpellCheckComponentTest extends BaseDistributedSearchTestCase { private String requestHandlerName; + private String reqHandlerWithWordbreak; public DistributedSpellCheckComponentTest() { @@ -52,7 +55,13 @@ public class DistributedSpellCheckComponentTest extends BaseDistributedSearchTes // this test requires FSDir saveProp = System.getProperty("solr.directoryFactory"); System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory"); - requestHandlerName = random().nextBoolean() ? "spellCheckCompRH" : "spellCheckCompRH_Direct"; + if(random().nextBoolean()) { + requestHandlerName = "spellCheckCompRH"; + reqHandlerWithWordbreak = "spellCheckWithWordbreak"; + } else { + requestHandlerName = "spellCheckCompRH_Direct"; + reqHandlerWithWordbreak = "spellCheckWithWordbreak_Direct"; + } super.setUp(); } @@ -141,5 +150,7 @@ public class DistributedSpellCheckComponentTest extends BaseDistributedSearchTes query("q", "lowerfilt:(\"quote red fox\")", "fl", "id,lowerfilt", "spellcheck", "true", "qt", "spellCheckCompRH", "shards.qt", "spellCheckCompRH", SpellCheckComponent.SPELLCHECK_EXTENDED_RESULTS, "true", SpellCheckComponent.SPELLCHECK_COUNT, "10", SpellCheckComponent.SPELLCHECK_COLLATE, "true", SpellCheckComponent.SPELLCHECK_MAX_COLLATION_TRIES, "10", SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "1", SpellCheckComponent.SPELLCHECK_COLLATE_EXTENDED_RESULTS, "true", SpellCheckComponent.SPELLCHECK_ALTERNATIVE_TERM_COUNT, "5", SpellCheckComponent.SPELLCHECK_MAX_RESULTS_FOR_SUGGEST, "10"); query("q", "lowerfilt:(\"rod fix\")", "fl", "id,lowerfilt", "spellcheck", "true", "qt", "spellCheckCompRH", "shards.qt", "spellCheckCompRH", SpellCheckComponent.SPELLCHECK_EXTENDED_RESULTS, "true", SpellCheckComponent.SPELLCHECK_COUNT, "10", SpellCheckComponent.SPELLCHECK_COLLATE, "true", SpellCheckComponent.SPELLCHECK_MAX_COLLATION_TRIES, "10", SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "1", SpellCheckComponent.SPELLCHECK_COLLATE_EXTENDED_RESULTS, "true", SpellCheckComponent.SPELLCHECK_ALTERNATIVE_TERM_COUNT, "5", SpellCheckComponent.SPELLCHECK_MAX_RESULTS_FOR_SUGGEST, "10"); + + query("q", "lowerfilt:(+quock +redfox +jum +ped)", "fl", "id,lowerfilt", "spellcheck", "true", "qt", reqHandlerWithWordbreak, "shards.qt", reqHandlerWithWordbreak, SpellCheckComponent.SPELLCHECK_EXTENDED_RESULTS, "true", SpellCheckComponent.SPELLCHECK_COUNT, "10", SpellCheckComponent.SPELLCHECK_COLLATE, "true", SpellCheckComponent.SPELLCHECK_MAX_COLLATION_TRIES, "0", SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "1", SpellCheckComponent.SPELLCHECK_COLLATE_EXTENDED_RESULTS, "true"); } } diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedTermsComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedTermsComponentTest.java index 89f3b66..c195677 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DistributedTermsComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedTermsComponentTest.java @@ -1,6 +1,6 @@ package org.apache.solr.handler.component; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/component/DummyCustomParamSpellChecker.java b/solr/core/src/test/org/apache/solr/handler/component/DummyCustomParamSpellChecker.java index 95428e7..b640e34 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DummyCustomParamSpellChecker.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DummyCustomParamSpellChecker.java @@ -10,7 +10,7 @@ import org.apache.solr.spelling.SpellingResult; import java.io.IOException; import java.util.Collections; import java.util.Iterator; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java index ec59ea3..02b52a3 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -55,7 +55,7 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 { private void init(String config, String schema) throws Exception { //write out elevate-data.xml to the Data dir first by copying it from conf, which we know exists, this way we can test both conf and data configurations createTempDir(); - File parent = new File(TEST_HOME(), "conf"); + File parent = new File(TEST_HOME() + "/collection1", "conf"); File elevateFile = new File(parent, "elevate.xml"); File elevateDataFile = new File(dataDir, "elevate-data.xml"); FileUtils.copyFile(elevateFile, elevateDataFile); diff --git a/solr/core/src/test/org/apache/solr/handler/component/SearchHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/component/SearchHandlerTest.java index bd8641d..014c381 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/SearchHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/SearchHandlerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/component/SpellCheckComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/SpellCheckComponentTest.java index 9daa92b..197a68b 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/SpellCheckComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/SpellCheckComponentTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,6 +20,7 @@ package org.apache.solr.handler.component; import java.io.File; import java.util.*; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; @@ -39,6 +40,7 @@ import org.junit.Test; /** * @since solr 1.3 */ +@Slow public class SpellCheckComponentTest extends SolrTestCaseJ4 { static String rh = "spellCheckCompRH"; diff --git a/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java index c52634f..c5e4e61 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.component; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/handler/component/TermVectorComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/TermVectorComponentTest.java index 1929645..3d46968 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/TermVectorComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/TermVectorComponentTest.java @@ -14,7 +14,7 @@ import java.util.HashMap; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -128,6 +128,73 @@ public class TermVectorComponentTest extends SolrTestCaseJ4 { " 'test_postv':{'anoth':{'tf':1},'titl':{'tf':2}}}," + " 'uniqueKeyFieldName':'id'}" ); + // tv.fl diff from fl + assertJQ(req("json.nl","map", + "qt",tv, + "q", "id:0", + "fl", "*,score", + "tv.fl", "test_basictv,test_offtv", + TermVectorComponent.COMPONENT_NAME, "true", + TermVectorParams.TF, "true") + ,"/termVectors=={'doc-0':{'uniqueKey':'0'," + + " 'test_basictv':{'anoth':{'tf':1},'titl':{'tf':2}}," + + " 'test_offtv':{'anoth':{'tf':1},'titl':{'tf':2}}}," + + " 'uniqueKeyFieldName':'id'}" + ); + // multi-valued tv.fl + assertJQ(req("json.nl","map", + "qt",tv, + "q", "id:0", + "fl", "*,score", + "tv.fl", "test_basictv", + "tv.fl","test_offtv", + TermVectorComponent.COMPONENT_NAME, "true", + TermVectorParams.TF, "true") + ,"/termVectors=={'doc-0':{'uniqueKey':'0'," + + " 'test_basictv':{'anoth':{'tf':1},'titl':{'tf':2}}," + + " 'test_offtv':{'anoth':{'tf':1},'titl':{'tf':2}}}," + + " 'uniqueKeyFieldName':'id'}" + ); + // re-use fl glob + assertJQ(req("json.nl","map", + "qt",tv, + "q", "id:0", + "fl", "*,score", + TermVectorComponent.COMPONENT_NAME, "true", + TermVectorParams.TF, "true") + ,"/termVectors=={'doc-0':{'uniqueKey':'0'," + + " 'test_basictv':{'anoth':{'tf':1},'titl':{'tf':2}}," + + " 'test_offtv':{'anoth':{'tf':1},'titl':{'tf':2}}," + + " 'test_posofftv':{'anoth':{'tf':1},'titl':{'tf':2}}," + + " 'test_postv':{'anoth':{'tf':1},'titl':{'tf':2}}}," + + " 'uniqueKeyFieldName':'id'}" + ); + // re-use fl, ignore things we can't handle + assertJQ(req("json.nl","map", + "qt",tv, + "q", "id:0", + "fl", "score,test_basictv,[docid],test_postv,val:sum(3,4)", + TermVectorComponent.COMPONENT_NAME, "true", + TermVectorParams.TF, "true") + ,"/termVectors=={'doc-0':{'uniqueKey':'0'," + + " 'test_basictv':{'anoth':{'tf':1},'titl':{'tf':2}}," + + " 'test_postv':{'anoth':{'tf':1},'titl':{'tf':2}}}," + + " 'uniqueKeyFieldName':'id'}" + ); + // re-use (multi-valued) fl, ignore things we can't handle + assertJQ(req("json.nl","map", + "qt",tv, + "q", "id:0", + "fl", "score,test_basictv", + "fl", "[docid],test_postv,val:sum(3,4)", + TermVectorComponent.COMPONENT_NAME, "true", + TermVectorParams.TF, "true") + ,"/termVectors=={'doc-0':{'uniqueKey':'0'," + + " 'test_basictv':{'anoth':{'tf':1},'titl':{'tf':2}}," + + " 'test_postv':{'anoth':{'tf':1},'titl':{'tf':2}}}," + + " 'uniqueKeyFieldName':'id'}" + ); + } @Test diff --git a/solr/core/src/test/org/apache/solr/handler/component/TermsComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/TermsComponentTest.java index eb176b2..dc0364d 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/TermsComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/TermsComponentTest.java @@ -1,5 +1,5 @@ package org.apache.solr.handler.component; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/highlight/DummyHighlighter.java b/solr/core/src/test/org/apache/solr/highlight/DummyHighlighter.java index cce6173..580ee11 100644 --- a/solr/core/src/test/org/apache/solr/highlight/DummyHighlighter.java +++ b/solr/core/src/test/org/apache/solr/highlight/DummyHighlighter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/highlight/FastVectorHighlighterTest.java b/solr/core/src/test/org/apache/solr/highlight/FastVectorHighlighterTest.java index 01b657a..88459cd 100644 --- a/solr/core/src/test/org/apache/solr/highlight/FastVectorHighlighterTest.java +++ b/solr/core/src/test/org/apache/solr/highlight/FastVectorHighlighterTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/highlight/HighlighterConfigTest.java b/solr/core/src/test/org/apache/solr/highlight/HighlighterConfigTest.java index 7581247..69f2305 100644 --- a/solr/core/src/test/org/apache/solr/highlight/HighlighterConfigTest.java +++ b/solr/core/src/test/org/apache/solr/highlight/HighlighterConfigTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java b/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java index 5c9246d..391a1fc 100755 --- a/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java +++ b/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/request/JSONWriterTest.java b/solr/core/src/test/org/apache/solr/request/JSONWriterTest.java index d568266..5c2600e 100644 --- a/solr/core/src/test/org/apache/solr/request/JSONWriterTest.java +++ b/solr/core/src/test/org/apache/solr/request/JSONWriterTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -44,7 +44,13 @@ public class JSONWriterTest extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { initCore("solrconfig.xml","schema.xml"); - } + } + + private void jsonEq(String expected, String received) { + expected = expected.trim(); + received = received.trim(); + assertEquals(expected, received); + } @Test public void testTypes() throws IOException { @@ -57,17 +63,17 @@ public class JSONWriterTest extends SolrTestCaseJ4 { rsp.add("data2", Double.NEGATIVE_INFINITY); rsp.add("data3", Float.POSITIVE_INFINITY); w.write(buf, req, rsp); - assertEquals(buf.toString(), "{'data1':float('NaN'),'data2':-float('Inf'),'data3':float('Inf')}"); + jsonEq(buf.toString(), "{'data1':float('NaN'),'data2':-float('Inf'),'data3':float('Inf')}"); w = new RubyResponseWriter(); buf = new StringWriter(); w.write(buf, req, rsp); - assertEquals(buf.toString(), "{'data1'=>(0.0/0.0),'data2'=>-(1.0/0.0),'data3'=>(1.0/0.0)}"); + jsonEq(buf.toString(), "{'data1'=>(0.0/0.0),'data2'=>-(1.0/0.0),'data3'=>(1.0/0.0)}"); w = new JSONResponseWriter(); buf = new StringWriter(); w.write(buf, req, rsp); - assertEquals(buf.toString(), "{\"data1\":\"NaN\",\"data2\":\"-Infinity\",\"data3\":\"Infinity\"}"); + jsonEq(buf.toString(), "{\"data1\":\"NaN\",\"data2\":\"-Infinity\",\"data3\":\"Infinity\"}"); req.close(); } @@ -88,7 +94,7 @@ public class JSONWriterTest extends SolrTestCaseJ4 { rsp.add("bytes", "abc".getBytes("UTF-8")); w.write(buf, req, rsp); - assertEquals("{\"nl\":[[\"data1\",\"he\\u2028llo\\u2029!\"],[null,42]],\"byte\":-3,\"short\":-4,\"bytes\":\"YWJj\"}", buf.toString()); + jsonEq("{\"nl\":[[\"data1\",\"he\\u2028llo\\u2029!\"],[null,42]],\"byte\":-3,\"short\":-4,\"bytes\":\"YWJj\"}", buf.toString()); req.close(); } diff --git a/solr/core/src/test/org/apache/solr/request/SimpleFacetsTest.java b/solr/core/src/test/org/apache/solr/request/SimpleFacetsTest.java index 3242e1c..5ea20ee 100644 --- a/solr/core/src/test/org/apache/solr/request/SimpleFacetsTest.java +++ b/solr/core/src/test/org/apache/solr/request/SimpleFacetsTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -26,14 +26,9 @@ import org.apache.solr.util.TimeZoneUtils; import org.junit.BeforeClass; import org.junit.Test; -import java.util.Arrays; import java.util.ArrayList; import java.util.Collections; -import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.HashSet; -import java.util.TimeZone; public class SimpleFacetsTest extends SolrTestCaseJ4 { @@ -120,6 +115,42 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 { } @Test + public void testSimpleGroupedQueryRangeFacets() throws Exception { + assertQ( + req( + "q", "*:*", + "fq", "id:[2000 TO 2004]", + "group", "true", + "group.facet", "true", + "group.field", "hotel_s1", + "facet", "true", + "facet.query", "airport_s1:ams" + ), + "//lst[@name='facet_queries']/int[@name='airport_s1:ams'][.='2']" + ); + assertQ( + req( + "q", "*:*", + "fq", "id:[2000 TO 2004]", + "group", "true", + "group.facet", "true", + "group.field", "hotel_s1", + "facet", "true", + "facet.range", "duration_i1", + "facet.range.start", "5", + "facet.range.end", "11", + "facet.range.gap", "1" + ), + "//lst[@name='facet_ranges']/lst[@name='duration_i1']/lst[@name='counts']/int[@name='5'][.='2']", + "//lst[@name='facet_ranges']/lst[@name='duration_i1']/lst[@name='counts']/int[@name='6'][.='0']", + "//lst[@name='facet_ranges']/lst[@name='duration_i1']/lst[@name='counts']/int[@name='7'][.='0']", + "//lst[@name='facet_ranges']/lst[@name='duration_i1']/lst[@name='counts']/int[@name='8'][.='0']", + "//lst[@name='facet_ranges']/lst[@name='duration_i1']/lst[@name='counts']/int[@name='9'][.='0']", + "//lst[@name='facet_ranges']/lst[@name='duration_i1']/lst[@name='counts']/int[@name='10'][.='2']" + ); + } + + @Test public void testSimpleGroupedFacets() throws Exception { assertQ( "Return 5 docs with id range 1937 till 1940", diff --git a/solr/core/src/test/org/apache/solr/request/TestBinaryResponseWriter.java b/solr/core/src/test/org/apache/solr/request/TestBinaryResponseWriter.java index b28e55a..c118bad 100644 --- a/solr/core/src/test/org/apache/solr/request/TestBinaryResponseWriter.java +++ b/solr/core/src/test/org/apache/solr/request/TestBinaryResponseWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/request/TestFaceting.java b/solr/core/src/test/org/apache/solr/request/TestFaceting.java index f6c21e3..c16fcdf 100755 --- a/solr/core/src/test/org/apache/solr/request/TestFaceting.java +++ b/solr/core/src/test/org/apache/solr/request/TestFaceting.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/request/TestWriterPerf.java b/solr/core/src/test/org/apache/solr/request/TestWriterPerf.java index 7bc0d77..5808ec9 100755 --- a/solr/core/src/test/org/apache/solr/request/TestWriterPerf.java +++ b/solr/core/src/test/org/apache/solr/request/TestWriterPerf.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -86,7 +86,7 @@ public class TestWriterPerf extends AbstractSolrTestCase { /** make sure to close req after you are done using the response */ - public SolrQueryResponse getResponse(SolrQueryRequest req) throws IOException, Exception { + public SolrQueryResponse getResponse(SolrQueryRequest req) throws Exception { SolrQueryResponse rsp = new SolrQueryResponse(); h.getCore().execute(h.getCore().getRequestHandler(null),req,rsp); if (rsp.getException() != null) { diff --git a/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java index e4afe09..20480c1 100644 --- a/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java +++ b/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/response/TestPHPSerializedResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestPHPSerializedResponseWriter.java index d67e1fb..d31190e 100644 --- a/solr/core/src/test/org/apache/solr/response/TestPHPSerializedResponseWriter.java +++ b/solr/core/src/test/org/apache/solr/response/TestPHPSerializedResponseWriter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java b/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java index 1076a32..e57ea96 100644 --- a/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java +++ b/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -38,7 +38,8 @@ public class BadIndexSchemaTest extends SolrTestCaseJ4 { // short circuit out if we found what we expected if (-1 != e.getMessage().indexOf(errString)) return; // Test the cause too in case the expected error is wrapped - if (-1 != e.getCause().getMessage().indexOf(errString)) return; + if (null != e.getCause() && + -1 != e.getCause().getMessage().indexOf(errString)) return; // otherwise, rethrow it, possibly completley unrelated throw new SolrException @@ -50,7 +51,6 @@ public class BadIndexSchemaTest extends SolrTestCaseJ4 { fail("Did not encounter any exception from: " + schema); } - @Test public void testSevereErrorsForInvalidFieldOptions() throws Exception { doTest("bad-schema-not-indexed-but-norms.xml", "bad_field"); doTest("bad-schema-not-indexed-but-tf.xml", "bad_field"); @@ -58,29 +58,37 @@ public class BadIndexSchemaTest extends SolrTestCaseJ4 { doTest("bad-schema-omit-tf-but-not-pos.xml", "bad_field"); } - @Test public void testSevereErrorsForDuplicateFields() throws Exception { doTest("bad-schema-dup-field.xml", "fAgain"); } - @Test public void testSevereErrorsForDuplicateDynamicField() throws Exception { doTest("bad-schema-dup-dynamicField.xml", "_twice"); } - @Test public void testSevereErrorsForDuplicateFieldType() throws Exception { doTest("bad-schema-dup-fieldType.xml", "ftAgain"); } - @Test public void testSevereErrorsForUnexpectedAnalyzer() throws Exception { doTest("bad-schema-nontext-analyzer.xml", "StrField (bad_type)"); } - @Test public void testBadExternalFileField() throws Exception { doTest("bad-schema-external-filefield.xml", - "Only float and pfloat"); + "Only float and pfloat"); } + + public void testUniqueKeyRules() throws Exception { + doTest("bad-schema-uniquekey-is-copyfield-dest.xml", + "can not be the dest of a copyField"); + doTest("bad-schema-uniquekey-uses-default.xml", + "can not be configured with a default value"); + } + + public void testPerFieldtypeSimButNoSchemaSimFactory() throws Exception { + doTest("bad-schema-sim-global-vs-ft-mismatch.xml", "global similarity does not support it"); + } + + } diff --git a/solr/core/src/test/org/apache/solr/schema/CopyFieldTest.java b/solr/core/src/test/org/apache/solr/schema/CopyFieldTest.java index 83904d1..0841e58 100644 --- a/solr/core/src/test/org/apache/solr/schema/CopyFieldTest.java +++ b/solr/core/src/test/org/apache/solr/schema/CopyFieldTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/schema/CurrencyFieldTest.java b/solr/core/src/test/org/apache/solr/schema/CurrencyFieldTest.java index da737a1..199b793 100644 --- a/solr/core/src/test/org/apache/solr/schema/CurrencyFieldTest.java +++ b/solr/core/src/test/org/apache/solr/schema/CurrencyFieldTest.java @@ -1,5 +1,5 @@ package org.apache.solr.schema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/schema/DateFieldTest.java b/solr/core/src/test/org/apache/solr/schema/DateFieldTest.java index 2cd1aea..cb507c6 100644 --- a/solr/core/src/test/org/apache/solr/schema/DateFieldTest.java +++ b/solr/core/src/test/org/apache/solr/schema/DateFieldTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/schema/IndexSchemaTest.java b/solr/core/src/test/org/apache/solr/schema/IndexSchemaTest.java index 3f611c1..9e69f33 100644 --- a/solr/core/src/test/org/apache/solr/schema/IndexSchemaTest.java +++ b/solr/core/src/test/org/apache/solr/schema/IndexSchemaTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/schema/MockExchangeRateProvider.java b/solr/core/src/test/org/apache/solr/schema/MockExchangeRateProvider.java index 56c534f..a8656aa 100644 --- a/solr/core/src/test/org/apache/solr/schema/MockExchangeRateProvider.java +++ b/solr/core/src/test/org/apache/solr/schema/MockExchangeRateProvider.java @@ -1,5 +1,5 @@ package org.apache.solr.schema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/schema/NotRequiredUniqueKeyTest.java b/solr/core/src/test/org/apache/solr/schema/NotRequiredUniqueKeyTest.java index a43fcfc..09fed1a 100644 --- a/solr/core/src/test/org/apache/solr/schema/NotRequiredUniqueKeyTest.java +++ b/solr/core/src/test/org/apache/solr/schema/NotRequiredUniqueKeyTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/schema/NumericFieldsTest.java b/solr/core/src/test/org/apache/solr/schema/NumericFieldsTest.java index 43cd7b1..ce98f5f 100644 --- a/solr/core/src/test/org/apache/solr/schema/NumericFieldsTest.java +++ b/solr/core/src/test/org/apache/solr/schema/NumericFieldsTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/schema/OpenExchangeRatesOrgProviderTest.java b/solr/core/src/test/org/apache/solr/schema/OpenExchangeRatesOrgProviderTest.java index 748ab74..8e20168 100644 --- a/solr/core/src/test/org/apache/solr/schema/OpenExchangeRatesOrgProviderTest.java +++ b/solr/core/src/test/org/apache/solr/schema/OpenExchangeRatesOrgProviderTest.java @@ -1,5 +1,5 @@ package org.apache.solr.schema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -41,7 +41,7 @@ public class OpenExchangeRatesOrgProviderTest extends SolrTestCaseJ4 { mockParams = new HashMap();; mockParams.put(OpenExchangeRatesOrgProvider.PARAM_RATES_FILE_LOCATION, "open-exchange-rates.json"); oerp = new OpenExchangeRatesOrgProvider(); - loader = new SolrResourceLoader("solr"); + loader = new SolrResourceLoader("solr/collection1"); } @Test diff --git a/solr/core/src/test/org/apache/solr/schema/PolyFieldTest.java b/solr/core/src/test/org/apache/solr/schema/PolyFieldTest.java index fa4df0f..ee02388 100644 --- a/solr/core/src/test/org/apache/solr/schema/PolyFieldTest.java +++ b/solr/core/src/test/org/apache/solr/schema/PolyFieldTest.java @@ -1,5 +1,5 @@ package org.apache.solr.schema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldTest.java b/solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldTest.java index 85ef69f..3659528 100644 --- a/solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldTest.java +++ b/solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldTest.java @@ -1,6 +1,6 @@ package org.apache.solr.schema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/schema/PrimitiveFieldTypeTest.java b/solr/core/src/test/org/apache/solr/schema/PrimitiveFieldTypeTest.java index 16136d8..29e27a9 100644 --- a/solr/core/src/test/org/apache/solr/schema/PrimitiveFieldTypeTest.java +++ b/solr/core/src/test/org/apache/solr/schema/PrimitiveFieldTypeTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,6 +19,7 @@ package org.apache.solr.schema; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.core.SolrConfig; +import org.apache.solr.core.SolrResourceLoader; import org.junit.Test; import java.io.File; @@ -29,7 +30,7 @@ import java.util.TimeZone; * Tests that defaults are set for Primitive (non-analyzed) fields */ public class PrimitiveFieldTypeTest extends SolrTestCaseJ4 { - private final String testConfHome = TEST_HOME() + File.separator + "conf"+ File.separator; + private final String testConfHome = TEST_HOME() + File.separator + "collection1" + File.separator + "conf"+ File.separator; public static TimeZone UTC = TimeZone.getTimeZone("UTC"); protected SolrConfig config; protected IndexSchema schema; @@ -43,7 +44,7 @@ public class PrimitiveFieldTypeTest extends SolrTestCaseJ4 { System.setProperty("solr.test.sys.prop2", "proptwo"); initMap = new HashMap(); - config = new SolrConfig(testConfHome + "solrconfig.xml"); + config = new SolrConfig(new SolrResourceLoader("solr/collection1"), testConfHome + "solrconfig.xml", null); } @SuppressWarnings("deprecation") diff --git a/solr/core/src/test/org/apache/solr/schema/RequiredFieldsTest.java b/solr/core/src/test/org/apache/solr/schema/RequiredFieldsTest.java index cf26a43..0407fc2 100644 --- a/solr/core/src/test/org/apache/solr/schema/RequiredFieldsTest.java +++ b/solr/core/src/test/org/apache/solr/schema/RequiredFieldsTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/schema/TestBinaryField.java b/solr/core/src/test/org/apache/solr/schema/TestBinaryField.java index f4112bf..3b27f1d 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestBinaryField.java +++ b/solr/core/src/test/org/apache/solr/schema/TestBinaryField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -55,21 +55,21 @@ public class TestBinaryField extends LuceneTestCase { "solrtest-TestBinaryField-" + System.currentTimeMillis()); File homeDir = new File(home, "example"); - File dataDir = new File(homeDir, "data"); - File confDir = new File(homeDir, "conf"); + File dataDir = new File(homeDir + "/collection1", "data"); + File confDir = new File(homeDir + "/collection1", "conf"); homeDir.mkdirs(); dataDir.mkdirs(); confDir.mkdirs(); - SolrResourceLoader loader = new SolrResourceLoader(null, null); + SolrResourceLoader loader = new SolrResourceLoader("solr/collection1"); File f = new File(confDir, "solrconfig.xml"); - String fname = "solr/conf/solrconfig-slave1.xml"; + String fname = "solr/collection1/conf/solrconfig-slave1.xml"; FileOutputStream out = new FileOutputStream(f); IOUtils.copy(loader.openResource(fname), out); out.close(); f = new File(confDir, "schema.xml"); - fname = "solr/conf/schema-binaryfield.xml"; + fname = "solr/collection1/conf/schema-binaryfield.xml"; out = new FileOutputStream(f); IOUtils.copy(loader.openResource(fname), out); out.close(); diff --git a/solr/core/src/test/org/apache/solr/schema/TestCollationField.java b/solr/core/src/test/org/apache/solr/schema/TestCollationField.java index e35c877..79bce64 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestCollationField.java +++ b/solr/core/src/test/org/apache/solr/schema/TestCollationField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -25,15 +25,12 @@ import java.util.Locale; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; -import org.apache.lucene.codecs.Codec; -import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.apache.solr.SolrTestCaseJ4; import org.junit.BeforeClass; /** * Tests {@link CollationField} with TermQueries, RangeQueries, and sort order. */ -@SuppressCodecs("Lucene3x") public class TestCollationField extends SolrTestCaseJ4 { @BeforeClass @@ -70,12 +67,12 @@ public class TestCollationField extends SolrTestCaseJ4 { // make data and conf dirs new File(tmpFile, "data").mkdir(); - File confDir = new File(tmpFile, "conf"); - confDir.mkdir(); + File confDir = new File(tmpFile + "/collection1", "conf"); + confDir.mkdirs(); // copy over configuration files - FileUtils.copyFile(getFile("solr/conf/solrconfig-basic.xml"), new File(confDir, "solrconfig.xml")); - FileUtils.copyFile(getFile("solr/conf/schema-collate.xml"), new File(confDir, "schema.xml")); + FileUtils.copyFile(getFile("solr/collection1/conf/solrconfig-basic.xml"), new File(confDir, "solrconfig.xml")); + FileUtils.copyFile(getFile("solr/collection1/conf/schema-collate.xml"), new File(confDir, "schema.xml")); // generate custom collation rules (DIN 5007-2), saving to customrules.dat RuleBasedCollator baseCollator = (RuleBasedCollator) Collator.getInstance(new Locale("de", "DE")); diff --git a/solr/core/src/test/org/apache/solr/schema/TestOmitPositions.java b/solr/core/src/test/org/apache/solr/schema/TestOmitPositions.java index cebcaea..c16d9e4 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestOmitPositions.java +++ b/solr/core/src/test/org/apache/solr/schema/TestOmitPositions.java @@ -1,6 +1,6 @@ package org.apache.solr.schema; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/schema/UUIDFieldTest.java b/solr/core/src/test/org/apache/solr/schema/UUIDFieldTest.java index 58fd090..b7fa874 100644 --- a/solr/core/src/test/org/apache/solr/schema/UUIDFieldTest.java +++ b/solr/core/src/test/org/apache/solr/schema/UUIDFieldTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/DocSetPerf.java b/solr/core/src/test/org/apache/solr/search/DocSetPerf.java index 319044b..0d904e6 100644 --- a/solr/core/src/test/org/apache/solr/search/DocSetPerf.java +++ b/solr/core/src/test/org/apache/solr/search/DocSetPerf.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/FooQParserPlugin.java b/solr/core/src/test/org/apache/solr/search/FooQParserPlugin.java index 36fe30b..bad1156 100755 --- a/solr/core/src/test/org/apache/solr/search/FooQParserPlugin.java +++ b/solr/core/src/test/org/apache/solr/search/FooQParserPlugin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java new file mode 100644 index 0000000..5c79a2d --- /dev/null +++ b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java @@ -0,0 +1,746 @@ +package org.apache.solr.search; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.HashSet; +import java.util.Set; + +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryUtils; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrRequestInfo; +import org.apache.solr.response.SolrQueryResponse; +import org.junit.AfterClass; +import org.junit.BeforeClass; + + + +/** + * Sanity checks that queries (generated by the QParser and ValueSourceParser + * framework) are appropraitely {@link Object#equals} and + * {@link Object#hashCode()} equivilent. If you are adding a new default + * QParser or ValueSourceParser, you will most likely get a failure from + * {@link #testParserCoverage} until you add a new test method to this class. + * + * @see ValueSourceParser#standardValueSourceParsers + * @see QParserPlugin#standardPlugins + * @see QueryUtils + **/ +public class QueryEqualityTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig.xml","schema15.xml"); + } + + /** @see #testParserCoverage */ + @AfterClass + public static void afterClassParserCoverageTest() { + + if ( ! doAssertParserCoverage) return; + + for (int i=0; i < QParserPlugin.standardPlugins.length; i+=2) { + assertTrue("qparser #"+i+" name not a string", + QParserPlugin.standardPlugins[i] instanceof String); + final String name = (String)QParserPlugin.standardPlugins[i]; + assertTrue("testParserCoverage was run w/o any other method explicitly testing qparser: " + name, qParsersTested.contains(name)); + } + + for (final String name : ValueSourceParser.standardValueSourceParsers.keySet()) { + assertTrue("testParserCoverage was run w/o any other method explicitly testing val parser: " + name, valParsersTested.contains(name)); + } + + } + + /** @see #testParserCoverage */ + private static boolean doAssertParserCoverage = false; + /** @see #testParserCoverage */ + private static final Set qParsersTested = new HashSet(); + /** @see #testParserCoverage */ + private static final Set valParsersTested = new HashSet(); + + + + public void testQueryLucene() throws Exception { + assertQueryEquals("lucene", "{!lucene}apache solr", + "apache solr", "apache solr "); + assertQueryEquals("lucene", "+apache +solr", "apache AND solr", + " +apache +solr"); + } + + public void testQueryLucenePlusSort() throws Exception { + assertQueryEquals("lucenePlusSort", + "apache solr", "apache solr", "apache solr ; score desc"); + assertQueryEquals("lucenePlusSort", + "+apache +solr", "apache AND solr", " +apache +solr; score desc"); + } + + public void testQueryPrefix() throws Exception { + SolrQueryRequest req = req("myField","foo_s"); + try { + assertQueryEquals("prefix", req, + "{!prefix f=$myField}asdf", + "{!prefix f=foo_s}asdf"); + } finally { + req.close(); + } + } + + public void testQueryBoost() throws Exception { + SolrQueryRequest req = req("df","foo_s","myBoost","sum(3,foo_i)"); + try { + assertQueryEquals("boost", req, + "{!boost b=$myBoost}asdf", + "{!boost b=$myBoost v=asdf}", + "{!boost b=sum(3,foo_i)}foo_s:asdf"); + } finally { + req.close(); + } + } + + public void testQueryDismax() throws Exception { + for (final String type : new String[]{"dismax","edismax"}) { + assertQueryEquals(type, "{!"+type+"}apache solr", + "apache solr", "apache solr", "apache solr "); + assertQueryEquals(type, "+apache +solr", "apache AND solr", + " +apache +solr"); + } + } + public void testField() throws Exception { + SolrQueryRequest req = req("myField","foo_s"); + try { + assertQueryEquals("field", req, + "{!field f=$myField}asdf", + "{!field f=$myField v=asdf}", + "{!field f=foo_s}asdf"); + } finally { + req.close(); + } + } + + public void testQueryRaw() throws Exception { + SolrQueryRequest req = req("myField","foo_s"); + try { + assertQueryEquals("raw", req, + "{!raw f=$myField}asdf", + "{!raw f=$myField v=asdf}", + "{!raw f=foo_s}asdf"); + } finally { + req.close(); + } + } + + public void testQueryTerm() throws Exception { + SolrQueryRequest req = req("myField","foo_s"); + try { + assertQueryEquals("term", req, + "{!term f=$myField}asdf", + "{!term f=$myField v=asdf}", + "{!term f=foo_s}asdf"); + } finally { + req.close(); + } + } + + public void testQueryNested() throws Exception { + SolrQueryRequest req = req("df", "foo_s"); + try { + assertQueryEquals("query", req, + "{!query defType=lucene}asdf", + "{!query v='foo_s:asdf'}", + "{!query}foo_s:asdf", + "{!query}asdf"); + } finally { + req.close(); + } + } + + public void testQueryFunc() throws Exception { + // more involved tests of specific functions in other methods + SolrQueryRequest req = req("myVar", "5", + "myField","foo_i", + "myInner","product(4,foo_i)"); + try { + assertQueryEquals("func", req, + "{!func}sum(4,5)", + "{!func}sum(4,$myVar)", + "sum(4,5)"); + assertQueryEquals("func", req, + "{!func}sum(1,2,3,4,5)", + "{!func}sum(1,2,3,4,$myVar)", + "sum(1,2,3,4,5)"); + assertQueryEquals("func", req, + "{!func}sum(4,$myInner)", + "{!func}sum(4,product(4,foo_i))", + "{!func}sum(4,product(4,$myField))", + "{!func}sum(4,product(4,field(foo_i)))"); + } finally { + req.close(); + } + } + + public void testQueryFrange() throws Exception { + SolrQueryRequest req = req("myVar", "5", + "low","0.2", + "high", "20.4", + "myField","foo_i", + "myInner","product(4,foo_i)"); + try { + assertQueryEquals("frange", req, + "{!frange l=0.2 h=20.4}sum(4,5)", + "{!frange l=$low h=$high}sum(4,$myVar)"); + } finally { + req.close(); + } + } + + public void testQueryGeofilt() throws Exception { + checkQuerySpatial("geofilt"); + } + public void testQueryBbox() throws Exception { + checkQuerySpatial("bbox"); + } + + private void checkQuerySpatial(final String type) throws Exception { + SolrQueryRequest req = req("myVar", "5", + "d","109", + "pt","10.312,-20.556", + "sfield","store"); + try { + assertQueryEquals(type, req, + "{!"+type+" d=109}", + "{!"+type+" sfield=$sfield}", + "{!"+type+" sfield=store d=109}", + "{!"+type+" sfield=store d=$d pt=$pt}", + "{!"+type+" sfield=store d=$d pt=10.312,-20.556}", + "{!"+type+"}"); + // diff SpatialQueryable FieldTypes matter for determining final query + assertQueryEquals(type, req, + "{!"+type+" sfield=point_hash}", + "{!"+type+" sfield=point_hash d=109}", + "{!"+type+" sfield=point_hash d=$d pt=$pt}", + "{!"+type+" sfield=point_hash d=$d pt=10.312,-20.556}"); + assertQueryEquals(type, req, + "{!"+type+" sfield=point}", + "{!"+type+" sfield=point d=109}", + "{!"+type+" sfield=point d=$d pt=$pt}", + "{!"+type+" sfield=point d=$d pt=10.312,-20.556}"); + } finally { + req.close(); + } + } + public void testQueryJoin() throws Exception { + SolrQueryRequest req = req("myVar", "5", + "df","text", + "ff","foo_s", + "tt", "bar_s"); + + try { + assertQueryEquals("join", req, + "{!join from=foo_s to=bar_s}asdf", + "{!join from=$ff to=$tt}asdf", + "{!join from=$ff to='bar_s'}text:asdf"); + } finally { + req.close(); + } + } + + public void testQuerySurround() throws Exception { + assertQueryEquals("surround", "{!surround}and(apache,solr)", + "and(apache,solr)", "apache AND solr"); + } + + public void testFuncTestfunc() throws Exception { + assertFuncEquals("testfunc(foo_i)","testfunc(field(foo_i))"); + assertFuncEquals("testfunc(23)"); + assertFuncEquals("testfunc(sum(23,foo_i))", + "testfunc(sum(23,field(foo_i)))"); + } + public void testFuncOrd() throws Exception { + assertFuncEquals("ord(foo_s)","ord(foo_s )"); + } + + public void testFuncLiteral() throws Exception { + SolrQueryRequest req = req("someVar","a string"); + try { + assertFuncEquals(req, + "literal('a string')","literal(\"a string\")", + "literal($someVar)"); + } finally { + req.close(); + } + } + public void testFuncRord() throws Exception { + assertFuncEquals("rord(foo_s)","rord(foo_s )"); + } + public void testFuncTop() throws Exception { + assertFuncEquals("top(sum(3,foo_i))"); + } + public void testFuncLinear() throws Exception { + SolrQueryRequest req = req("someVar","27"); + try { + assertFuncEquals(req, + "linear(foo_i,$someVar,42)", + "linear(foo_i, 27, 42)"); + } finally { + req.close(); + } + } + public void testFuncRecip() throws Exception { + SolrQueryRequest req = req("someVar","27"); + try { + assertFuncEquals(req, + "recip(foo_i,$someVar,42, 27 )", + "recip(foo_i, 27, 42,$someVar)"); + } finally { + req.close(); + } + } + public void testFuncScale() throws Exception { + SolrQueryRequest req = req("someVar","27"); + try { + assertFuncEquals(req, + "scale(field(foo_i),$someVar,42)", + "scale(foo_i, 27, 42)"); + } finally { + req.close(); + } + } + public void testFuncDiv() throws Exception { + assertFuncEquals("div(5,4)", "div(5, 4)"); + assertFuncEquals("div(foo_i,4)", "div(foo_i, 4)", + "div(field('foo_i'), 4)"); + assertFuncEquals("div(foo_i,sub(4,field('bar_i')))", + "div(field(foo_i), sub(4,bar_i))"); + + } + public void testFuncMap() throws Exception { + assertFuncEquals("map(field(foo_i), 0, 45, 100)", + "map(foo_i, 0.0, 45, 100)"); + } + + public void testFuncSum() throws Exception { + assertFuncEquals("sum(5,4)", "add(5, 4)"); + assertFuncEquals("sum(5,4,3,2,1)", "add(5, 4, 3, 2, 1)"); + assertFuncEquals("sum(foo_i,4)", "sum(foo_i, 4)", + "sum(field('foo_i'), 4)"); + assertFuncEquals("add(foo_i,sub(4,field('bar_i')))", + "sum(field(foo_i), sub(4,bar_i))"); + + } + + public void testFuncProduct() throws Exception { + assertFuncEquals("product(5,4,3,2,1)", "mul(5, 4, 3, 2, 1)"); + assertFuncEquals("product(5,4)", "mul(5, 4)"); + assertFuncEquals("product(foo_i,4)", "product(foo_i, 4)", + "product(field('foo_i'), 4)"); + assertFuncEquals("mul(foo_i,sub(4,field('bar_i')))", + "product(field(foo_i), sub(4,bar_i))"); + + } + public void testFuncSub() throws Exception { + assertFuncEquals("sub(5,4)", "sub(5, 4)"); + assertFuncEquals("sub(foo_i,4)", "sub(foo_i, 4)"); + assertFuncEquals("sub(foo_i,sum(4,bar_i))", "sub(foo_i, sum(4,bar_i))"); + } + public void testFuncVector() throws Exception { + assertFuncEquals("vector(5,4, field(foo_i))", "vector(5, 4, foo_i)"); + assertFuncEquals("vector(foo_i,4)", "vector(foo_i, 4)"); + assertFuncEquals("vector(foo_i,sum(4,bar_i))", "vector(foo_i, sum(4,bar_i))"); + } + public void testFuncQuery() throws Exception { + SolrQueryRequest req = req("myQ","asdf"); + try { + assertFuncEquals(req, + "query($myQ)", + "query($myQ,0)", + "query({!lucene v=$myQ},0)"); + } finally { + req.close(); + } + } + public void testFuncBoost() throws Exception { + SolrQueryRequest req = req("myQ","asdf"); + try { + assertFuncEquals(req, + "boost($myQ,sum(4,5))", + "boost({!lucene v=$myQ},sum(4,5))"); + } finally { + req.close(); + } + } + public void testFuncJoindf() throws Exception { + assertFuncEquals("joindf(foo,bar)"); + } + + public void testFuncGeodist() throws Exception { + SolrQueryRequest req = req("pt","10.312,-20.556", + "sfield","store"); + try { + assertFuncEquals(req, + "geodist()", + "geodist($sfield,$pt)", + "geodist(store,$pt)", + "geodist(field(store),$pt)", + "geodist(store,10.312,-20.556)"); + } finally { + req.close(); + } + } + + public void testFuncHsin() throws Exception { + assertFuncEquals("hsin(45,true,0,0,45,45)"); + } + public void testFuncGhhsin() throws Exception { + assertFuncEquals("ghhsin(45,point_hash,'asdf')", + "ghhsin(45,field(point_hash),'asdf')"); + } + public void testFuncGeohash() throws Exception { + assertFuncEquals("geohash(45,99)"); + } + public void testFuncDist() throws Exception { + assertFuncEquals("dist(2,45,99,101,111)", + "dist(2,vector(45,99),vector(101,111))"); + } + public void testFuncSqedist() throws Exception { + assertFuncEquals("sqedist(45,99,101,111)", + "sqedist(vector(45,99),vector(101,111))"); + } + public void testFuncMin() throws Exception { + assertFuncEquals("min(5,4,3,2,1)", "min(5, 4, 3, 2, 1)"); + assertFuncEquals("min(foo_i,4)", "min(field('foo_i'), 4)"); + assertFuncEquals("min(foo_i,sub(4,field('bar_i')))", + "min(field(foo_i), sub(4,bar_i))"); + } + public void testFuncMax() throws Exception { + assertFuncEquals("max(5,4,3,2,1)", "max(5, 4, 3, 2, 1)"); + assertFuncEquals("max(foo_i,4)", "max(field('foo_i'), 4)"); + assertFuncEquals("max(foo_i,sub(4,field('bar_i')))", + "max(field(foo_i), sub(4,bar_i))"); + } + + public void testFuncMs() throws Exception { + // Note ms() takes in field name, not field(...) + assertFuncEquals("ms()", "ms(NOW)"); + assertFuncEquals("ms(2000-01-01T00:00:00Z)", + "ms('2000-01-01T00:00:00Z')"); + assertFuncEquals("ms(myDateField_dt)", + "ms('myDateField_dt')"); + assertFuncEquals("ms(2000-01-01T00:00:00Z,myDateField_dt)", + "ms('2000-01-01T00:00:00Z','myDateField_dt')"); + assertFuncEquals("ms(myDateField_dt, NOW)", + "ms('myDateField_dt', NOW)"); + } + public void testFuncMathConsts() throws Exception { + assertFuncEquals("pi()"); + assertFuncEquals("e()"); + } + public void testFuncTerms() throws Exception { + SolrQueryRequest req = req("myField","field_t","myTerm","my term"); + try { + for (final String type : new String[]{"docfreq","termfreq", + "totaltermfreq","ttf", + "idf","tf"}) { + // NOTE: these functions takes a field *name* not a field(..) source + assertFuncEquals(req, + type + "('field_t','my term')", + type + "(field_t,'my term')", + type + "(field_t,$myTerm)", + type + "(field_t,$myTerm)", + type + "($myField,$myTerm)"); + } + + // ttf is an alias for totaltermfreq + assertFuncEquals(req, + "ttf(field_t,'my term')", "ttf('field_t','my term')", + "totaltermfreq(field_t,'my term')"); + + } finally { + req.close(); + } + } + public void testFuncSttf() throws Exception { + // sttf is an alias for sumtotaltermfreq + assertFuncEquals("sttf(foo_t)", "sttf('foo_t')", + "sumtotaltermfreq(foo_t)", "sumtotaltermfreq('foo_t')"); + assertFuncEquals("sumtotaltermfreq('foo_t')"); + } + public void testFuncNorm() throws Exception { + assertFuncEquals("norm(foo_t)","norm('foo_t')"); + } + public void testFuncMaxdoc() throws Exception { + assertFuncEquals("maxdoc()"); + } + public void testFuncNumdocs() throws Exception { + assertFuncEquals("numdocs()"); + } + + public void testFuncBools() throws Exception { + SolrQueryRequest req = req("myTrue","true","myFalse","false"); + try { + assertFuncEquals(req, "true","$myTrue"); + assertFuncEquals(req, "false","$myFalse"); + } finally { + req.close(); + } + } + + public void testFuncExists() throws Exception { + SolrQueryRequest req = req("myField","field_t","myQ","asdf"); + try { + assertFuncEquals(req, + "exists(field_t)", + "exists($myField)", + "exists(field('field_t'))", + "exists(field($myField))"); + assertFuncEquals(req, + "exists(query($myQ))", + "exists(query({!lucene v=$myQ}))"); + } finally { + req.close(); + } + } + + public void testFuncNot() throws Exception { + SolrQueryRequest req = req("myField","field_b", "myTrue","true"); + try { + assertFuncEquals(req, "not(true)", "not($myTrue)"); + assertFuncEquals(req, "not(not(true))", "not(not($myTrue))"); + assertFuncEquals(req, + "not(field_b)", + "not($myField)", + "not(field('field_b'))", + "not(field($myField))"); + assertFuncEquals(req, + "not(exists(field_b))", + "not(exists($myField))", + "not(exists(field('field_b')))", + "not(exists(field($myField)))"); + + } finally { + req.close(); + } + } + public void testFuncDoubleValueBools() throws Exception { + SolrQueryRequest req = req("myField","field_b","myTrue","true"); + try { + for (final String type : new String[]{"and","or","xor"}) { + assertFuncEquals(req, + type + "(field_b,true)", + type + "(field_b,$myTrue)", + type + "(field('field_b'),true)", + type + "(field($myField),$myTrue)", + type + "($myField,$myTrue)"); + } + } finally { + req.close(); + } + } + + public void testFuncIf() throws Exception { + SolrQueryRequest req = req("myBoolField","foo_b", + "myIntField","bar_i", + "myTrue","true"); + try { + assertFuncEquals(req, + "if(foo_b,bar_i,25)", + "if($myBoolField,bar_i,25)", + "if(field('foo_b'),$myIntField,25)", + "if(field($myBoolField),field('bar_i'),25)"); + assertFuncEquals(req, + "if(true,37,field($myIntField))", + "if($myTrue,37,$myIntField)"); + } finally { + req.close(); + } + } + + public void testFuncDef() throws Exception { + SolrQueryRequest req = req("myField","bar_f"); + + try { + assertFuncEquals(req, + "def(bar_f,25)", + "def($myField,25)", + "def(field('bar_f'),25)"); + assertFuncEquals(req, + "def(ceil(bar_f),25)", + "def(ceil($myField),25)", + "def(ceil(field('bar_f')),25)"); + } finally { + req.close(); + } + } + + public void testFuncSingleValueMathFuncs() throws Exception { + SolrQueryRequest req = req("myVal","45", "myField","foo_i"); + for (final String func : new String[] {"abs","rad","deg","sqrt","cbrt", + "log","ln","exp","sin","cos","tan", + "asin","acos","atan", + "sinh","cosh","tanh", + "ceil","floor","rint"}) { + try { + assertFuncEquals(req, + func + "(field(foo_i))", func + "(foo_i)", + func + "($myField)"); + assertFuncEquals(req, func + "(45)", func+ "($myVal)"); + } finally { + req.close(); + } + } + } + + public void testFuncDoubleValueMathFuncs() throws Exception { + SolrQueryRequest req = req("myVal","45", "myOtherVal", "27", + "myField","foo_i"); + for (final String func : new String[] {"pow","hypot","atan2"}) { + try { + assertFuncEquals(req, + func + "(field(foo_i),$myVal)", func+"(foo_i,$myVal)", + func + "($myField,45)"); + assertFuncEquals(req, + func+"(45,$myOtherVal)", func+"($myVal,27)", + func+"($myVal,$myOtherVal)"); + + } finally { + req.close(); + } + } + } + + public void testFuncStrdist() throws Exception { + SolrQueryRequest req = req("myVal","zot", "myOtherVal", "yak", + "myField","foo_s1"); + try { + assertFuncEquals(req, + "strdist(\"zot\",literal('yak'),edit)", + "strdist(literal(\"zot\"),'yak', edit )", + "strdist(literal($myVal),literal($myOtherVal),edit)"); + assertFuncEquals(req, + "strdist(\"zot\",literal($myOtherVal),ngram)", + "strdist(\"zot\",'yak', ngram, 2)"); + assertFuncEquals(req, + "strdist(field('foo_s1'),literal($myOtherVal),jw)", + "strdist(field($myField),\"yak\",jw)", + "strdist($myField,'yak', jw)"); + } finally { + req.close(); + } + } + public void testFuncField() throws Exception { + assertFuncEquals("field(\"foo_i\")", + "field('foo_i\')", + "foo_i"); + } + + /** + * this test does not assert anything itself, it simply toggles a static + * boolean informing an @AfterClass method to assert that every default + * qparser and valuesource parser configured was recorded by + * assertQueryEquals and assertFuncEquals. + */ + public void testParserCoverage() { + doAssertParserCoverage = true; + } + + + /** + * NOTE: defType is not only used to pick the parser, but also to record + * the parser being tested for coverage sanity checking + * @see #testParserCoverage + * @see #assertQueryEquals + */ + protected void assertQueryEquals(final String defType, + final String... inputs) throws Exception { + SolrQueryRequest req = req(); + try { + assertQueryEquals(defType, req, inputs); + } finally { + req.close(); + } + } + + /** + * NOTE: defType is not only used to pick the parser, but also to record + * the parser being tested for coverage sanity checking + * + * @see QueryUtils#check + * @see QueryUtils#checkEqual + * @see #testParserCoverage + */ + protected void assertQueryEquals(final String defType, + final SolrQueryRequest req, + final String... inputs) throws Exception { + qParsersTested.add(defType); + + final Query[] queries = new Query[inputs.length]; + + try { + SolrQueryResponse rsp = new SolrQueryResponse(); + SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req,rsp)); + for (int i = 0; i < inputs.length; i++) { + queries[i] = (QParser.getParser(inputs[i], defType, req).getQuery()); + } + } finally { + SolrRequestInfo.clearRequestInfo(); + } + + for (int i = 0; i < queries.length; i++) { + QueryUtils.check(queries[i]); + // yes starting j=0 is redundent, we're making sure every query + // is equal to itself, and that the quality checks work regardless + // of which caller/callee is used. + for (int j = 0; j < queries.length; j++) { + QueryUtils.checkEqual(queries[i], queries[j]); + } + } + } + + /** + * the function name for val parser coverage checking is extracted from + * the first input + * @see #assertQueryEquals + * @see #testParserCoverage + */ + protected void assertFuncEquals(final String... inputs) throws Exception { + SolrQueryRequest req = req(); + try { + assertFuncEquals(req, inputs); + } finally { + req.close(); + } + } + + /** + * the function name for val parser coverage checking is extracted from + * the first input + * @see #assertQueryEquals + * @see #testParserCoverage + */ + protected void assertFuncEquals(final SolrQueryRequest req, + final String... inputs) throws Exception { + // pull out the function name + final String funcName = (new QueryParsing.StrParser(inputs[0])).getId(); + valParsersTested.add(funcName); + + assertQueryEquals(FunctionQParserPlugin.NAME, req, inputs); + } + + + +} diff --git a/solr/core/src/test/org/apache/solr/search/QueryParsingTest.java b/solr/core/src/test/org/apache/solr/search/QueryParsingTest.java index 8ac4a90..6e13ce2 100644 --- a/solr/core/src/test/org/apache/solr/search/QueryParsingTest.java +++ b/solr/core/src/test/org/apache/solr/search/QueryParsingTest.java @@ -1,5 +1,5 @@ package org.apache.solr.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -231,4 +231,26 @@ public class QueryParsingTest extends SolrTestCaseJ4 { req.close(); } + public void testLiteralFunction() throws Exception { + + final String NAME = FunctionQParserPlugin.NAME; + + SolrQueryRequest req = req("variable", "foobar"); + + assertNotNull(QParser.getParser + ("literal('a value')", + NAME, req).getQuery()); + assertNotNull(QParser.getParser + ("literal('a value')", + NAME, req).getQuery()); + assertNotNull(QParser.getParser + ("literal(\"a value\")", + NAME, req).getQuery()); + assertNotNull(QParser.getParser + ("literal($variable)", + NAME, req).getQuery()); + assertNotNull(QParser.getParser + ("strdist(\"a value\",literal('a value'),edit)", + NAME, req).getQuery()); + } } diff --git a/solr/core/src/test/org/apache/solr/search/ReturnFieldsTest.java b/solr/core/src/test/org/apache/solr/search/ReturnFieldsTest.java index 5381f4f..cc57cdb 100644 --- a/solr/core/src/test/org/apache/solr/search/ReturnFieldsTest.java +++ b/solr/core/src/test/org/apache/solr/search/ReturnFieldsTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/SpatialFilterTest.java b/solr/core/src/test/org/apache/solr/search/SpatialFilterTest.java index 6012870..550108b 100644 --- a/solr/core/src/test/org/apache/solr/search/SpatialFilterTest.java +++ b/solr/core/src/test/org/apache/solr/search/SpatialFilterTest.java @@ -1,5 +1,5 @@ package org.apache.solr.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/TestDocSet.java b/solr/core/src/test/org/apache/solr/search/TestDocSet.java index 2222775..b96c341 100644 --- a/solr/core/src/test/org/apache/solr/search/TestDocSet.java +++ b/solr/core/src/test/org/apache/solr/search/TestDocSet.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,6 +19,7 @@ package org.apache.solr.search; import java.io.IOException; import java.util.Arrays; +import java.util.List; import java.util.Random; import org.apache.lucene.index.FieldInfo; @@ -471,7 +472,7 @@ public class TestDocSet extends LuceneTestCase { DocIdSet da; DocIdSet db; - AtomicReaderContext[] leaves = topLevelContext.leaves(); + List leaves = topLevelContext.leaves(); // first test in-sequence sub readers for (AtomicReaderContext readerContext : leaves) { @@ -480,10 +481,10 @@ public class TestDocSet extends LuceneTestCase { doTestIteratorEqual(da, db); } - int nReaders = leaves.length; + int nReaders = leaves.size(); // now test out-of-sequence sub readers for (int i=0; i fastCache = new FastLRUCache(); Map params = new HashMap(); params.put("size", String.valueOf(limit)); @@ -177,7 +177,7 @@ public class TestFastLRUCache extends LuceneTestCase { private CacheRegenerator createCodeRegenerator() { CacheRegenerator cr = new CacheRegenerator() { public boolean regenerateItem(SolrIndexSearcher newSearcher, SolrCache newCache, - SolrCache oldCache, Object oldKey, Object oldVal) throws IOException { + SolrCache oldCache, Object oldKey, Object oldVal) { newCache.put(oldKey, oldVal); return true; } diff --git a/solr/core/src/test/org/apache/solr/search/TestFiltering.java b/solr/core/src/test/org/apache/solr/search/TestFiltering.java index a2caf9f..cc71f82 100644 --- a/solr/core/src/test/org/apache/solr/search/TestFiltering.java +++ b/solr/core/src/test/org/apache/solr/search/TestFiltering.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/TestIndexSearcher.java b/solr/core/src/test/org/apache/solr/search/TestIndexSearcher.java index 4b7159e..229fc28 100755 --- a/solr/core/src/test/org/apache/solr/search/TestIndexSearcher.java +++ b/solr/core/src/test/org/apache/solr/search/TestIndexSearcher.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,14 +18,15 @@ package org.apache.solr.search; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReaderContext; +import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; -import org.apache.lucene.util.ReaderUtil; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.schema.SchemaField; import org.junit.BeforeClass; +import java.util.List; import java.util.Map; import java.io.IOException; @@ -50,9 +51,9 @@ public class TestIndexSearcher extends SolrTestCaseJ4 { Map context = ValueSource.newContext(sqr.getSearcher()); vs.createWeight(context, sqr.getSearcher()); IndexReaderContext topReaderContext = sqr.getSearcher().getTopReaderContext(); - AtomicReaderContext[] leaves = topReaderContext.leaves(); + List leaves = topReaderContext.leaves(); int idx = ReaderUtil.subIndex(doc, leaves); - AtomicReaderContext leaf = leaves[idx]; + AtomicReaderContext leaf = leaves.get(idx); FunctionValues vals = vs.getValues(context, leaf); return vals.strVal(doc-leaf.docBase); } @@ -78,7 +79,7 @@ public class TestIndexSearcher extends SolrTestCaseJ4 { // make sure the readers share the first segment // Didn't work w/ older versions of lucene2.9 going from segment -> multi - assertEquals(rCtx1.leaves()[0].reader(), rCtx2.leaves()[0].reader()); + assertEquals(rCtx1.leaves().get(0).reader(), rCtx2.leaves().get(0).reader()); assertU(adoc("id","5", "v_f","3.14159")); assertU(adoc("id","6", "v_f","8983", "v_s1","string6")); @@ -88,8 +89,8 @@ public class TestIndexSearcher extends SolrTestCaseJ4 { IndexReaderContext rCtx3 = sr3.getSearcher().getTopReaderContext(); // make sure the readers share segments // assertEquals(r1.getLeafReaders()[0], r3.getLeafReaders()[0]); - assertEquals(rCtx2.leaves()[0].reader(), rCtx3.leaves()[0].reader()); - assertEquals(rCtx2.leaves()[1].reader(), rCtx3.leaves()[1].reader()); + assertEquals(rCtx2.leaves().get(0).reader(), rCtx3.leaves().get(0).reader()); + assertEquals(rCtx2.leaves().get(1).reader(), rCtx3.leaves().get(1).reader()); sr1.close(); sr2.close(); @@ -123,8 +124,8 @@ public class TestIndexSearcher extends SolrTestCaseJ4 { assertU(commit()); SolrQueryRequest sr6 = req("q","foo"); IndexReaderContext rCtx6 = sr6.getSearcher().getTopReaderContext(); - assertEquals(1, rCtx6.leaves()[0].reader().numDocs()); // only a single doc left in the first segment - assertTrue( !rCtx5.leaves()[0].reader().equals(rCtx6.leaves()[0].reader()) ); // readers now different + assertEquals(1, rCtx6.leaves().get(0).reader().numDocs()); // only a single doc left in the first segment + assertTrue( !rCtx5.leaves().get(0).reader().equals(rCtx6.leaves().get(0).reader()) ); // readers now different sr5.close(); sr6.close(); diff --git a/solr/core/src/test/org/apache/solr/search/TestLRUCache.java b/solr/core/src/test/org/apache/solr/search/TestLRUCache.java index 7ff5b76..2bc886b 100644 --- a/solr/core/src/test/org/apache/solr/search/TestLRUCache.java +++ b/solr/core/src/test/org/apache/solr/search/TestLRUCache.java @@ -1,6 +1,6 @@ package org.apache.solr.search; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -63,7 +63,7 @@ public class TestLRUCache extends LuceneTestCase { doTestPercentageAutowarm(10, 10, new int[]{10}, new int[]{1, 5, 9, 100, 200, 300, 400, 800, 899, 900}); } - private void doTestPercentageAutowarm(int limit, int percentage, int[] hits, int[]misses) throws IOException { + private void doTestPercentageAutowarm(int limit, int percentage, int[] hits, int[]misses) { LRUCache lruCache = new LRUCache(); Map params = new HashMap(); params.put("size", String.valueOf(limit)); @@ -126,7 +126,7 @@ public class TestLRUCache extends LuceneTestCase { CacheRegenerator cr = new CacheRegenerator() { @SuppressWarnings("unchecked") public boolean regenerateItem(SolrIndexSearcher newSearcher, SolrCache newCache, - SolrCache oldCache, Object oldKey, Object oldVal) throws IOException { + SolrCache oldCache, Object oldKey, Object oldVal) { newCache.put(oldKey, oldVal); return true; } diff --git a/solr/core/src/test/org/apache/solr/search/TestPseudoReturnFields.java b/solr/core/src/test/org/apache/solr/search/TestPseudoReturnFields.java index 78a793a..05a8de1 100644 --- a/solr/core/src/test/org/apache/solr/search/TestPseudoReturnFields.java +++ b/solr/core/src/test/org/apache/solr/search/TestPseudoReturnFields.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/TestQueryTypes.java b/solr/core/src/test/org/apache/solr/search/TestQueryTypes.java index d02b53d..10bbd96 100755 --- a/solr/core/src/test/org/apache/solr/search/TestQueryTypes.java +++ b/solr/core/src/test/org/apache/solr/search/TestQueryTypes.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/TestQueryUtils.java b/solr/core/src/test/org/apache/solr/search/TestQueryUtils.java index 4904d9b..11d7f88 100755 --- a/solr/core/src/test/org/apache/solr/search/TestQueryUtils.java +++ b/solr/core/src/test/org/apache/solr/search/TestQueryUtils.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/TestRTGBase.java b/solr/core/src/test/org/apache/solr/search/TestRTGBase.java new file mode 100644 index 0000000..1746885 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/search/TestRTGBase.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search; + + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BytesRef; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.update.UpdateLog; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; + +import static org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase; +import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM; + +public class TestRTGBase extends SolrTestCaseJ4 { + + // means we've seen the leader and have version info (i.e. we are a non-leader replica) + public static String FROM_LEADER = DistribPhase.FROMLEADER.toString(); + + // since we make up fake versions in these tests, we can get messed up by a DBQ with a real version + // since Solr can think following updates were reordered. + @Override + public void clearIndex() { + try { + deleteByQueryAndGetVersion("*:*", params("_version_", Long.toString(-Long.MAX_VALUE), DISTRIB_UPDATE_PARAM,FROM_LEADER)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + protected final ConcurrentHashMap model = new ConcurrentHashMap(); + protected Map committedModel = new HashMap(); + protected long snapshotCount; + protected long committedModelClock; + protected volatile int lastId; + protected final String field = "val_l"; + protected Object[] syncArr; + + protected Object globalLock = this; + + protected void initModel(int ndocs) { + snapshotCount = 0; + committedModelClock = 0; + lastId = 0; + + syncArr = new Object[ndocs]; + + for (int i=0; i 0) { + // return a random number not equal to version + for (;;) { + long badVersion = rand.nextInt(); + if (badVersion != version && badVersion != 0) return badVersion; + } + } + + // if the version does not exist, then we can only specify a positive version + for (;;) { + long badVersion = rand.nextInt() & 0x7fffffff; // mask off sign bit + if (badVersion != 0) return badVersion; + } + } + + + protected List getLatestVersions() { + List recentVersions; + UpdateLog.RecentUpdates startingRecentUpdates = h.getCore().getUpdateHandler().getUpdateLog().getRecentUpdates(); + try { + recentVersions = startingRecentUpdates.getVersions(100); + } finally { + startingRecentUpdates.close(); + } + return recentVersions; + } + + + + protected int getFirstMatch(IndexReader r, Term t) throws IOException { + Fields fields = MultiFields.getFields(r); + if (fields == null) return -1; + Terms terms = fields.terms(t.field()); + if (terms == null) return -1; + BytesRef termBytes = t.bytes(); + final TermsEnum termsEnum = terms.iterator(null); + if (!termsEnum.seekExact(termBytes, false)) { + return -1; + } + DocsEnum docs = termsEnum.docs(MultiFields.getLiveDocs(r), null, false); + int id = docs.nextDoc(); + if (id != DocIdSetIterator.NO_MORE_DOCS) { + int next = docs.nextDoc(); + assertEquals(DocIdSetIterator.NO_MORE_DOCS, next); + } + return id == DocIdSetIterator.NO_MORE_DOCS ? -1 : id; + } + +} diff --git a/solr/core/src/test/org/apache/solr/search/TestRangeQuery.java b/solr/core/src/test/org/apache/solr/search/TestRangeQuery.java index 8aa85de..bcafde6 100644 --- a/solr/core/src/test/org/apache/solr/search/TestRangeQuery.java +++ b/solr/core/src/test/org/apache/solr/search/TestRangeQuery.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java b/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java index c8e39e4..653340b 100644 --- a/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java +++ b/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,50 +17,34 @@ package org.apache.solr.search; -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.index.*; -import org.apache.lucene.search.*; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.Constants; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.noggit.ObjectBuilder; -import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.params.SolrParams; import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.update.UpdateHandler; -import org.apache.solr.update.UpdateLog; -import org.apache.solr.update.VersionInfo; import org.apache.solr.util.TestHarness; import org.junit.BeforeClass; import org.junit.Test; -import java.io.IOException; -import java.util.*; -import java.util.concurrent.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import static org.apache.solr.core.SolrCore.verbose; +import static org.apache.solr.core.SolrCore.verbose; import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM; -import static org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase; -public class TestRealTimeGet extends SolrTestCaseJ4 { - - // means we've seen the leader and have version info (i.e. we are a non-leader replica) - private static String FROM_LEADER = DistribPhase.FROMLEADER.toString(); +public class TestRealTimeGet extends TestRTGBase { @BeforeClass public static void beforeClass() throws Exception { initCore("solrconfig-tlog.xml","schema15.xml"); } + @Test public void testGetRealtime() throws Exception { clearIndex(); @@ -210,7 +194,7 @@ public class TestRealTimeGet extends SolrTestCaseJ4 { assertU(commit()); // make sure a reordered add doesn't take affect. - updateJ(jsonAdd(sdoc("id","1", "_version_",Long.toString(version - 1))), params(DISTRIB_UPDATE_PARAM,FROM_LEADER)); + long version2 = deleteByQueryAndGetVersion("id:2", null); // test that it's still deleted assertJQ(req("qt","/get","id","1") @@ -218,9 +202,33 @@ public class TestRealTimeGet extends SolrTestCaseJ4 { ); version = addAndGetVersion(sdoc("id","2"), null); - long version2 = deleteByQueryAndGetVersion("id:2", null); + version2 = deleteByQueryAndGetVersion("id:2", null); assertTrue(Math.abs(version2) > version ); - + + // test that it's deleted + assertJQ(req("qt","/get","id","2") + ,"=={'doc':null}"); + + + version2 = Math.abs(version2) + 1000; + updateJ(jsonAdd(sdoc("id","3", "_version_",Long.toString(version2+100))), params(DISTRIB_UPDATE_PARAM,FROM_LEADER)); + updateJ(jsonAdd(sdoc("id","4", "_version_",Long.toString(version2+200))), params(DISTRIB_UPDATE_PARAM,FROM_LEADER)); + + // this should only affect id:3 so far + deleteByQueryAndGetVersion("id:(3 4 5 6)", params(DISTRIB_UPDATE_PARAM,FROM_LEADER, "_version_",Long.toString(-(version2+150))) ); + + assertJQ(req("qt","/get","id","3"),"=={'doc':null}"); + assertJQ(req("qt","/get","id","4", "fl","id"),"=={'doc':{'id':'4'}}"); + + updateJ(jsonAdd(sdoc("id","5", "_version_",Long.toString(version2+201))), params(DISTRIB_UPDATE_PARAM,FROM_LEADER)); + updateJ(jsonAdd(sdoc("id","6", "_version_",Long.toString(version2+101))), params(DISTRIB_UPDATE_PARAM,FROM_LEADER)); + + // the DBQ should also have caused id:6 to be removed + assertJQ(req("qt","/get","id","5", "fl","id"),"=={'doc':{'id':'5'}}"); + assertJQ(req("qt","/get","id","6"),"=={'doc':null}"); + + assertU(commit()); + } @Test @@ -390,59 +398,6 @@ public class TestRealTimeGet extends SolrTestCaseJ4 { ***/ - final ConcurrentHashMap model = new ConcurrentHashMap(); - Map committedModel = new HashMap(); - long snapshotCount; - long committedModelClock; - volatile int lastId; - final String field = "val_l"; - Object[] syncArr; - - private void initModel(int ndocs) { - snapshotCount = 0; - committedModelClock = 0; - lastId = 0; - - syncArr = new Object[ndocs]; - - for (int i=0; i 0) { - // return a random number not equal to version - for (;;) { - long badVersion = rand.nextInt(); - if (badVersion != version && badVersion != 0) return badVersion; - } - } - - // if the version does not exist, then we can only specify a positive version - for (;;) { - long badVersion = rand.nextInt() & 0x7fffffff; // mask off sign bit - if (badVersion != 0) return badVersion; - } - } - @Test public void testStressGetRealtime() throws Exception { clearIndex(); @@ -731,1184 +686,4 @@ public class TestRealTimeGet extends SolrTestCaseJ4 { } - // This version doesn't synchronize on id to tell what update won, but instead uses versions - @Test - public void testStressGetRealtimeVersions() throws Exception { - clearIndex(); - assertU(commit()); - - final int commitPercent = 5 + random().nextInt(20); - final int softCommitPercent = 30+random().nextInt(75); // what percent of the commits are soft - final int deletePercent = 4+random().nextInt(25); - final int deleteByQueryPercent = 1 + random().nextInt(5); - final int optimisticPercent = 1+random().nextInt(50); // percent change that an update uses optimistic locking - final int optimisticCorrectPercent = 25+random().nextInt(70); // percent change that a version specified will be correct - final int ndocs = 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200)); - int nWriteThreads = 5 + random().nextInt(25); - - final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers - - // query variables - final int percentRealtimeQuery = 75; - final AtomicLong operations = new AtomicLong(50000); // number of query operations to perform in total - int nReadThreads = 5 + random().nextInt(25); - - - - initModel(ndocs); - - final AtomicInteger numCommitting = new AtomicInteger(); - - List threads = new ArrayList(); - - for (int i=0; i 0) { - int oper = rand.nextInt(100); - - if (oper < commitPercent) { - if (numCommitting.incrementAndGet() <= maxConcurrentCommits) { - Map newCommittedModel; - long version; - - synchronized(TestRealTimeGet.this) { - newCommittedModel = new HashMap(model); // take a snapshot - version = snapshotCount++; - } - - if (rand.nextInt(100) < softCommitPercent) { - verbose("softCommit start"); - assertU(TestHarness.commit("softCommit","true")); - verbose("softCommit end"); - } else { - verbose("hardCommit start"); - assertU(commit()); - verbose("hardCommit end"); - } - - synchronized(TestRealTimeGet.this) { - // install this model snapshot only if it's newer than the current one - if (version >= committedModelClock) { - if (VERBOSE) { - verbose("installing new committedModel version="+committedModelClock); - } - committedModel = newCommittedModel; - committedModelClock = version; - } - } - } - numCommitting.decrementAndGet(); - continue; - } - - - int id = rand.nextInt(ndocs); - Object sync = syncArr[id]; - - // set the lastId before we actually change it sometimes to try and - // uncover more race conditions between writing and reading - boolean before = rand.nextBoolean(); - if (before) { - lastId = id; - } - - // We can't concurrently update the same document and retain our invariants of increasing values - // since we can't guarantee what order the updates will be executed. - // Even with versions, we can't remove the sync because increasing versions does not mean increasing vals. - // - // NOTE: versioning means we can now remove the sync and tell what update "won" - // synchronized (sync) { - DocInfo info = model.get(id); - - long val = info.val; - long nextVal = Math.abs(val)+1; - - if (oper < commitPercent + deletePercent) { - verbose("deleting id",id,"val=",nextVal); - - Long version = deleteAndGetVersion(Integer.toString(id), null); - assertTrue(version < 0); - - // only update model if the version is newer - synchronized (model) { - DocInfo currInfo = model.get(id); - if (Math.abs(version) > Math.abs(currInfo.version)) { - model.put(id, new DocInfo(version, -nextVal)); - } - } - - verbose("deleting id", id, "val=",nextVal,"DONE"); - } else if (oper < commitPercent + deletePercent + deleteByQueryPercent) { - verbose("deleteByQyery id",id,"val=",nextVal); - - Long version = deleteByQueryAndGetVersion("id:"+Integer.toString(id), null); - assertTrue(version < 0); - - // only update model if the version is newer - synchronized (model) { - DocInfo currInfo = model.get(id); - if (Math.abs(version) > Math.abs(currInfo.version)) { - model.put(id, new DocInfo(version, -nextVal)); - } - } - - verbose("deleteByQyery id", id, "val=",nextVal,"DONE"); - } else { - verbose("adding id", id, "val=", nextVal); - - // assertU(adoc("id",Integer.toString(id), field, Long.toString(nextVal))); - Long version = addAndGetVersion(sdoc("id", Integer.toString(id), field, Long.toString(nextVal)), null); - assertTrue(version > 0); - - // only update model if the version is newer - synchronized (model) { - DocInfo currInfo = model.get(id); - if (version > currInfo.version) { - model.put(id, new DocInfo(version, nextVal)); - } - } - - if (VERBOSE) { - verbose("adding id", id, "val=", nextVal,"DONE"); - } - - } - // } // end sync - - if (!before) { - lastId = id; - } - } - } catch (Throwable e) { - operations.set(-1L); - throw new RuntimeException(e); - } - } - }; - - threads.add(thread); - } - - - for (int i=0; i= 0) { - // bias toward a recently changed doc - int id = rand.nextInt(100) < 25 ? lastId : rand.nextInt(ndocs); - - // when indexing, we update the index, then the model - // so when querying, we should first check the model, and then the index - - boolean realTime = rand.nextInt(100) < percentRealtimeQuery; - DocInfo info; - - if (realTime) { - info = model.get(id); - } else { - synchronized(TestRealTimeGet.this) { - info = committedModel.get(id); - } - } - - if (VERBOSE) { - verbose("querying id", id); - } - SolrQueryRequest sreq; - if (realTime) { - sreq = req("wt","json", "qt","/get", "ids",Integer.toString(id)); - } else { - sreq = req("wt","json", "q","id:"+Integer.toString(id), "omitHeader","true"); - } - - String response = h.query(sreq); - Map rsp = (Map)ObjectBuilder.fromJSON(response); - List doclist = (List)(((Map)rsp.get("response")).get("docs")); - if (doclist.size() == 0) { - // there's no info we can get back with a delete, so not much we can check without further synchronization - } else { - assertEquals(1, doclist.size()); - long foundVal = (Long)(((Map)doclist.get(0)).get(field)); - long foundVer = (Long)(((Map)doclist.get(0)).get("_version_")); - if (foundVer < Math.abs(info.version) - || (foundVer == info.version && foundVal != info.val) ) { // if the version matches, the val must - verbose("ERROR, id=", id, "found=",response,"model",info); - assertTrue(false); - } - } - } - } catch (Throwable e) { - operations.set(-1L); - throw new RuntimeException(e); - } - } - }; - - threads.add(thread); - } - - - for (Thread thread : threads) { - thread.start(); - } - - for (Thread thread : threads) { - thread.join(); - } - - } - - // This version simulates updates coming from the leader and sometimes being reordered - @Test - public void testStressReorderVersions() throws Exception { - clearIndex(); - assertU(commit()); - - final int commitPercent = 5 + random().nextInt(20); - final int softCommitPercent = 30+random().nextInt(75); // what percent of the commits are soft - final int deletePercent = 4+random().nextInt(25); - final int deleteByQueryPercent = 0; // delete-by-query can't be reordered on replicas - final int ndocs = 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200)); - int nWriteThreads = 5 + random().nextInt(25); - - final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers - - // query variables - final int percentRealtimeQuery = 75; - final AtomicLong operations = new AtomicLong(50000); // number of query operations to perform in total - int nReadThreads = 5 + random().nextInt(25); - - initModel(ndocs); - - final AtomicInteger numCommitting = new AtomicInteger(); - - List threads = new ArrayList(); - - - final AtomicLong testVersion = new AtomicLong(0); - - for (int i=0; i 0) { - int oper = rand.nextInt(100); - - if (oper < commitPercent) { - if (numCommitting.incrementAndGet() <= maxConcurrentCommits) { - Map newCommittedModel; - long version; - - synchronized(TestRealTimeGet.this) { - newCommittedModel = new HashMap(model); // take a snapshot - version = snapshotCount++; - } - - if (rand.nextInt(100) < softCommitPercent) { - verbose("softCommit start"); - assertU(TestHarness.commit("softCommit","true")); - verbose("softCommit end"); - } else { - verbose("hardCommit start"); - assertU(commit()); - verbose("hardCommit end"); - } - - synchronized(TestRealTimeGet.this) { - // install this model snapshot only if it's newer than the current one - if (version >= committedModelClock) { - if (VERBOSE) { - verbose("installing new committedModel version="+committedModelClock); - } - committedModel = newCommittedModel; - committedModelClock = version; - } - } - } - numCommitting.decrementAndGet(); - continue; - } - - - int id; - - if (rand.nextBoolean()) { - id = rand.nextInt(ndocs); - } else { - id = lastId; // reuse the last ID half of the time to force more race conditions - } - - // set the lastId before we actually change it sometimes to try and - // uncover more race conditions between writing and reading - boolean before = rand.nextBoolean(); - if (before) { - lastId = id; - } - - DocInfo info = model.get(id); - - long val = info.val; - long nextVal = Math.abs(val)+1; - - // the version we set on the update should determine who wins - // These versions are not derived from the actual leader update handler hand hence this - // test may need to change depending on how we handle version numbers. - long version = testVersion.incrementAndGet(); - - // yield after getting the next version to increase the odds of updates happening out of order - if (rand.nextBoolean()) Thread.yield(); - - if (oper < commitPercent + deletePercent) { - verbose("deleting id",id,"val=",nextVal,"version",version); - - Long returnedVersion = deleteAndGetVersion(Integer.toString(id), params("_version_",Long.toString(-version), DISTRIB_UPDATE_PARAM,FROM_LEADER)); - - // TODO: returning versions for these types of updates is redundant - // but if we do return, they had better be equal - if (returnedVersion != null) { - assertEquals(-version, returnedVersion.longValue()); - } - - // only update model if the version is newer - synchronized (model) { - DocInfo currInfo = model.get(id); - if (Math.abs(version) > Math.abs(currInfo.version)) { - model.put(id, new DocInfo(version, -nextVal)); - } - } - - verbose("deleting id", id, "val=",nextVal,"version",version,"DONE"); - } else if (oper < commitPercent + deletePercent + deleteByQueryPercent) { - - } else { - verbose("adding id", id, "val=", nextVal,"version",version); - - Long returnedVersion = addAndGetVersion(sdoc("id", Integer.toString(id), field, Long.toString(nextVal), "_version_",Long.toString(version)), params(DISTRIB_UPDATE_PARAM,FROM_LEADER)); - if (returnedVersion != null) { - assertEquals(version, returnedVersion.longValue()); - } - - // only update model if the version is newer - synchronized (model) { - DocInfo currInfo = model.get(id); - if (version > currInfo.version) { - model.put(id, new DocInfo(version, nextVal)); - } - } - - if (VERBOSE) { - verbose("adding id", id, "val=", nextVal,"version",version,"DONE"); - } - - } - // } // end sync - - if (!before) { - lastId = id; - } - } - } catch (Throwable e) { - operations.set(-1L); - throw new RuntimeException(e); - } - } - }; - - threads.add(thread); - } - - - for (int i=0; i= 0) { - // bias toward a recently changed doc - int id = rand.nextInt(100) < 25 ? lastId : rand.nextInt(ndocs); - - // when indexing, we update the index, then the model - // so when querying, we should first check the model, and then the index - - boolean realTime = rand.nextInt(100) < percentRealtimeQuery; - DocInfo info; - - if (realTime) { - info = model.get(id); - } else { - synchronized(TestRealTimeGet.this) { - info = committedModel.get(id); - } - } - - if (VERBOSE) { - verbose("querying id", id); - } - SolrQueryRequest sreq; - if (realTime) { - sreq = req("wt","json", "qt","/get", "ids",Integer.toString(id)); - } else { - sreq = req("wt","json", "q","id:"+Integer.toString(id), "omitHeader","true"); - } - - String response = h.query(sreq); - Map rsp = (Map)ObjectBuilder.fromJSON(response); - List doclist = (List)(((Map)rsp.get("response")).get("docs")); - if (doclist.size() == 0) { - // there's no info we can get back with a delete, so not much we can check without further synchronization - } else { - assertEquals(1, doclist.size()); - long foundVal = (Long)(((Map)doclist.get(0)).get(field)); - long foundVer = (Long)(((Map)doclist.get(0)).get("_version_")); - if (foundVer < Math.abs(info.version) - || (foundVer == info.version && foundVal != info.val) ) { // if the version matches, the val must - verbose("ERROR, id=", id, "found=",response,"model",info); - assertTrue(false); - } - } - } - } catch (Throwable e) { - operations.set(-1L); - throw new RuntimeException(e); - } - } - }; - - threads.add(thread); - } - - - for (Thread thread : threads) { - thread.start(); - } - - for (Thread thread : threads) { - thread.join(); - } - - } - - - - - - - - // This points to the live model when state is ACTIVE, but a snapshot of the - // past when recovering. - volatile ConcurrentHashMap visibleModel; - - // This version simulates updates coming from the leader and sometimes being reordered - // and tests the ability to buffer updates and apply them later - @Test - public void testStressRecovery() throws Exception { - assumeFalse("FIXME: This test is horribly slow sometimes on Windows!", Constants.WINDOWS); - clearIndex(); - assertU(commit()); - - final int commitPercent = 5 + random().nextInt(10); - final int softCommitPercent = 30+random().nextInt(75); // what percent of the commits are soft - final int deletePercent = 4+random().nextInt(25); - final int deleteByQueryPercent = 0; // real-time get isn't currently supported with delete-by-query - final int ndocs = 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200)); - int nWriteThreads = 2 + random().nextInt(10); // fewer write threads to give recovery thread more of a chance - - final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers - - // query variables - final int percentRealtimeQuery = 75; - final int percentGetLatestVersions = random().nextInt(4); - final AtomicLong operations = new AtomicLong(atLeast(75)); // number of recovery loops to perform - int nReadThreads = 2 + random().nextInt(10); // fewer read threads to give writers more of a chance - - initModel(ndocs); - - final AtomicInteger numCommitting = new AtomicInteger(); - - List threads = new ArrayList(); - - - final AtomicLong testVersion = new AtomicLong(0); - - - final UpdateHandler uHandler = h.getCore().getUpdateHandler(); - final UpdateLog uLog = uHandler.getUpdateLog(); - final VersionInfo vInfo = uLog.getVersionInfo(); - final Object stateChangeLock = new Object(); - this.visibleModel = model; - final Semaphore[] writePermissions = new Semaphore[nWriteThreads]; - for (int i=0; i 0) { - writePermission.acquire(); - - int oper = rand.nextInt(10); - - if (oper < commitPercent) { - if (numCommitting.incrementAndGet() <= maxConcurrentCommits) { - Map newCommittedModel; - long version; - - synchronized(TestRealTimeGet.this) { - newCommittedModel = new HashMap(model); // take a snapshot - version = snapshotCount++; - } - - synchronized (stateChangeLock) { - // These commits won't take affect if we are in recovery mode, - // so change the version to -1 so we won't update our model. - if (uLog.getState() != UpdateLog.State.ACTIVE) version = -1; - if (rand.nextInt(100) < softCommitPercent) { - verbose("softCommit start"); - assertU(TestHarness.commit("softCommit","true")); - verbose("softCommit end"); - } else { - verbose("hardCommit start"); - assertU(commit()); - verbose("hardCommit end"); - } - } - - synchronized(TestRealTimeGet.this) { - // install this model snapshot only if it's newer than the current one - // install this model only if we are not in recovery mode. - if (version >= committedModelClock) { - if (VERBOSE) { - verbose("installing new committedModel version="+committedModelClock); - } - committedModel = newCommittedModel; - committedModelClock = version; - } - } - } - numCommitting.decrementAndGet(); - continue; - } - - - int id; - - if (rand.nextBoolean()) { - id = rand.nextInt(ndocs); - } else { - id = lastId; // reuse the last ID half of the time to force more race conditions - } - - // set the lastId before we actually change it sometimes to try and - // uncover more race conditions between writing and reading - boolean before = rand.nextBoolean(); - if (before) { - lastId = id; - } - - DocInfo info = model.get(id); - - long val = info.val; - long nextVal = Math.abs(val)+1; - - // the version we set on the update should determine who wins - // These versions are not derived from the actual leader update handler hand hence this - // test may need to change depending on how we handle version numbers. - long version = testVersion.incrementAndGet(); - - // yield after getting the next version to increase the odds of updates happening out of order - if (rand.nextBoolean()) Thread.yield(); - - if (oper < commitPercent + deletePercent) { - verbose("deleting id",id,"val=",nextVal,"version",version); - - Long returnedVersion = deleteAndGetVersion(Integer.toString(id), params("_version_",Long.toString(-version), DISTRIB_UPDATE_PARAM,FROM_LEADER)); - - // TODO: returning versions for these types of updates is redundant - // but if we do return, they had better be equal - if (returnedVersion != null) { - assertEquals(-version, returnedVersion.longValue()); - } - - // only update model if the version is newer - synchronized (model) { - DocInfo currInfo = model.get(id); - if (Math.abs(version) > Math.abs(currInfo.version)) { - model.put(id, new DocInfo(version, -nextVal)); - } - } - - verbose("deleting id", id, "val=",nextVal,"version",version,"DONE"); - } else if (oper < commitPercent + deletePercent + deleteByQueryPercent) { - - } else { - verbose("adding id", id, "val=", nextVal,"version",version); - - Long returnedVersion = addAndGetVersion(sdoc("id", Integer.toString(id), field, Long.toString(nextVal), "_version_",Long.toString(version)), params(DISTRIB_UPDATE_PARAM,FROM_LEADER)); - if (returnedVersion != null) { - assertEquals(version, returnedVersion.longValue()); - } - - // only update model if the version is newer - synchronized (model) { - DocInfo currInfo = model.get(id); - if (version > currInfo.version) { - model.put(id, new DocInfo(version, nextVal)); - } - } - - if (VERBOSE) { - verbose("adding id", id, "val=", nextVal,"version",version,"DONE"); - } - - } - // } // end sync - - if (!before) { - lastId = id; - } - } - } catch (Throwable e) { - operations.set(-1L); - throw new RuntimeException(e); - } - } - }; - - threads.add(thread); - } - - - for (int i=0; i 0) { - // throttle reads (don't completely stop) - readPermission.tryAcquire(10, TimeUnit.MILLISECONDS); - - - // bias toward a recently changed doc - int id = rand.nextInt(100) < 25 ? lastId : rand.nextInt(ndocs); - - // when indexing, we update the index, then the model - // so when querying, we should first check the model, and then the index - - boolean realTime = rand.nextInt(100) < percentRealtimeQuery; - DocInfo info; - - if (realTime) { - info = visibleModel.get(id); - } else { - synchronized(TestRealTimeGet.this) { - info = committedModel.get(id); - } - } - - - if (VERBOSE) { - verbose("querying id", id); - } - SolrQueryRequest sreq; - if (realTime) { - sreq = req("wt","json", "qt","/get", "ids",Integer.toString(id)); - } else { - sreq = req("wt","json", "q","id:"+Integer.toString(id), "omitHeader","true"); - } - - String response = h.query(sreq); - Map rsp = (Map)ObjectBuilder.fromJSON(response); - List doclist = (List)(((Map)rsp.get("response")).get("docs")); - if (doclist.size() == 0) { - // there's no info we can get back with a delete, so not much we can check without further synchronization - } else { - assertEquals(1, doclist.size()); - long foundVal = (Long)(((Map)doclist.get(0)).get(field)); - long foundVer = (Long)(((Map)doclist.get(0)).get("_version_")); - if (foundVer < Math.abs(info.version) - || (foundVer == info.version && foundVal != info.val) ) { // if the version matches, the val must - verbose("ERROR, id=", id, "found=",response,"model",info); - assertTrue(false); - } - } - } - - - if (rand.nextInt(100) < percentGetLatestVersions) { - getLatestVersions(); - // TODO: some sort of validation that the latest version is >= to the latest version we added? - } - - } catch (Throwable e) { - operations.set(-1L); - throw new RuntimeException(e); - } - } - }; - - threads.add(thread); - } - - - for (Thread thread : threads) { - thread.start(); - } - - int bufferedAddsApplied = 0; - do { - assertTrue(uLog.getState() == UpdateLog.State.ACTIVE); - - // before we start buffering updates, we want to point - // visibleModel away from the live model. - - visibleModel = new ConcurrentHashMap(model); - - synchronized (stateChangeLock) { - uLog.bufferUpdates(); - } - - assertTrue(uLog.getState() == UpdateLog.State.BUFFERING); - - // sometimes wait for a second to allow time for writers to write something - if (random().nextBoolean()) Thread.sleep(random().nextInt(10)+1); - - Future recoveryInfoF = uLog.applyBufferedUpdates(); - if (recoveryInfoF != null) { - UpdateLog.RecoveryInfo recInfo = null; - - int writeThreadNumber = 0; - while (recInfo == null) { - try { - // wait a short period of time for recovery to complete (and to give a chance for more writers to concurrently add docs) - recInfo = recoveryInfoF.get(random().nextInt(100/nWriteThreads), TimeUnit.MILLISECONDS); - } catch (TimeoutException e) { - // idle one more write thread - verbose("Operation",operations.get(),"Draining permits for write thread",writeThreadNumber); - writePermissions[writeThreadNumber++].drainPermits(); - if (writeThreadNumber >= nWriteThreads) { - // if we hit the end, back up and give a few write permits - writeThreadNumber--; - writePermissions[writeThreadNumber].release(random().nextInt(2) + 1); - } - - // throttle readers so they don't steal too much CPU from the recovery thread - readPermission.drainPermits(); - } - } - - bufferedAddsApplied += recInfo.adds; - } - - // put all writers back at full blast - for (Semaphore writePerm : writePermissions) { - // I don't think semaphores check for overflow, so we need to check mow many remain - int neededPermits = Integer.MAX_VALUE - writePerm.availablePermits(); - if (neededPermits > 0) writePerm.release( neededPermits ); - } - - // put back readers at full blast and point back to live model - visibleModel = model; - int neededPermits = Integer.MAX_VALUE - readPermission.availablePermits(); - if (neededPermits > 0) readPermission.release( neededPermits ); - - verbose("ROUND=",operations.get()); - } while (operations.decrementAndGet() > 0); - - verbose("bufferedAddsApplied=",bufferedAddsApplied); - - for (Thread thread : threads) { - thread.join(); - } - - } - - - List getLatestVersions() { - List recentVersions; - UpdateLog.RecentUpdates startingRecentUpdates = h.getCore().getUpdateHandler().getUpdateLog().getRecentUpdates(); - try { - recentVersions = startingRecentUpdates.getVersions(100); - } finally { - startingRecentUpdates.close(); - } - return recentVersions; - } - - - - - - - // The purpose of this test is to roughly model how solr uses lucene - DirectoryReader reader; - @Test - public void testStressLuceneNRT() throws Exception { - final int commitPercent = 5 + random().nextInt(20); - final int softCommitPercent = 30+random().nextInt(75); // what percent of the commits are soft - final int deletePercent = 4+random().nextInt(25); - final int deleteByQueryPercent = 1+random().nextInt(5); - final int ndocs = 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200)); - int nWriteThreads = 5 + random().nextInt(25); - - final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers - - final AtomicLong operations = new AtomicLong(1000); // number of query operations to perform in total - crank up if - int nReadThreads = 5 + random().nextInt(25); - final boolean tombstones = random().nextBoolean(); - final boolean syncCommits = random().nextBoolean(); - - verbose("commitPercent=", commitPercent); - verbose("softCommitPercent=",softCommitPercent); - verbose("deletePercent=",deletePercent); - verbose("deleteByQueryPercent=", deleteByQueryPercent); - verbose("ndocs=", ndocs); - verbose("nWriteThreads=", nWriteThreads); - verbose("nReadThreads=", nReadThreads); - verbose("maxConcurrentCommits=", maxConcurrentCommits); - verbose("operations=", operations); - verbose("tombstones=", tombstones); - verbose("syncCommits=", syncCommits); - - initModel(ndocs); - - final AtomicInteger numCommitting = new AtomicInteger(); - - List threads = new ArrayList(); - - - final FieldType idFt = new FieldType(); - idFt.setIndexed(true); - idFt.setStored(true); - idFt.setOmitNorms(true); - idFt.setTokenized(false); - idFt.setIndexOptions(FieldInfo.IndexOptions.DOCS_ONLY); - - final FieldType ft2 = new FieldType(); - ft2.setIndexed(false); - ft2.setStored(true); - - - // model how solr does locking - only allow one thread to do a hard commit at once, and only one thread to do a soft commit, but - // a hard commit in progress does not stop a soft commit. - final Lock hardCommitLock = syncCommits ? new ReentrantLock() : null; - final Lock reopenLock = syncCommits ? new ReentrantLock() : null; - - - // RAMDirectory dir = new RAMDirectory(); - // final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_40, new WhitespaceAnalyzer(Version.LUCENE_40))); - - Directory dir = newDirectory(); - - final RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); - writer.setDoRandomForceMergeAssert(false); - - // writer.commit(); - // reader = IndexReader.open(dir); - // make this reader an NRT reader from the start to avoid the first non-writer openIfChanged - // to only opening at the last commit point. - reader = DirectoryReader.open(writer.w, true); - - for (int i=0; i 0) { - int oper = rand.nextInt(100); - - if (oper < commitPercent) { - if (numCommitting.incrementAndGet() <= maxConcurrentCommits) { - Map newCommittedModel; - long version; - DirectoryReader oldReader; - - boolean softCommit = rand.nextInt(100) < softCommitPercent; - - if (!softCommit) { - // only allow one hard commit to proceed at once - if (hardCommitLock != null) hardCommitLock.lock(); - verbose("hardCommit start"); - - writer.commit(); - } - - if (reopenLock != null) reopenLock.lock(); - - synchronized(TestRealTimeGet.this) { - newCommittedModel = new HashMap(model); // take a snapshot - version = snapshotCount++; - oldReader = reader; - oldReader.incRef(); // increment the reference since we will use this for reopening - } - - if (!softCommit) { - // must commit after taking a snapshot of the model - // writer.commit(); - } - - verbose("reopen start using", oldReader); - - DirectoryReader newReader; - if (softCommit) { - newReader = DirectoryReader.openIfChanged(oldReader, writer.w, true); - } else { - // will only open to last commit - newReader = DirectoryReader.openIfChanged(oldReader); - } - - - if (newReader == null) { - oldReader.incRef(); - newReader = oldReader; - } - oldReader.decRef(); - - verbose("reopen result", newReader); - - synchronized(TestRealTimeGet.this) { - assert newReader.getRefCount() > 0; - assert reader.getRefCount() > 0; - - // install the new reader if it's newest (and check the current version since another reader may have already been installed) - if (newReader.getVersion() > reader.getVersion()) { - reader.decRef(); - reader = newReader; - - // install this snapshot only if it's newer than the current one - if (version >= committedModelClock) { - committedModel = newCommittedModel; - committedModelClock = version; - } - - } else { - // close if unused - newReader.decRef(); - } - - } - - if (reopenLock != null) reopenLock.unlock(); - - if (!softCommit) { - if (hardCommitLock != null) hardCommitLock.unlock(); - } - - } - numCommitting.decrementAndGet(); - continue; - } - - - int id = rand.nextInt(ndocs); - Object sync = syncArr[id]; - - // set the lastId before we actually change it sometimes to try and - // uncover more race conditions between writing and reading - boolean before = rand.nextBoolean(); - if (before) { - lastId = id; - } - - // We can't concurrently update the same document and retain our invariants of increasing values - // since we can't guarantee what order the updates will be executed. - synchronized (sync) { - DocInfo info = model.get(id); - long val = info.val; - long nextVal = Math.abs(val)+1; - - if (oper < commitPercent + deletePercent) { - // add tombstone first - if (tombstones) { - Document d = new Document(); - d.add(new Field("id","-"+Integer.toString(id), idFt)); - d.add(new Field(field, Long.toString(nextVal), ft2)); - verbose("adding tombstone for id",id,"val=",nextVal); - writer.updateDocument(new Term("id", "-"+Integer.toString(id)), d); - } - - verbose("deleting id",id,"val=",nextVal); - writer.deleteDocuments(new Term("id",Integer.toString(id))); - model.put(id, new DocInfo(0,-nextVal)); - verbose("deleting id",id,"val=",nextVal,"DONE"); - - } else if (oper < commitPercent + deletePercent + deleteByQueryPercent) { - //assertU("id:" + id + ""); - - // add tombstone first - if (tombstones) { - Document d = new Document(); - d.add(new Field("id","-"+Integer.toString(id), idFt)); - d.add(new Field(field, Long.toString(nextVal), ft2)); - verbose("adding tombstone for id",id,"val=",nextVal); - writer.updateDocument(new Term("id", "-"+Integer.toString(id)), d); - } - - verbose("deleteByQuery",id,"val=",nextVal); - writer.deleteDocuments(new TermQuery(new Term("id", Integer.toString(id)))); - model.put(id, new DocInfo(0,-nextVal)); - verbose("deleteByQuery",id,"val=",nextVal,"DONE"); - } else { - // model.put(id, nextVal); // uncomment this and this test should fail. - - // assertU(adoc("id",Integer.toString(id), field, Long.toString(nextVal))); - Document d = new Document(); - d.add(new Field("id",Integer.toString(id), idFt)); - d.add(new Field(field, Long.toString(nextVal), ft2)); - verbose("adding id",id,"val=",nextVal); - writer.updateDocument(new Term("id", Integer.toString(id)), d); - if (tombstones) { - // remove tombstone after new addition (this should be optional?) - verbose("deleting tombstone for id",id); - writer.deleteDocuments(new Term("id","-"+Integer.toString(id))); - verbose("deleting tombstone for id",id,"DONE"); - } - - model.put(id, new DocInfo(0,nextVal)); - verbose("adding id",id,"val=",nextVal,"DONE"); - } - } - - if (!before) { - lastId = id; - } - } - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } - }; - - threads.add(thread); - } - - - for (int i=0; i= 0) { - // bias toward a recently changed doc - int id = rand.nextInt(100) < 25 ? lastId : rand.nextInt(ndocs); - - // when indexing, we update the index, then the model - // so when querying, we should first check the model, and then the index - - DocInfo info; - synchronized(TestRealTimeGet.this) { - info = committedModel.get(id); - } - long val = info.val; - - IndexReader r; - synchronized(TestRealTimeGet.this) { - r = reader; - r.incRef(); - } - - int docid = getFirstMatch(r, new Term("id",Integer.toString(id))); - - if (docid < 0 && tombstones) { - // if we couldn't find the doc, look for it's tombstone - docid = getFirstMatch(r, new Term("id","-"+Integer.toString(id))); - if (docid < 0) { - if (val == -1L) { - // expected... no doc was added yet - r.decRef(); - continue; - } - verbose("ERROR: Couldn't find a doc or tombstone for id", id, "using reader",r,"expected value",val); - fail("No documents or tombstones found for id " + id + ", expected at least " + val); - } - } - - if (docid < 0 && !tombstones) { - // nothing to do - we can't tell anything from a deleted doc without tombstones - } else { - if (docid < 0) { - verbose("ERROR: Couldn't find a doc for id", id, "using reader",r); - } - assertTrue(docid >= 0); // we should have found the document, or it's tombstone - Document doc = r.document(docid); - long foundVal = Long.parseLong(doc.get(field)); - if (foundVal < Math.abs(val)) { - verbose("ERROR: id",id,"model_val=",val," foundVal=",foundVal,"reader=",reader); - } - assertTrue(foundVal >= Math.abs(val)); - } - - r.decRef(); - } - } catch (Throwable e) { - operations.set(-1L); - throw new RuntimeException(e); - } - } - }; - - threads.add(thread); - } - - - for (Thread thread : threads) { - thread.start(); - } - - for (Thread thread : threads) { - thread.join(); - } - - writer.close(); - reader.close(); - dir.close(); - } - - - public int getFirstMatch(IndexReader r, Term t) throws IOException { - Fields fields = MultiFields.getFields(r); - if (fields == null) return -1; - Terms terms = fields.terms(t.field()); - if (terms == null) return -1; - BytesRef termBytes = t.bytes(); - final TermsEnum termsEnum = terms.iterator(null); - if (!termsEnum.seekExact(termBytes, false)) { - return -1; - } - DocsEnum docs = termsEnum.docs(MultiFields.getLiveDocs(r), null, false); - int id = docs.nextDoc(); - if (id != DocIdSetIterator.NO_MORE_DOCS) { - int next = docs.nextDoc(); - assertEquals(DocIdSetIterator.NO_MORE_DOCS, next); - } - return id == DocIdSetIterator.NO_MORE_DOCS ? -1 : id; - } - } diff --git a/solr/core/src/test/org/apache/solr/search/TestRecovery.java b/solr/core/src/test/org/apache/solr/search/TestRecovery.java index 0b54fbf..94a2fa8 100644 --- a/solr/core/src/test/org/apache/solr/search/TestRecovery.java +++ b/solr/core/src/test/org/apache/solr/search/TestRecovery.java @@ -17,24 +17,25 @@ package org.apache.solr.search; -import org.apache.lucene.util.BytesRef; -import org.apache.noggit.JSONUtil; import org.apache.noggit.ObjectBuilder; import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.common.util.ByteUtils; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.update.DirectUpdateHandler2; import org.apache.solr.update.UpdateLog; import org.apache.solr.update.UpdateHandler; -import org.apache.solr.update.UpdateLog; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import java.io.File; import java.io.RandomAccessFile; -import java.util.*; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Deque; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.concurrent.Future; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; @@ -60,7 +61,7 @@ public class TestRecovery extends SolrTestCaseJ4 { } @AfterClass - public static void afterClass() throws Exception { + public static void afterClass() { if (savedFactory == null) { System.clearProperty("solr.directoryFactory"); } else { @@ -68,6 +69,19 @@ public class TestRecovery extends SolrTestCaseJ4 { } } + + // since we make up fake versions in these tests, we can get messed up by a DBQ with a real version + // since Solr can think following updates were reordered. + @Override + public void clearIndex() { + try { + deleteByQueryAndGetVersion("*:*", params("_version_", Long.toString(-Long.MAX_VALUE), DISTRIB_UPDATE_PARAM,FROM_LEADER)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Test public void testLogReplay() throws Exception { try { @@ -892,7 +906,7 @@ public class TestRecovery extends SolrTestCaseJ4 { // WARNING... assumes format of .00000n where n is less than 9 long logNumber = Long.parseLong(fname.substring(fname.lastIndexOf(".") + 1)); - String fname2 = String.format(Locale.ENGLISH, + String fname2 = String.format(Locale.ENGLISH, UpdateLog.LOG_FILENAME_PATTERN, UpdateLog.TLOG_NAME, logNumber + 1); diff --git a/solr/core/src/test/org/apache/solr/search/TestSearchPerf.java b/solr/core/src/test/org/apache/solr/search/TestSearchPerf.java index b5c4136..ddbde88 100755 --- a/solr/core/src/test/org/apache/solr/search/TestSearchPerf.java +++ b/solr/core/src/test/org/apache/solr/search/TestSearchPerf.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java b/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java index e897e27..f65333a 100644 --- a/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java +++ b/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/TestSort.java b/solr/core/src/test/org/apache/solr/search/TestSort.java index 75416d3..135eced 100755 --- a/solr/core/src/test/org/apache/solr/search/TestSort.java +++ b/solr/core/src/test/org/apache/solr/search/TestSort.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -74,29 +74,37 @@ public class TestSort extends SolrTestCaseJ4 { SolrQueryRequest req = lrf.makeRequest("q", "*:*"); final int iters = atLeast(5000); - int numberOfOddities = 0; + + // infinite loop abort when trying to generate a non-blank sort "name" + final int nonBlankAttempts = 37; for (int i = 0; i < iters; i++) { final StringBuilder input = new StringBuilder(); final String[] names = new String[_TestUtil.nextInt(r,1,10)]; final boolean[] reverse = new boolean[names.length]; for (int j = 0; j < names.length; j++) { - names[j] = _TestUtil.randomRealisticUnicodeString(r, 1, 20); - - // reduce the likelyhood that the random str is a valid query or func - names[j] = names[j].replaceFirst("\\{","\\{\\{"); - names[j] = names[j].replaceFirst("\\(","\\(\\("); - names[j] = names[j].replaceFirst("(\\\"|\\')","$1$1"); - names[j] = names[j].replaceFirst("(\\d)","$1x"); - - // eliminate pesky problem chars - names[j] = names[j].replaceAll("\\p{Cntrl}|\\p{javaWhitespace}",""); - - if (0 == names[j].length()) { - numberOfOddities++; - // screw it, i'm taking my toys and going home - names[j] = "last_ditch_i_give_up"; + names[j] = null; + for (int k = 0; k < nonBlankAttempts && null == names[j]; k++) { + names[j] = _TestUtil.randomRealisticUnicodeString(r, 1, 100); + + // munge anything that might make this a function + names[j] = names[j].replaceFirst("\\{","\\{\\{"); + names[j] = names[j].replaceFirst("\\(","\\(\\("); + names[j] = names[j].replaceFirst("(\\\"|\\')","$1$1z"); + names[j] = names[j].replaceFirst("(\\d)","$1x"); + + // eliminate pesky problem chars + names[j] = names[j].replaceAll("\\p{Cntrl}|\\p{javaWhitespace}",""); + + if (0 == names[j].length()) { + names[j] = null; + } } + // with luck this bad, never go to vegas + // alternatively: if (null == names[j]) names[j] = "never_go_to_vegas"; + assertNotNull("Unable to generate a (non-blank) names["+j+"] after " + + nonBlankAttempts + " attempts", names[j]); + reverse[j] = r.nextBoolean(); input.append(r.nextBoolean() ? " " : ""); @@ -120,46 +128,33 @@ public class TestSort extends SolrTestCaseJ4 { final Type type = sorts[j].getType(); if (Type.SCORE.equals(type)) { - numberOfOddities++; assertEquals("sorts["+j+"] is (unexpectedly) type score : " + input, "score", names[j]); } else if (Type.DOC.equals(type)) { - numberOfOddities++; assertEquals("sorts["+j+"] is (unexpectedly) type doc : " + input, "_docid_", names[j]); } else if (Type.CUSTOM.equals(type) || Type.REWRITEABLE.equals(type)) { - numberOfOddities++; - - // our orig string better be parsable as a func/query - QParser qp = - QParser.getParser(names[j], FunctionQParserPlugin.NAME, req); - try { - Query q = qp.getQuery(); - assertNotNull("sorts["+j+"] had type " + type + - " but parsed to null func/query: " + input, q); - } catch (Exception e) { - assertNull("sorts["+j+"] had type " + type + - " but errored parsing as func/query: " + input, e); - } + + fail("sorts["+j+"] resulted in a '" + type.toString() + + "', either sort parsing code is broken, or func/query " + + "semantics have gotten broader and munging in this test " + + "needs improved: " + input); + } else { - assertEquals("sorts["+j+"] had unexpected field: " + input, + assertEquals("sorts["+j+"] ("+type.toString()+ + ") had unexpected field in: " + input, names[j], sorts[j].getField()); } } } - - assertTrue("Over 0.2% oddities in test: " + - numberOfOddities + "/" + iters + - " have func/query parsing semenatics gotten broader?", - numberOfOddities < 0.002 * iters); } public void testSort() throws Exception { Directory dir = new RAMDirectory(); - Field f = new Field("f", "0", StringField.TYPE_UNSTORED); - Field f2 = new Field("f2", "0", StringField.TYPE_UNSTORED); + Field f = new StringField("f", "0", Field.Store.NO); + Field f2 = new StringField("f2", "0", Field.Store.NO); for (int iterCnt = 0; iterCnt 0) { + int oper = rand.nextInt(100); + + if (oper < commitPercent) { + if (numCommitting.incrementAndGet() <= maxConcurrentCommits) { + Map newCommittedModel; + long version; + DirectoryReader oldReader; + + boolean softCommit = rand.nextInt(100) < softCommitPercent; + + if (!softCommit) { + // only allow one hard commit to proceed at once + if (hardCommitLock != null) hardCommitLock.lock(); + verbose("hardCommit start"); + + writer.commit(); + } + + if (reopenLock != null) reopenLock.lock(); + + synchronized(globalLock) { + newCommittedModel = new HashMap(model); // take a snapshot + version = snapshotCount++; + oldReader = reader; + oldReader.incRef(); // increment the reference since we will use this for reopening + } + + if (!softCommit) { + // must commit after taking a snapshot of the model + // writer.commit(); + } + + verbose("reopen start using", oldReader); + + DirectoryReader newReader; + if (softCommit) { + newReader = DirectoryReader.openIfChanged(oldReader, writer.w, true); + } else { + // will only open to last commit + newReader = DirectoryReader.openIfChanged(oldReader); + } + + + if (newReader == null) { + oldReader.incRef(); + newReader = oldReader; + } + oldReader.decRef(); + + verbose("reopen result", newReader); + + synchronized(globalLock) { + assert newReader.getRefCount() > 0; + assert reader.getRefCount() > 0; + + // install the new reader if it's newest (and check the current version since another reader may have already been installed) + if (newReader.getVersion() > reader.getVersion()) { + reader.decRef(); + reader = newReader; + + // install this snapshot only if it's newer than the current one + if (version >= committedModelClock) { + committedModel = newCommittedModel; + committedModelClock = version; + } + + } else { + // close if unused + newReader.decRef(); + } + + } + + if (reopenLock != null) reopenLock.unlock(); + + if (!softCommit) { + if (hardCommitLock != null) hardCommitLock.unlock(); + } + + } + numCommitting.decrementAndGet(); + continue; + } + + + int id = rand.nextInt(ndocs); + Object sync = syncArr[id]; + + // set the lastId before we actually change it sometimes to try and + // uncover more race conditions between writing and reading + boolean before = rand.nextBoolean(); + if (before) { + lastId = id; + } + + // We can't concurrently update the same document and retain our invariants of increasing values + // since we can't guarantee what order the updates will be executed. + synchronized (sync) { + DocInfo info = model.get(id); + long val = info.val; + long nextVal = Math.abs(val)+1; + + if (oper < commitPercent + deletePercent) { + // add tombstone first + if (tombstones) { + Document d = new Document(); + d.add(new Field("id","-"+Integer.toString(id), idFt)); + d.add(new Field(field, Long.toString(nextVal), ft2)); + verbose("adding tombstone for id",id,"val=",nextVal); + writer.updateDocument(new Term("id", "-"+Integer.toString(id)), d); + } + + verbose("deleting id",id,"val=",nextVal); + writer.deleteDocuments(new Term("id",Integer.toString(id))); + model.put(id, new DocInfo(0,-nextVal)); + verbose("deleting id",id,"val=",nextVal,"DONE"); + + } else if (oper < commitPercent + deletePercent + deleteByQueryPercent) { + //assertU("id:" + id + ""); + + // add tombstone first + if (tombstones) { + Document d = new Document(); + d.add(new Field("id","-"+Integer.toString(id), idFt)); + d.add(new Field(field, Long.toString(nextVal), ft2)); + verbose("adding tombstone for id",id,"val=",nextVal); + writer.updateDocument(new Term("id", "-"+Integer.toString(id)), d); + } + + verbose("deleteByQuery",id,"val=",nextVal); + writer.deleteDocuments(new TermQuery(new Term("id", Integer.toString(id)))); + model.put(id, new DocInfo(0,-nextVal)); + verbose("deleteByQuery",id,"val=",nextVal,"DONE"); + } else { + // model.put(id, nextVal); // uncomment this and this test should fail. + + // assertU(adoc("id",Integer.toString(id), field, Long.toString(nextVal))); + Document d = new Document(); + d.add(new Field("id",Integer.toString(id), idFt)); + d.add(new Field(field, Long.toString(nextVal), ft2)); + verbose("adding id",id,"val=",nextVal); + writer.updateDocument(new Term("id", Integer.toString(id)), d); + if (tombstones) { + // remove tombstone after new addition (this should be optional?) + verbose("deleting tombstone for id",id); + writer.deleteDocuments(new Term("id","-"+Integer.toString(id))); + verbose("deleting tombstone for id",id,"DONE"); + } + + model.put(id, new DocInfo(0,nextVal)); + verbose("adding id",id,"val=",nextVal,"DONE"); + } + } + + if (!before) { + lastId = id; + } + } + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + }; + + threads.add(thread); + } + + + for (int i=0; i= 0) { + // bias toward a recently changed doc + int id = rand.nextInt(100) < 25 ? lastId : rand.nextInt(ndocs); + + // when indexing, we update the index, then the model + // so when querying, we should first check the model, and then the index + + DocInfo info; + synchronized(globalLock) { + info = committedModel.get(id); + } + long val = info.val; + + IndexReader r; + synchronized(globalLock) { + r = reader; + r.incRef(); + } + + int docid = getFirstMatch(r, new Term("id",Integer.toString(id))); + + if (docid < 0 && tombstones) { + // if we couldn't find the doc, look for it's tombstone + docid = getFirstMatch(r, new Term("id","-"+Integer.toString(id))); + if (docid < 0) { + if (val == -1L) { + // expected... no doc was added yet + r.decRef(); + continue; + } + verbose("ERROR: Couldn't find a doc or tombstone for id", id, "using reader",r,"expected value",val); + fail("No documents or tombstones found for id " + id + ", expected at least " + val); + } + } + + if (docid < 0 && !tombstones) { + // nothing to do - we can't tell anything from a deleted doc without tombstones + } else { + if (docid < 0) { + verbose("ERROR: Couldn't find a doc for id", id, "using reader",r); + } + assertTrue(docid >= 0); // we should have found the document, or it's tombstone + Document doc = r.document(docid); + long foundVal = Long.parseLong(doc.get(field)); + if (foundVal < Math.abs(val)) { + verbose("ERROR: id",id,"model_val=",val," foundVal=",foundVal,"reader=",reader); + } + assertTrue(foundVal >= Math.abs(val)); + } + + r.decRef(); + } + } catch (Throwable e) { + operations.set(-1L); + throw new RuntimeException(e); + } + } + }; + + threads.add(thread); + } + + + for (Thread thread : threads) { + thread.start(); + } + + for (Thread thread : threads) { + thread.join(); + } + + writer.close(); + reader.close(); + dir.close(); + } + + + +} diff --git a/solr/core/src/test/org/apache/solr/search/TestStressRecovery.java b/solr/core/src/test/org/apache/solr/search/TestStressRecovery.java new file mode 100644 index 0000000..e99223f --- /dev/null +++ b/solr/core/src/test/org/apache/solr/search/TestStressRecovery.java @@ -0,0 +1,412 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search; + + +import org.apache.lucene.util.Constants; +import org.apache.noggit.ObjectBuilder; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.update.UpdateHandler; +import org.apache.solr.update.UpdateLog; +import org.apache.solr.update.VersionInfo; +import org.apache.solr.util.TestHarness; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Future; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import static org.apache.solr.core.SolrCore.verbose; +import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM; + +public class TestStressRecovery extends TestRTGBase { + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig-tlog.xml","schema15.xml"); + } + + + // This points to the live model when state is ACTIVE, but a snapshot of the + // past when recovering. + volatile ConcurrentHashMap visibleModel; + + // This version simulates updates coming from the leader and sometimes being reordered + // and tests the ability to buffer updates and apply them later + @Test + public void testStressRecovery() throws Exception { + assumeFalse("FIXME: This test is horribly slow sometimes on Windows!", Constants.WINDOWS); + clearIndex(); + assertU(commit()); + + final int commitPercent = 5 + random().nextInt(10); + final int softCommitPercent = 30+random().nextInt(75); // what percent of the commits are soft + final int deletePercent = 4+random().nextInt(25); + final int deleteByQueryPercent = random().nextInt(5); + final int ndocs = 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200)); + int nWriteThreads = 2 + random().nextInt(10); // fewer write threads to give recovery thread more of a chance + + final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers + + // query variables + final int percentRealtimeQuery = 75; + final int percentGetLatestVersions = random().nextInt(4); + final AtomicLong operations = new AtomicLong(atLeast(100)); // number of recovery loops to perform + int nReadThreads = 2 + random().nextInt(10); // fewer read threads to give writers more of a chance + + initModel(ndocs); + + final AtomicInteger numCommitting = new AtomicInteger(); + + List threads = new ArrayList(); + + + final AtomicLong testVersion = new AtomicLong(0); + + + final UpdateHandler uHandler = h.getCore().getUpdateHandler(); + final UpdateLog uLog = uHandler.getUpdateLog(); + final VersionInfo vInfo = uLog.getVersionInfo(); + final Object stateChangeLock = new Object(); + this.visibleModel = model; + final Semaphore[] writePermissions = new Semaphore[nWriteThreads]; + for (int i=0; i 0) { + writePermission.acquire(); + + int oper = rand.nextInt(10); + + if (oper < commitPercent) { + if (numCommitting.incrementAndGet() <= maxConcurrentCommits) { + Map newCommittedModel; + long version; + + synchronized(globalLock) { + newCommittedModel = new HashMap(model); // take a snapshot + version = snapshotCount++; + } + + synchronized (stateChangeLock) { + // These commits won't take affect if we are in recovery mode, + // so change the version to -1 so we won't update our model. + if (uLog.getState() != UpdateLog.State.ACTIVE) version = -1; + if (rand.nextInt(100) < softCommitPercent) { + verbose("softCommit start"); + assertU(TestHarness.commit("softCommit","true")); + verbose("softCommit end"); + } else { + verbose("hardCommit start"); + assertU(commit()); + verbose("hardCommit end"); + } + } + + synchronized(globalLock) { + // install this model snapshot only if it's newer than the current one + // install this model only if we are not in recovery mode. + if (version >= committedModelClock) { + if (VERBOSE) { + verbose("installing new committedModel version="+committedModelClock); + } + committedModel = newCommittedModel; + committedModelClock = version; + } + } + } + numCommitting.decrementAndGet(); + continue; + } + + + int id; + + if (rand.nextBoolean()) { + id = rand.nextInt(ndocs); + } else { + id = lastId; // reuse the last ID half of the time to force more race conditions + } + + // set the lastId before we actually change it sometimes to try and + // uncover more race conditions between writing and reading + boolean before = rand.nextBoolean(); + if (before) { + lastId = id; + } + + DocInfo info = model.get(id); + + long val = info.val; + long nextVal = Math.abs(val)+1; + + // the version we set on the update should determine who wins + // These versions are not derived from the actual leader update handler hand hence this + // test may need to change depending on how we handle version numbers. + long version = testVersion.incrementAndGet(); + + // yield after getting the next version to increase the odds of updates happening out of order + if (rand.nextBoolean()) Thread.yield(); + + if (oper < commitPercent + deletePercent) { + verbose("deleting id",id,"val=",nextVal,"version",version); + + Long returnedVersion = deleteAndGetVersion(Integer.toString(id), params("_version_",Long.toString(-version), DISTRIB_UPDATE_PARAM,FROM_LEADER)); + + // TODO: returning versions for these types of updates is redundant + // but if we do return, they had better be equal + if (returnedVersion != null) { + assertEquals(-version, returnedVersion.longValue()); + } + + // only update model if the version is newer + synchronized (model) { + DocInfo currInfo = model.get(id); + if (Math.abs(version) > Math.abs(currInfo.version)) { + model.put(id, new DocInfo(version, -nextVal)); + } + } + + verbose("deleting id", id, "val=",nextVal,"version",version,"DONE"); + } else if (oper < commitPercent + deletePercent + deleteByQueryPercent) { + + verbose("deleteByQuery id",id,"val=",nextVal,"version",version); + + Long returnedVersion = deleteByQueryAndGetVersion("id:"+Integer.toString(id), params("_version_",Long.toString(-version), DISTRIB_UPDATE_PARAM,FROM_LEADER)); + + // TODO: returning versions for these types of updates is redundant + // but if we do return, they had better be equal + if (returnedVersion != null) { + assertEquals(-version, returnedVersion.longValue()); + } + + // only update model if the version is newer + synchronized (model) { + DocInfo currInfo = model.get(id); + if (Math.abs(version) > Math.abs(currInfo.version)) { + model.put(id, new DocInfo(version, -nextVal)); + } + } + + verbose("deleteByQuery id", id, "val=",nextVal,"version",version,"DONE"); + + } else { + verbose("adding id", id, "val=", nextVal,"version",version); + + Long returnedVersion = addAndGetVersion(sdoc("id", Integer.toString(id), field, Long.toString(nextVal), "_version_",Long.toString(version)), params(DISTRIB_UPDATE_PARAM,FROM_LEADER)); + if (returnedVersion != null) { + assertEquals(version, returnedVersion.longValue()); + } + + // only update model if the version is newer + synchronized (model) { + DocInfo currInfo = model.get(id); + if (version > currInfo.version) { + model.put(id, new DocInfo(version, nextVal)); + } + } + + if (VERBOSE) { + verbose("adding id", id, "val=", nextVal,"version",version,"DONE"); + } + + } + // } // end sync + + if (!before) { + lastId = id; + } + } + } catch (Throwable e) { + operations.set(-1L); + throw new RuntimeException(e); + } + } + }; + + threads.add(thread); + } + + + for (int i=0; i 0) { + // throttle reads (don't completely stop) + readPermission.tryAcquire(10, TimeUnit.MILLISECONDS); + + + // bias toward a recently changed doc + int id = rand.nextInt(100) < 25 ? lastId : rand.nextInt(ndocs); + + // when indexing, we update the index, then the model + // so when querying, we should first check the model, and then the index + + boolean realTime = rand.nextInt(100) < percentRealtimeQuery; + DocInfo info; + + if (realTime) { + info = visibleModel.get(id); + } else { + synchronized(globalLock) { + info = committedModel.get(id); + } + } + + + if (VERBOSE) { + verbose("querying id", id); + } + SolrQueryRequest sreq; + if (realTime) { + sreq = req("wt","json", "qt","/get", "ids",Integer.toString(id)); + } else { + sreq = req("wt","json", "q","id:"+Integer.toString(id), "omitHeader","true"); + } + + String response = h.query(sreq); + Map rsp = (Map)ObjectBuilder.fromJSON(response); + List doclist = (List)(((Map)rsp.get("response")).get("docs")); + if (doclist.size() == 0) { + // there's no info we can get back with a delete, so not much we can check without further synchronization + } else { + assertEquals(1, doclist.size()); + long foundVal = (Long)(((Map)doclist.get(0)).get(field)); + long foundVer = (Long)(((Map)doclist.get(0)).get("_version_")); + if (foundVer < Math.abs(info.version) + || (foundVer == info.version && foundVal != info.val) ) { // if the version matches, the val must + verbose("ERROR, id=", id, "found=",response,"model",info); + assertTrue(false); + } + } + } + + + if (rand.nextInt(100) < percentGetLatestVersions) { + getLatestVersions(); + // TODO: some sort of validation that the latest version is >= to the latest version we added? + } + + } catch (Throwable e) { + operations.set(-1L); + throw new RuntimeException(e); + } + } + }; + + threads.add(thread); + } + + + for (Thread thread : threads) { + thread.start(); + } + + int bufferedAddsApplied = 0; + do { + assertTrue(uLog.getState() == UpdateLog.State.ACTIVE); + + // before we start buffering updates, we want to point + // visibleModel away from the live model. + + visibleModel = new ConcurrentHashMap(model); + + synchronized (stateChangeLock) { + uLog.bufferUpdates(); + } + + assertTrue(uLog.getState() == UpdateLog.State.BUFFERING); + + // sometimes wait for a second to allow time for writers to write something + if (random().nextBoolean()) Thread.sleep(random().nextInt(10)+1); + + Future recoveryInfoF = uLog.applyBufferedUpdates(); + if (recoveryInfoF != null) { + UpdateLog.RecoveryInfo recInfo = null; + + int writeThreadNumber = 0; + while (recInfo == null) { + try { + // wait a short period of time for recovery to complete (and to give a chance for more writers to concurrently add docs) + recInfo = recoveryInfoF.get(random().nextInt(100/nWriteThreads), TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + // idle one more write thread + verbose("Operation",operations.get(),"Draining permits for write thread",writeThreadNumber); + writePermissions[writeThreadNumber++].drainPermits(); + if (writeThreadNumber >= nWriteThreads) { + // if we hit the end, back up and give a few write permits + writeThreadNumber--; + writePermissions[writeThreadNumber].release(random().nextInt(2) + 1); + } + + // throttle readers so they don't steal too much CPU from the recovery thread + readPermission.drainPermits(); + } + } + + bufferedAddsApplied += recInfo.adds; + } + + // put all writers back at full blast + for (Semaphore writePerm : writePermissions) { + // I don't think semaphores check for overflow, so we need to check mow many remain + int neededPermits = Integer.MAX_VALUE - writePerm.availablePermits(); + if (neededPermits > 0) writePerm.release( neededPermits ); + } + + // put back readers at full blast and point back to live model + visibleModel = model; + int neededPermits = Integer.MAX_VALUE - readPermission.availablePermits(); + if (neededPermits > 0) readPermission.release( neededPermits ); + + verbose("ROUND=",operations.get()); + } while (operations.decrementAndGet() > 0); + + verbose("bufferedAddsApplied=",bufferedAddsApplied); + + for (Thread thread : threads) { + thread.join(); + } + + } + +} diff --git a/solr/core/src/test/org/apache/solr/search/TestStressReorder.java b/solr/core/src/test/org/apache/solr/search/TestStressReorder.java new file mode 100644 index 0000000..775fcf0 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/search/TestStressReorder.java @@ -0,0 +1,335 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search; + + +import org.apache.noggit.ObjectBuilder; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.util.TestHarness; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM; + +public class TestStressReorder extends TestRTGBase { + public static Logger log = LoggerFactory.getLogger(TestStressReorder.class); + + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig-tlog.xml","schema15.xml"); + } + + public static void verbose(Object... args) { + // if (!log.isDebugEnabled()) return; + StringBuilder sb = new StringBuilder("VERBOSE:"); + for (Object o : args) { + sb.append(' '); + sb.append(o==null ? "(null)" : o.toString()); + } + log.info(sb.toString()); + } + + // This version simulates updates coming from the leader and sometimes being reordered + @Test + public void testStressReorderVersions() throws Exception { + clearIndex(); + assertU(commit()); + + final int commitPercent = 5 + random().nextInt(20); + final int softCommitPercent = 30+random().nextInt(75); // what percent of the commits are soft + final int deletePercent = 4+random().nextInt(25); + final int deleteByQueryPercent = random().nextInt(8); + final int ndocs = 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200)); + int nWriteThreads = 5 + random().nextInt(25); + + final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers + + // query variables + final int percentRealtimeQuery = 75; + final AtomicLong operations = new AtomicLong(50000); // number of query operations to perform in total + int nReadThreads = 5 + random().nextInt(25); + + + /** // testing + final int commitPercent = 5; + final int softCommitPercent = 100; // what percent of the commits are soft + final int deletePercent = 0; + final int deleteByQueryPercent = 50; + final int ndocs = 1; + int nWriteThreads = 2; + + final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers + + // query variables + final int percentRealtimeQuery = 101; + final AtomicLong operations = new AtomicLong(50000); // number of query operations to perform in total + int nReadThreads = 1; + **/ + + + verbose("commitPercent",commitPercent, "softCommitPercent",softCommitPercent, "deletePercent",deletePercent, "deleteByQueryPercent",deleteByQueryPercent + , "ndocs",ndocs,"nWriteThreads",nWriteThreads,"percentRealtimeQuery",percentRealtimeQuery,"operations",operations, "nReadThreads",nReadThreads); + + initModel(ndocs); + + final AtomicInteger numCommitting = new AtomicInteger(); + + List threads = new ArrayList(); + + + final AtomicLong testVersion = new AtomicLong(0); + + for (int i=0; i 0) { + int oper = rand.nextInt(100); + + if (oper < commitPercent) { + if (numCommitting.incrementAndGet() <= maxConcurrentCommits) { + Map newCommittedModel; + long version; + + synchronized(TestStressReorder.this) { + newCommittedModel = new HashMap(model); // take a snapshot + version = snapshotCount++; + } + + if (rand.nextInt(100) < softCommitPercent) { + verbose("softCommit start"); + assertU(TestHarness.commit("softCommit","true")); + verbose("softCommit end"); + } else { + verbose("hardCommit start"); + assertU(commit()); + verbose("hardCommit end"); + } + + synchronized(TestStressReorder.this) { + // install this model snapshot only if it's newer than the current one + if (version >= committedModelClock) { + if (VERBOSE) { + verbose("installing new committedModel version="+committedModelClock); + } + committedModel = newCommittedModel; + committedModelClock = version; + } + } + } + numCommitting.decrementAndGet(); + continue; + } + + + int id; + + if (rand.nextBoolean()) { + id = rand.nextInt(ndocs); + } else { + id = lastId; // reuse the last ID half of the time to force more race conditions + } + + // set the lastId before we actually change it sometimes to try and + // uncover more race conditions between writing and reading + boolean before = rand.nextBoolean(); + if (before) { + lastId = id; + } + + DocInfo info = model.get(id); + + long val = info.val; + long nextVal = Math.abs(val)+1; + + // the version we set on the update should determine who wins + // These versions are not derived from the actual leader update handler hand hence this + // test may need to change depending on how we handle version numbers. + long version = testVersion.incrementAndGet(); + + // yield after getting the next version to increase the odds of updates happening out of order + if (rand.nextBoolean()) Thread.yield(); + + if (oper < commitPercent + deletePercent) { + verbose("deleting id",id,"val=",nextVal,"version",version); + + Long returnedVersion = deleteAndGetVersion(Integer.toString(id), params("_version_",Long.toString(-version), DISTRIB_UPDATE_PARAM,FROM_LEADER)); + + // TODO: returning versions for these types of updates is redundant + // but if we do return, they had better be equal + if (returnedVersion != null) { + assertEquals(-version, returnedVersion.longValue()); + } + + // only update model if the version is newer + synchronized (model) { + DocInfo currInfo = model.get(id); + if (Math.abs(version) > Math.abs(currInfo.version)) { + model.put(id, new DocInfo(version, -nextVal)); + } + } + + verbose("deleting id", id, "val=",nextVal,"version",version,"DONE"); + } else if (oper < commitPercent + deletePercent + deleteByQueryPercent) { + + verbose("deleteByQuery id",id,"val=",nextVal,"version",version); + + Long returnedVersion = deleteByQueryAndGetVersion("id:"+Integer.toString(id), params("_version_",Long.toString(-version), DISTRIB_UPDATE_PARAM,FROM_LEADER)); + + // TODO: returning versions for these types of updates is redundant + // but if we do return, they had better be equal + if (returnedVersion != null) { + assertEquals(-version, returnedVersion.longValue()); + } + + // only update model if the version is newer + synchronized (model) { + DocInfo currInfo = model.get(id); + if (Math.abs(version) > Math.abs(currInfo.version)) { + model.put(id, new DocInfo(version, -nextVal)); + } + } + + verbose("deleteByQuery id", id, "val=",nextVal,"version",version,"DONE"); + + } else { + verbose("adding id", id, "val=", nextVal,"version",version); + + Long returnedVersion = addAndGetVersion(sdoc("id", Integer.toString(id), field, Long.toString(nextVal), "_version_",Long.toString(version)), params(DISTRIB_UPDATE_PARAM,FROM_LEADER)); + if (returnedVersion != null) { + assertEquals(version, returnedVersion.longValue()); + } + + // only update model if the version is newer + synchronized (model) { + DocInfo currInfo = model.get(id); + if (version > currInfo.version) { + model.put(id, new DocInfo(version, nextVal)); + } + } + + if (VERBOSE) { + verbose("adding id", id, "val=", nextVal,"version",version,"DONE"); + } + + } + // } // end sync + + if (!before) { + lastId = id; + } + } + } catch (Throwable e) { + operations.set(-1L); + log.error("",e); + throw new RuntimeException(e); + } + } + }; + + threads.add(thread); + } + + + for (int i=0; i= 0) { + // bias toward a recently changed doc + int id = rand.nextInt(100) < 25 ? lastId : rand.nextInt(ndocs); + + // when indexing, we update the index, then the model + // so when querying, we should first check the model, and then the index + + boolean realTime = rand.nextInt(100) < percentRealtimeQuery; + DocInfo info; + + if (realTime) { + info = model.get(id); + } else { + synchronized(TestStressReorder.this) { + info = committedModel.get(id); + } + } + + if (VERBOSE) { + verbose("querying id", id); + } + SolrQueryRequest sreq; + if (realTime) { + sreq = req("wt","json", "qt","/get", "ids",Integer.toString(id)); + } else { + sreq = req("wt","json", "q","id:"+Integer.toString(id), "omitHeader","true"); + } + + String response = h.query(sreq); + Map rsp = (Map)ObjectBuilder.fromJSON(response); + List doclist = (List)(((Map)rsp.get("response")).get("docs")); + if (doclist.size() == 0) { + // there's no info we can get back with a delete, so not much we can check without further synchronization + } else { + assertEquals(1, doclist.size()); + long foundVal = (Long)(((Map)doclist.get(0)).get(field)); + long foundVer = (Long)(((Map)doclist.get(0)).get("_version_")); + if (foundVer < Math.abs(info.version) + || (foundVer == info.version && foundVal != info.val) ) { // if the version matches, the val must + log.error("ERROR, id=" + id + " found=" + response + " model" + info); + assertTrue(false); + } + } + } + } catch (Throwable e) { + operations.set(-1L); + log.error("",e); + throw new RuntimeException(e); + } + } + }; + + threads.add(thread); + } + + + for (Thread thread : threads) { + thread.start(); + } + + for (Thread thread : threads) { + thread.join(); + } + + } + +} diff --git a/solr/core/src/test/org/apache/solr/search/TestStressVersions.java b/solr/core/src/test/org/apache/solr/search/TestStressVersions.java new file mode 100644 index 0000000..cce28a2 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/search/TestStressVersions.java @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search; + + +import org.apache.noggit.ObjectBuilder; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.util.TestHarness; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import static org.apache.solr.core.SolrCore.verbose; + +public class TestStressVersions extends TestRTGBase { + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig-tlog.xml","schema15.xml"); + } + + + // This version doesn't synchronize on id to tell what update won, but instead uses versions + @Test + public void testStressGetRealtimeVersions() throws Exception { + clearIndex(); + assertU(commit()); + + final int commitPercent = 5 + random().nextInt(20); + final int softCommitPercent = 30+random().nextInt(75); // what percent of the commits are soft + final int deletePercent = 4+random().nextInt(25); + final int deleteByQueryPercent = 1 + random().nextInt(5); + final int optimisticPercent = 1+random().nextInt(50); // percent change that an update uses optimistic locking + final int optimisticCorrectPercent = 25+random().nextInt(70); // percent change that a version specified will be correct + final int ndocs = 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200)); + int nWriteThreads = 5 + random().nextInt(25); + + final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers + + // query variables + final int percentRealtimeQuery = 75; + final AtomicLong operations = new AtomicLong(50000); // number of query operations to perform in total + int nReadThreads = 5 + random().nextInt(25); + + + + initModel(ndocs); + + final AtomicInteger numCommitting = new AtomicInteger(); + + List threads = new ArrayList(); + + for (int i=0; i 0) { + int oper = rand.nextInt(100); + + if (oper < commitPercent) { + if (numCommitting.incrementAndGet() <= maxConcurrentCommits) { + Map newCommittedModel; + long version; + + synchronized(globalLock) { + newCommittedModel = new HashMap(model); // take a snapshot + version = snapshotCount++; + } + + if (rand.nextInt(100) < softCommitPercent) { + verbose("softCommit start"); + assertU(TestHarness.commit("softCommit","true")); + verbose("softCommit end"); + } else { + verbose("hardCommit start"); + assertU(commit()); + verbose("hardCommit end"); + } + + synchronized(globalLock) { + // install this model snapshot only if it's newer than the current one + if (version >= committedModelClock) { + if (VERBOSE) { + verbose("installing new committedModel version="+committedModelClock); + } + committedModel = newCommittedModel; + committedModelClock = version; + } + } + } + numCommitting.decrementAndGet(); + continue; + } + + + int id = rand.nextInt(ndocs); + Object sync = syncArr[id]; + + // set the lastId before we actually change it sometimes to try and + // uncover more race conditions between writing and reading + boolean before = rand.nextBoolean(); + if (before) { + lastId = id; + } + + // We can't concurrently update the same document and retain our invariants of increasing values + // since we can't guarantee what order the updates will be executed. + // Even with versions, we can't remove the sync because increasing versions does not mean increasing vals. + // + // NOTE: versioning means we can now remove the sync and tell what update "won" + // synchronized (sync) { + DocInfo info = model.get(id); + + long val = info.val; + long nextVal = Math.abs(val)+1; + + if (oper < commitPercent + deletePercent) { + verbose("deleting id",id,"val=",nextVal); + + Long version = deleteAndGetVersion(Integer.toString(id), null); + assertTrue(version < 0); + + // only update model if the version is newer + synchronized (model) { + DocInfo currInfo = model.get(id); + if (Math.abs(version) > Math.abs(currInfo.version)) { + model.put(id, new DocInfo(version, -nextVal)); + } + } + + verbose("deleting id", id, "val=",nextVal,"DONE"); + } else if (oper < commitPercent + deletePercent + deleteByQueryPercent) { + verbose("deleteByQyery id",id,"val=",nextVal); + + Long version = deleteByQueryAndGetVersion("id:"+Integer.toString(id), null); + assertTrue(version < 0); + + // only update model if the version is newer + synchronized (model) { + DocInfo currInfo = model.get(id); + if (Math.abs(version) > Math.abs(currInfo.version)) { + model.put(id, new DocInfo(version, -nextVal)); + } + } + + verbose("deleteByQyery id", id, "val=",nextVal,"DONE"); + } else { + verbose("adding id", id, "val=", nextVal); + + // assertU(adoc("id",Integer.toString(id), field, Long.toString(nextVal))); + Long version = addAndGetVersion(sdoc("id", Integer.toString(id), field, Long.toString(nextVal)), null); + assertTrue(version > 0); + + // only update model if the version is newer + synchronized (model) { + DocInfo currInfo = model.get(id); + if (version > currInfo.version) { + model.put(id, new DocInfo(version, nextVal)); + } + } + + if (VERBOSE) { + verbose("adding id", id, "val=", nextVal,"DONE"); + } + + } + // } // end sync + + if (!before) { + lastId = id; + } + } + } catch (Throwable e) { + operations.set(-1L); + throw new RuntimeException(e); + } + } + }; + + threads.add(thread); + } + + + for (int i=0; i= 0) { + // bias toward a recently changed doc + int id = rand.nextInt(100) < 25 ? lastId : rand.nextInt(ndocs); + + // when indexing, we update the index, then the model + // so when querying, we should first check the model, and then the index + + boolean realTime = rand.nextInt(100) < percentRealtimeQuery; + DocInfo info; + + if (realTime) { + info = model.get(id); + } else { + synchronized(globalLock) { + info = committedModel.get(id); + } + } + + if (VERBOSE) { + verbose("querying id", id); + } + SolrQueryRequest sreq; + if (realTime) { + sreq = req("wt","json", "qt","/get", "ids",Integer.toString(id)); + } else { + sreq = req("wt","json", "q","id:"+Integer.toString(id), "omitHeader","true"); + } + + String response = h.query(sreq); + Map rsp = (Map)ObjectBuilder.fromJSON(response); + List doclist = (List)(((Map)rsp.get("response")).get("docs")); + if (doclist.size() == 0) { + // there's no info we can get back with a delete, so not much we can check without further synchronization + } else { + assertEquals(1, doclist.size()); + long foundVal = (Long)(((Map)doclist.get(0)).get(field)); + long foundVer = (Long)(((Map)doclist.get(0)).get("_version_")); + if (foundVer < Math.abs(info.version) + || (foundVer == info.version && foundVal != info.val) ) { // if the version matches, the val must + verbose("ERROR, id=", id, "found=",response,"model",info); + assertTrue(false); + } + } + } + } catch (Throwable e) { + operations.set(-1L); + throw new RuntimeException(e); + } + } + }; + + threads.add(thread); + } + + + for (Thread thread : threads) { + thread.start(); + } + + for (Thread thread : threads) { + thread.join(); + } + + } + + +} diff --git a/solr/core/src/test/org/apache/solr/search/function/NvlValueSourceParser.java b/solr/core/src/test/org/apache/solr/search/function/NvlValueSourceParser.java index dbc8d90..d12ed97 100755 --- a/solr/core/src/test/org/apache/solr/search/function/NvlValueSourceParser.java +++ b/solr/core/src/test/org/apache/solr/search/function/NvlValueSourceParser.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/function/SortByFunctionTest.java b/solr/core/src/test/org/apache/solr/search/function/SortByFunctionTest.java index dc3bf07..6a13218 100644 --- a/solr/core/src/test/org/apache/solr/search/function/SortByFunctionTest.java +++ b/solr/core/src/test/org/apache/solr/search/function/SortByFunctionTest.java @@ -1,5 +1,5 @@ package org.apache.solr.search.function; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java b/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java index 9ef9067..4f33da9 100755 --- a/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java +++ b/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -427,10 +427,7 @@ public class TestFunctionQuery extends SolrTestCaseJ4 { /** * test collection-level term stats (new in 4.x indexes) */ - public void testTotalTermFreq() throws Exception { - assumeFalse("PreFlex codec does not support collection-level term stats", - "Lucene3x".equals(Codec.getDefault().getName())); - + public void testTotalTermFreq() throws Exception { clearIndex(); assertU(adoc("id","1", "a_tdt","2009-08-31T12:10:10.123Z", "b_tdt","2009-08-31T12:10:10.124Z")); diff --git a/solr/core/src/test/org/apache/solr/search/function/distance/DistanceFunctionTest.java b/solr/core/src/test/org/apache/solr/search/function/distance/DistanceFunctionTest.java index ee68152..9c06ff4 100644 --- a/solr/core/src/test/org/apache/solr/search/function/distance/DistanceFunctionTest.java +++ b/solr/core/src/test/org/apache/solr/search/function/distance/DistanceFunctionTest.java @@ -1,5 +1,5 @@ package org.apache.solr.search.function.distance; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/similarities/BaseSimilarityTestCase.java b/solr/core/src/test/org/apache/solr/search/similarities/BaseSimilarityTestCase.java index 3ed3448..d05867c 100644 --- a/solr/core/src/test/org/apache/solr/search/similarities/BaseSimilarityTestCase.java +++ b/solr/core/src/test/org/apache/solr/search/similarities/BaseSimilarityTestCase.java @@ -1,6 +1,6 @@ package org.apache.solr.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/similarities/CustomSimilarityFactory.java b/solr/core/src/test/org/apache/solr/search/similarities/CustomSimilarityFactory.java index 9f036ba..bc4391a 100644 --- a/solr/core/src/test/org/apache/solr/search/similarities/CustomSimilarityFactory.java +++ b/solr/core/src/test/org/apache/solr/search/similarities/CustomSimilarityFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/similarities/MockConfigurableSimilarity.java b/solr/core/src/test/org/apache/solr/search/similarities/MockConfigurableSimilarity.java index 194bad8..ac90657 100644 --- a/solr/core/src/test/org/apache/solr/search/similarities/MockConfigurableSimilarity.java +++ b/solr/core/src/test/org/apache/solr/search/similarities/MockConfigurableSimilarity.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/similarities/TestBM25SimilarityFactory.java b/solr/core/src/test/org/apache/solr/search/similarities/TestBM25SimilarityFactory.java index e1fbe2a..2b893bc 100644 --- a/solr/core/src/test/org/apache/solr/search/similarities/TestBM25SimilarityFactory.java +++ b/solr/core/src/test/org/apache/solr/search/similarities/TestBM25SimilarityFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/similarities/TestDFRSimilarityFactory.java b/solr/core/src/test/org/apache/solr/search/similarities/TestDFRSimilarityFactory.java index 42b39f5..47dd14f 100644 --- a/solr/core/src/test/org/apache/solr/search/similarities/TestDFRSimilarityFactory.java +++ b/solr/core/src/test/org/apache/solr/search/similarities/TestDFRSimilarityFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/similarities/TestDefaultSimilarityFactory.java b/solr/core/src/test/org/apache/solr/search/similarities/TestDefaultSimilarityFactory.java index f939995..cd09027 100644 --- a/solr/core/src/test/org/apache/solr/search/similarities/TestDefaultSimilarityFactory.java +++ b/solr/core/src/test/org/apache/solr/search/similarities/TestDefaultSimilarityFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,6 +18,7 @@ package org.apache.solr.search.similarities; */ import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.Similarity; import org.junit.BeforeClass; /** @@ -30,7 +31,16 @@ public class TestDefaultSimilarityFactory extends BaseSimilarityTestCase { } /** default parameters */ - public void test() throws Exception { - assertEquals(DefaultSimilarity.class, getSimilarity("text").getClass()); + public void testDefaults() throws Exception { + Similarity sim = getSimilarity("text"); + assertEquals(DefaultSimilarity.class, sim.getClass()); + assertEquals(true, ((DefaultSimilarity)sim).getDiscountOverlaps()); + } + /** explicit params */ + public void testParams() throws Exception { + Similarity sim = getSimilarity("text_overlap"); + assertEquals(DefaultSimilarity.class, sim.getClass()); + assertEquals(false, ((DefaultSimilarity)sim).getDiscountOverlaps()); } + } diff --git a/solr/core/src/test/org/apache/solr/search/similarities/TestIBSimilarityFactory.java b/solr/core/src/test/org/apache/solr/search/similarities/TestIBSimilarityFactory.java index c53f4be..d857993 100644 --- a/solr/core/src/test/org/apache/solr/search/similarities/TestIBSimilarityFactory.java +++ b/solr/core/src/test/org/apache/solr/search/similarities/TestIBSimilarityFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/similarities/TestLMDirichletSimilarityFactory.java b/solr/core/src/test/org/apache/solr/search/similarities/TestLMDirichletSimilarityFactory.java index 8567e9d..318fc71 100644 --- a/solr/core/src/test/org/apache/solr/search/similarities/TestLMDirichletSimilarityFactory.java +++ b/solr/core/src/test/org/apache/solr/search/similarities/TestLMDirichletSimilarityFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/similarities/TestLMJelinekMercerSimilarityFactory.java b/solr/core/src/test/org/apache/solr/search/similarities/TestLMJelinekMercerSimilarityFactory.java index 6d25bbe..d56ad28 100644 --- a/solr/core/src/test/org/apache/solr/search/similarities/TestLMJelinekMercerSimilarityFactory.java +++ b/solr/core/src/test/org/apache/solr/search/similarities/TestLMJelinekMercerSimilarityFactory.java @@ -1,6 +1,6 @@ package org.apache.solr.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/search/similarities/TestPerFieldSimilarity.java b/solr/core/src/test/org/apache/solr/search/similarities/TestPerFieldSimilarity.java index 19c93eb..b4edee5 100644 --- a/solr/core/src/test/org/apache/solr/search/similarities/TestPerFieldSimilarity.java +++ b/solr/core/src/test/org/apache/solr/search/similarities/TestPerFieldSimilarity.java @@ -1,6 +1,6 @@ package org.apache.solr.search.similarities; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/servlet/CacheHeaderTest.java b/solr/core/src/test/org/apache/solr/servlet/CacheHeaderTest.java index b28f7c7..6ce92e4 100644 --- a/solr/core/src/test/org/apache/solr/servlet/CacheHeaderTest.java +++ b/solr/core/src/test/org/apache/solr/servlet/CacheHeaderTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/servlet/CacheHeaderTestBase.java b/solr/core/src/test/org/apache/solr/servlet/CacheHeaderTestBase.java index 2271f60..d4b83bc 100644 --- a/solr/core/src/test/org/apache/solr/servlet/CacheHeaderTestBase.java +++ b/solr/core/src/test/org/apache/solr/servlet/CacheHeaderTestBase.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/servlet/DirectSolrConnectionTest.java b/solr/core/src/test/org/apache/solr/servlet/DirectSolrConnectionTest.java index bb4031e..9d48b55 100644 --- a/solr/core/src/test/org/apache/solr/servlet/DirectSolrConnectionTest.java +++ b/solr/core/src/test/org/apache/solr/servlet/DirectSolrConnectionTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/servlet/NoCacheHeaderTest.java b/solr/core/src/test/org/apache/solr/servlet/NoCacheHeaderTest.java index c0a5ad0..1ff8891 100644 --- a/solr/core/src/test/org/apache/solr/servlet/NoCacheHeaderTest.java +++ b/solr/core/src/test/org/apache/solr/servlet/NoCacheHeaderTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -33,7 +33,7 @@ public class NoCacheHeaderTest extends CacheHeaderTestBase { // as its home. it could interfere with other tests! @BeforeClass public static void beforeTest() throws Exception { - createJetty(TEST_HOME(), "solr/conf/solrconfig-nocache.xml", null); + createJetty(TEST_HOME(), "solr/collection1/conf/solrconfig-nocache.xml", null); } // The tests diff --git a/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java b/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java index 6f7cb42..c7519f6 100644 --- a/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java +++ b/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -54,7 +54,7 @@ public class SolrRequestParserTest extends SolrTestCaseJ4 { static SolrRequestParsers parser; @AfterClass - public static void afterClass() throws Exception { + public static void afterClass() { parser = null; } diff --git a/solr/core/src/test/org/apache/solr/spelling/DirectSolrSpellCheckerTest.java b/solr/core/src/test/org/apache/solr/spelling/DirectSolrSpellCheckerTest.java index 7a9aeba..2ea50f3 100644 --- a/solr/core/src/test/org/apache/solr/spelling/DirectSolrSpellCheckerTest.java +++ b/solr/core/src/test/org/apache/solr/spelling/DirectSolrSpellCheckerTest.java @@ -1,6 +1,6 @@ package org.apache.solr.spelling; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/spelling/FileBasedSpellCheckerTest.java b/solr/core/src/test/org/apache/solr/spelling/FileBasedSpellCheckerTest.java index 990c888..616b0f0 100644 --- a/solr/core/src/test/org/apache/solr/spelling/FileBasedSpellCheckerTest.java +++ b/solr/core/src/test/org/apache/solr/spelling/FileBasedSpellCheckerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -54,7 +54,7 @@ public class FileBasedSpellCheckerTest extends SolrTestCaseJ4 { } @AfterClass - public static void afterClass() throws Exception { + public static void afterClass() { queryConverter = null; } diff --git a/solr/core/src/test/org/apache/solr/spelling/IndexBasedSpellCheckerTest.java b/solr/core/src/test/org/apache/solr/spelling/IndexBasedSpellCheckerTest.java index 8b2cc71..a16dd68 100644 --- a/solr/core/src/test/org/apache/solr/spelling/IndexBasedSpellCheckerTest.java +++ b/solr/core/src/test/org/apache/solr/spelling/IndexBasedSpellCheckerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -76,7 +76,7 @@ public class IndexBasedSpellCheckerTest extends SolrTestCaseJ4 { } @AfterClass - public static void afterClass() throws Exception { + public static void afterClass() { queryConverter = null; } @@ -291,7 +291,7 @@ public class IndexBasedSpellCheckerTest extends SolrTestCaseJ4 { ); for (int i = 0; i < ALT_DOCS.length; i++) { Document doc = new Document(); - doc.add(new Field("title", ALT_DOCS[i], TextField.TYPE_STORED)); + doc.add(new TextField("title", ALT_DOCS[i], Field.Store.YES)); iw.addDocument(doc); } iw.forceMerge(1); diff --git a/solr/core/src/test/org/apache/solr/spelling/SampleComparator.java b/solr/core/src/test/org/apache/solr/spelling/SampleComparator.java index 0d67b60..a4bb055 100644 --- a/solr/core/src/test/org/apache/solr/spelling/SampleComparator.java +++ b/solr/core/src/test/org/apache/solr/spelling/SampleComparator.java @@ -1,5 +1,5 @@ package org.apache.solr.spelling; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java b/solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java index fee6848..63ede26 100644 --- a/solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java +++ b/solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/spelling/SpellCheckCollatorTest.java b/solr/core/src/test/org/apache/solr/spelling/SpellCheckCollatorTest.java index 08f2985..ccca7ac 100644 --- a/solr/core/src/test/org/apache/solr/spelling/SpellCheckCollatorTest.java +++ b/solr/core/src/test/org/apache/solr/spelling/SpellCheckCollatorTest.java @@ -1,5 +1,5 @@ package org.apache.solr.spelling; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,6 +20,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.GroupParams; @@ -37,6 +38,7 @@ import org.apache.solr.response.SolrQueryResponse; import org.junit.BeforeClass; import org.junit.Test; +@Slow public class SpellCheckCollatorTest extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { @@ -54,6 +56,7 @@ public class SpellCheckCollatorTest extends SolrTestCaseJ4 { assertNull(h.validateUpdate(adoc("id", "10", "teststop", "Once in Paris Dick built a fire on the hearth"))); assertNull(h.validateUpdate(adoc("id", "11", "teststop", "Dick waited for Jane as he watched the sparks flow upward"))); assertNull(h.validateUpdate(adoc("id", "12", "teststop", "This June parisian rendez-vous is ruined because of a customs snafu"))); + assertNull(h.validateUpdate(adoc("id", "13", "teststop", "partisan political machine"))); assertNull(h.validateUpdate(commit())); } @@ -108,6 +111,44 @@ public class SpellCheckCollatorTest extends SolrTestCaseJ4 { } } + + public void testCollateWithOverride() throws Exception + { + assertQ( + req( + SpellCheckComponent.COMPONENT_NAME, "true", + SpellCheckComponent.SPELLCHECK_DICT, "direct", + SpellingParams.SPELLCHECK_COUNT, "10", + SpellingParams.SPELLCHECK_COLLATE, "true", + SpellingParams.SPELLCHECK_MAX_COLLATION_TRIES, "10", + SpellingParams.SPELLCHECK_MAX_COLLATIONS, "10", + "qt", "spellCheckCompRH", + "defType", "edismax", + "qf", "teststop", + "mm", "1", + CommonParams.Q, "partisian politcal mashine" + ), + "//lst[@name='spellcheck']/lst[@name='suggestions']/str[@name='collation']='parisian political machine'" + ); + assertQ( + req( + SpellCheckComponent.COMPONENT_NAME, "true", + SpellCheckComponent.SPELLCHECK_DICT, "direct", + SpellingParams.SPELLCHECK_COUNT, "10", + SpellingParams.SPELLCHECK_COLLATE, "true", + SpellingParams.SPELLCHECK_MAX_COLLATION_TRIES, "10", + SpellingParams.SPELLCHECK_MAX_COLLATIONS, "10", + "qt", "spellCheckCompRH", + "defType", "edismax", + "qf", "teststop", + "mm", "1", + SpellingParams.SPELLCHECK_COLLATE_PARAM_OVERRIDE + "mm", "100%", + CommonParams.Q, "partisian politcal mashine" + ), + "//lst[@name='spellcheck']/lst[@name='suggestions']/str[@name='collation']='partisan political machine'" + ); + + } @Test public void testCollateWithFilter() throws Exception diff --git a/solr/core/src/test/org/apache/solr/spelling/SpellPossibilityIteratorTest.java b/solr/core/src/test/org/apache/solr/spelling/SpellPossibilityIteratorTest.java index 6cf12d7..e233ebc 100644 --- a/solr/core/src/test/org/apache/solr/spelling/SpellPossibilityIteratorTest.java +++ b/solr/core/src/test/org/apache/solr/spelling/SpellPossibilityIteratorTest.java @@ -1,5 +1,5 @@ package org.apache.solr.spelling; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -16,8 +16,10 @@ package org.apache.solr.spelling; * limitations under the License. */ +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; import org.apache.lucene.analysis.Token; import org.apache.solr.SolrTestCaseJ4; @@ -26,17 +28,22 @@ import org.junit.Before; import org.junit.Test; public class SpellPossibilityIteratorTest extends SolrTestCaseJ4 { - - private static Map> suggestions = new LinkedHashMap>(); - private static Map> lotsaSuggestions = new LinkedHashMap>(); - + private static final Token TOKEN_AYE = new Token("AYE", 0, 3); + private static final Token TOKEN_BEE = new Token("BEE", 4, 7); + private static final Token TOKEN_AYE_BEE = new Token("AYE BEE", 0, 7); + private static final Token TOKEN_CEE = new Token("CEE", 8, 11); + + private LinkedHashMap AYE; + private LinkedHashMap BEE; + private LinkedHashMap AYE_BEE; + private LinkedHashMap CEE; + @Override @Before public void setUp() throws Exception { super.setUp(); - suggestions.clear(); - LinkedHashMap AYE = new LinkedHashMap(); + AYE = new LinkedHashMap(); AYE.put("I", 0); AYE.put("II", 0); AYE.put("III", 0); @@ -46,7 +53,7 @@ public class SpellPossibilityIteratorTest extends SolrTestCaseJ4 { AYE.put("VII", 0); AYE.put("VIII", 0); - LinkedHashMap BEE = new LinkedHashMap(); + BEE = new LinkedHashMap(); BEE.put("alpha", 0); BEE.put("beta", 0); BEE.put("gamma", 0); @@ -57,8 +64,19 @@ public class SpellPossibilityIteratorTest extends SolrTestCaseJ4 { BEE.put("theta", 0); BEE.put("iota", 0); + AYE_BEE = new LinkedHashMap(); + AYE_BEE.put("one-alpha", 0); + AYE_BEE.put("two-beta", 0); + AYE_BEE.put("three-gamma", 0); + AYE_BEE.put("four-delta", 0); + AYE_BEE.put("five-epsilon", 0); + AYE_BEE.put("six-zeta", 0); + AYE_BEE.put("seven-eta", 0); + AYE_BEE.put("eight-theta", 0); + AYE_BEE.put("nine-iota", 0); + - LinkedHashMap CEE = new LinkedHashMap(); + CEE = new LinkedHashMap(); CEE.put("one", 0); CEE.put("two", 0); CEE.put("three", 0); @@ -69,61 +87,75 @@ public class SpellPossibilityIteratorTest extends SolrTestCaseJ4 { CEE.put("eight", 0); CEE.put("nine", 0); CEE.put("ten", 0); - - suggestions.put(new Token("AYE", 0, 2), AYE); - suggestions.put(new Token("BEE", 0, 2), BEE); - suggestions.put(new Token("CEE", 0, 2), CEE); - - lotsaSuggestions.put(new Token("AYE", 0, 2), AYE); - lotsaSuggestions.put(new Token("BEE", 0, 2), BEE); - lotsaSuggestions.put(new Token("CEE", 0, 2), CEE); - - lotsaSuggestions.put(new Token("AYE1", 0, 3), AYE); - lotsaSuggestions.put(new Token("BEE1", 0, 3), BEE); - lotsaSuggestions.put(new Token("CEE1", 0, 3), CEE); - - lotsaSuggestions.put(new Token("AYE2", 0, 3), AYE); - lotsaSuggestions.put(new Token("BEE2", 0, 3), BEE); - lotsaSuggestions.put(new Token("CEE2", 0, 3), CEE); - - lotsaSuggestions.put(new Token("AYE3", 0, 3), AYE); - lotsaSuggestions.put(new Token("BEE3", 0, 3), BEE); - lotsaSuggestions.put(new Token("CEE3", 0, 3), CEE); - - lotsaSuggestions.put(new Token("AYE4", 0, 3), AYE); - lotsaSuggestions.put(new Token("BEE4", 0, 3), BEE); - lotsaSuggestions.put(new Token("CEE4", 0, 3), CEE); } @Test public void testScalability() throws Exception { - PossibilityIterator iter = new PossibilityIterator(lotsaSuggestions, 1000, 10000); + Map> lotsaSuggestions = new LinkedHashMap>(); + lotsaSuggestions.put(TOKEN_AYE , AYE); + lotsaSuggestions.put(TOKEN_BEE , BEE); + lotsaSuggestions.put(TOKEN_CEE , CEE); + + lotsaSuggestions.put(new Token("AYE1", 0, 3), AYE); + lotsaSuggestions.put(new Token("BEE1", 4, 7), BEE); + lotsaSuggestions.put(new Token("CEE1", 8, 11), CEE); + + lotsaSuggestions.put(new Token("AYE2", 0, 3), AYE); + lotsaSuggestions.put(new Token("BEE2", 4, 7), BEE); + lotsaSuggestions.put(new Token("CEE2", 8, 11), CEE); + + lotsaSuggestions.put(new Token("AYE3", 0, 3), AYE); + lotsaSuggestions.put(new Token("BEE3", 4, 7), BEE); + lotsaSuggestions.put(new Token("CEE3", 8, 11), CEE); + + lotsaSuggestions.put(new Token("AYE4", 0, 3), AYE); + lotsaSuggestions.put(new Token("BEE4", 4, 7), BEE); + lotsaSuggestions.put(new Token("CEE4", 8, 11), CEE); + + PossibilityIterator iter = new PossibilityIterator(lotsaSuggestions, 1000, 10000, false); int count = 0; while (iter.hasNext()) { - RankedSpellPossibility rsp = iter.next(); + PossibilityIterator.RankedSpellPossibility rsp = iter.next(); count++; } assertTrue(count==1000); + + lotsaSuggestions.put(new Token("AYE_BEE1", 0, 7), AYE_BEE); + lotsaSuggestions.put(new Token("AYE_BEE2", 0, 7), AYE_BEE); + lotsaSuggestions.put(new Token("AYE_BEE3", 0, 7), AYE_BEE); + lotsaSuggestions.put(new Token("AYE_BEE4", 0, 7), AYE_BEE); + iter = new PossibilityIterator(lotsaSuggestions, 1000, 10000, true); + count = 0; + while (iter.hasNext()) { + PossibilityIterator.RankedSpellPossibility rsp = iter.next(); + count++; + } + assertTrue(count<100); } @Test public void testSpellPossibilityIterator() throws Exception { - PossibilityIterator iter = new PossibilityIterator(suggestions, 1000, 10000); + Map> suggestions = new LinkedHashMap>(); + suggestions.put(TOKEN_AYE , AYE); + suggestions.put(TOKEN_BEE , BEE); + suggestions.put(TOKEN_CEE , CEE); + + PossibilityIterator iter = new PossibilityIterator(suggestions, 1000, 10000, false); int count = 0; while (iter.hasNext()) { - RankedSpellPossibility rsp = iter.next(); + PossibilityIterator.RankedSpellPossibility rsp = iter.next(); if(count==0) { - assertTrue("I".equals(rsp.getCorrections().get(0).getCorrection())); - assertTrue("alpha".equals(rsp.getCorrections().get(1).getCorrection())); - assertTrue("one".equals(rsp.getCorrections().get(2).getCorrection())); + assertTrue("I".equals(rsp.corrections.get(0).getCorrection())); + assertTrue("alpha".equals(rsp.corrections.get(1).getCorrection())); + assertTrue("one".equals(rsp.corrections.get(2).getCorrection())); } count++; } assertTrue(("Three maps (8*9*10) should return 720 iterations but instead returned " + count), count == 720); - suggestions.remove(new Token("CEE", 0, 2)); - iter = new PossibilityIterator(suggestions, 100, 10000); + suggestions.remove(TOKEN_CEE); + iter = new PossibilityIterator(suggestions, 100, 10000, false); count = 0; while (iter.hasNext()) { iter.next(); @@ -131,8 +163,8 @@ public class SpellPossibilityIteratorTest extends SolrTestCaseJ4 { } assertTrue(("Two maps (8*9) should return 72 iterations but instead returned " + count), count == 72); - suggestions.remove(new Token("BEE", 0, 2)); - iter = new PossibilityIterator(suggestions, 5, 10000); + suggestions.remove(TOKEN_BEE); + iter = new PossibilityIterator(suggestions, 5, 10000, false); count = 0; while (iter.hasNext()) { iter.next(); @@ -140,8 +172,8 @@ public class SpellPossibilityIteratorTest extends SolrTestCaseJ4 { } assertTrue(("We requested 5 suggestions but got " + count), count == 5); - suggestions.remove(new Token("AYE", 0, 2)); - iter = new PossibilityIterator(suggestions, Integer.MAX_VALUE, 10000); + suggestions.remove(TOKEN_AYE); + iter = new PossibilityIterator(suggestions, Integer.MAX_VALUE, 10000, false); count = 0; while (iter.hasNext()) { iter.next(); @@ -150,4 +182,47 @@ public class SpellPossibilityIteratorTest extends SolrTestCaseJ4 { assertTrue(("No maps should return 0 iterations but instead returned " + count), count == 0); } + + @Test + public void testOverlappingTokens() throws Exception { + Map> overlappingSuggestions = new LinkedHashMap>(); + overlappingSuggestions.put(TOKEN_AYE, AYE); + overlappingSuggestions.put(TOKEN_BEE, BEE); + overlappingSuggestions.put(TOKEN_AYE_BEE, AYE_BEE); + overlappingSuggestions.put(TOKEN_CEE, CEE); + + PossibilityIterator iter = new PossibilityIterator(overlappingSuggestions, Integer.MAX_VALUE, Integer.MAX_VALUE, true); + int aCount = 0; + int abCount = 0; + Set dupChecker = new HashSet(); + while (iter.hasNext()) { + PossibilityIterator.RankedSpellPossibility rsp = iter.next(); + Token a = null; + Token b = null; + Token ab = null; + Token c = null; + for(SpellCheckCorrection scc : rsp.corrections) { + if(scc.getOriginal().equals(TOKEN_AYE)) { + a = scc.getOriginal(); + } else if(scc.getOriginal().equals(TOKEN_BEE)) { + b = scc.getOriginal(); + } else if(scc.getOriginal().equals(TOKEN_AYE_BEE)) { + ab = scc.getOriginal(); + } else if(scc.getOriginal().equals(TOKEN_CEE)) { + c = scc.getOriginal(); + } + if(ab!=null) { + abCount++; + } else { + aCount++; + } + } + assertTrue(c != null); + assertTrue(ab != null || (a!=null && b!=null)); + assertTrue(ab == null || (a==null && b==null)); + assertTrue(dupChecker.add(rsp)); + } + assertTrue(aCount==2160); + assertTrue(abCount==180); + } } diff --git a/solr/core/src/test/org/apache/solr/spelling/SpellingQueryConverterTest.java b/solr/core/src/test/org/apache/solr/spelling/SpellingQueryConverterTest.java index 4cbebf9..b76cad4 100644 --- a/solr/core/src/test/org/apache/solr/spelling/SpellingQueryConverterTest.java +++ b/solr/core/src/test/org/apache/solr/spelling/SpellingQueryConverterTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,7 +23,9 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.common.util.NamedList; import org.junit.Test; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; /** @@ -126,4 +128,75 @@ public class SpellingQueryConverterTest extends LuceneTestCase { assertTrue("tokens is null and it shouldn't be", tokens != null); assertEquals("tokens Size: " + tokens.size() + " is not 2", 2, tokens.size()); } + + @Test + public void testRequiredOrProhibitedFlags() { + SpellingQueryConverter converter = new SpellingQueryConverter(); + converter.init(new NamedList()); + converter.setAnalyzer(new WhitespaceAnalyzer(TEST_VERSION_CURRENT)); + + { + List tokens = new ArrayList(converter.convert("aaa bbb ccc")); + assertTrue("Should have 3 tokens", tokens != null && tokens.size()==3); + assertTrue("token 1 should be optional", !hasRequiredFlag(tokens.get(0)) && !hasProhibitedFlag(tokens.get(0))); + assertTrue("token 2 should be optional", !hasRequiredFlag(tokens.get(1)) && !hasProhibitedFlag(tokens.get(1))); + assertTrue("token 3 should be optional", !hasRequiredFlag(tokens.get(2)) && !hasProhibitedFlag(tokens.get(2))); + } + { + List tokens = new ArrayList(converter.convert("+aaa bbb -ccc")); + assertTrue("Should have 3 tokens", tokens != null && tokens.size()==3); + assertTrue("token 1 should be required", hasRequiredFlag(tokens.get(0)) && !hasProhibitedFlag(tokens.get(0))); + assertTrue("token 2 should be optional", !hasRequiredFlag(tokens.get(1)) && !hasProhibitedFlag(tokens.get(1))); + assertTrue("token 3 should be prohibited", !hasRequiredFlag(tokens.get(2)) && hasProhibitedFlag(tokens.get(2))); + } + { + List tokens = new ArrayList(converter.convert("aaa AND bbb ccc")); + assertTrue("Should have 3 tokens", tokens != null && tokens.size()==3); + assertTrue("token 1 doesn't precede n.b.o.", !hasNBOFlag(tokens.get(0)) && hasInBooleanFlag(tokens.get(0))); + assertTrue("token 2 doesn't precede n.b.o.", !hasNBOFlag(tokens.get(1)) && hasInBooleanFlag(tokens.get(0))); + assertTrue("token 3 doesn't precede n.b.o.", !hasNBOFlag(tokens.get(2)) && hasInBooleanFlag(tokens.get(0))); + } + { + List tokens = new ArrayList(converter.convert("aaa OR bbb OR ccc")); + assertTrue("Should have 3 tokens", tokens != null && tokens.size()==3); + assertTrue("token 1 doesn't precede n.b.o.", !hasNBOFlag(tokens.get(0)) && hasInBooleanFlag(tokens.get(0))); + assertTrue("token 2 doesn't precede n.b.o.", !hasNBOFlag(tokens.get(1)) && hasInBooleanFlag(tokens.get(0))); + assertTrue("token 3 doesn't precede n.b.o.", !hasNBOFlag(tokens.get(2)) && hasInBooleanFlag(tokens.get(0))); + } + { + List tokens = new ArrayList(converter.convert("aaa AND bbb NOT ccc")); + assertTrue("Should have 3 tokens", tokens != null && tokens.size()==3); + assertTrue("token 1 doesn't precede n.b.o.", !hasNBOFlag(tokens.get(0)) && hasInBooleanFlag(tokens.get(0))); + assertTrue("token 2 precedes n.b.o.", hasNBOFlag(tokens.get(1)) && hasInBooleanFlag(tokens.get(0))); + assertTrue("token 3 doesn't precede n.b.o.", !hasNBOFlag(tokens.get(2)) && hasInBooleanFlag(tokens.get(0))); + } + { + List tokens = new ArrayList(converter.convert("aaa NOT bbb AND ccc")); + assertTrue("Should have 3 tokens", tokens != null && tokens.size()==3); + assertTrue("token 1 precedes n.b.o.", hasNBOFlag(tokens.get(0)) && hasInBooleanFlag(tokens.get(0))); + assertTrue("token 2 precedes n.b.o.", hasNBOFlag(tokens.get(1)) && hasInBooleanFlag(tokens.get(0))); + assertTrue("token 3 doesn't precedes n.b.o.", !hasNBOFlag(tokens.get(2)) && hasInBooleanFlag(tokens.get(0))); + } + { + List tokens = new ArrayList(converter.convert("aaa AND NOT bbb AND ccc")); + assertTrue("Should have 3 tokens", tokens != null && tokens.size()==3); + assertTrue("token 1 precedes n.b.o.", hasNBOFlag(tokens.get(0)) && hasInBooleanFlag(tokens.get(0))); + assertTrue("token 2 precedes n.b.o.", hasNBOFlag(tokens.get(1)) && hasInBooleanFlag(tokens.get(0))); + assertTrue("token 3 doesn't precedes n.b.o.", !hasNBOFlag(tokens.get(2)) && hasInBooleanFlag(tokens.get(0))); + } + + } + + private boolean hasRequiredFlag(Token t) { + return (t.getFlags() & QueryConverter.REQUIRED_TERM_FLAG) == QueryConverter.REQUIRED_TERM_FLAG; + } + private boolean hasProhibitedFlag(Token t) { + return (t.getFlags() & QueryConverter.PROHIBITED_TERM_FLAG) == QueryConverter.PROHIBITED_TERM_FLAG; + } + private boolean hasNBOFlag(Token t) { + return (t.getFlags() & QueryConverter.TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG) == QueryConverter.TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG; + } + private boolean hasInBooleanFlag(Token t) { + return (t.getFlags() & QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG) == QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG; + } } diff --git a/solr/core/src/test/org/apache/solr/spelling/TestSuggestSpellingConverter.java b/solr/core/src/test/org/apache/solr/spelling/TestSuggestSpellingConverter.java index a2f0110..8fa61b8 100644 --- a/solr/core/src/test/org/apache/solr/spelling/TestSuggestSpellingConverter.java +++ b/solr/core/src/test/org/apache/solr/spelling/TestSuggestSpellingConverter.java @@ -1,6 +1,6 @@ package org.apache.solr.spelling; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/spelling/WordBreakSolrSpellCheckerTest.java b/solr/core/src/test/org/apache/solr/spelling/WordBreakSolrSpellCheckerTest.java new file mode 100644 index 0000000..2caf2a2 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/spelling/WordBreakSolrSpellCheckerTest.java @@ -0,0 +1,282 @@ +package org.apache.solr.spelling; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.Token; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.component.SpellCheckComponent; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.util.RefCounted; +import org.junit.BeforeClass; +import org.junit.Test; + +public class WordBreakSolrSpellCheckerTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig-spellcheckcomponent.xml","schema.xml"); + assertNull(h.validateUpdate(adoc("id", "0", "lowerfilt", "pain table paintablepine pi ne in able"))); + assertNull(h.validateUpdate(adoc("id", "1", "lowerfilt", "paint able pineapple goodness in"))); + assertNull(h.validateUpdate(adoc("id", "2", "lowerfilt", "pa in table pineapplegoodness"))); + assertNull(h.validateUpdate(adoc("id", "3", "lowerfilt", "printable line in ample food mess"))); + assertNull(h.validateUpdate(adoc("id", "4", "lowerfilt", "printable in pointable paint able"))); + assertNull(h.validateUpdate(adoc("id", "5", "lowerfilt", "printable in puntable paint able "))); + assertNull(h.validateUpdate(adoc("id", "6", "lowerfilt", "paint able in pintable plantable"))); + assertNull(h.validateUpdate(commit())); + //docfreq=7: in + //docfreq=5: able + //docfreq=4: paint + //docfreq=3: printable + //docfreq=2: table + //docfreq=1: {all others} + } + + @Test + public void testStandAlone() throws Exception { + SolrCore core = h.getCore(); + WordBreakSolrSpellChecker checker = new WordBreakSolrSpellChecker(); + NamedList params = new NamedList(); + params.add("field", "lowerfilt"); + params.add(WordBreakSolrSpellChecker.PARAM_BREAK_WORDS, "true"); + params.add(WordBreakSolrSpellChecker.PARAM_COMBINE_WORDS, "true"); + params.add(WordBreakSolrSpellChecker.PARAM_MAX_CHANGES, "10"); + checker.init(params, core); + + RefCounted searcher = core.getSearcher(); + QueryConverter qc = new SpellingQueryConverter(); + qc.setAnalyzer(new MockAnalyzer(random())); + Collection tokens = qc.convert("paintable pine apple good ness"); + SpellingOptions spellOpts = new SpellingOptions(tokens, searcher.get().getIndexReader(), 10); + SpellingResult result = checker.getSuggestions(spellOpts); + searcher.decref(); + + assertTrue(result != null && result.getSuggestions() != null); + assertTrue(result.getSuggestions().size()==6); + + for(Map.Entry> s : result.getSuggestions().entrySet()) { + Token orig = s.getKey(); + String[] corr = s.getValue().keySet().toArray(new String[0]); + if(orig.toString().equals("paintable")) { + assertTrue(orig.startOffset()==0); + assertTrue(orig.endOffset()==9); + assertTrue(orig.length()==9); + assertTrue(corr.length==3); + assertTrue(corr[0].equals("paint able")); //1 op ; max doc freq=5 + assertTrue(corr[1].equals("pain table")); //1 op ; max doc freq=2 + assertTrue(corr[2].equals("pa in table")); //2 ops + } else if(orig.toString().equals("pine apple")) { + assertTrue(orig.startOffset()==10); + assertTrue(orig.endOffset()==20); + assertTrue(orig.length()==10); + assertTrue(corr.length==1); + assertTrue(corr[0].equals("pineapple")); + } else if(orig.toString().equals("paintable pine")) { + assertTrue(orig.startOffset()==0); + assertTrue(orig.endOffset()==14); + assertTrue(orig.length()==14); + assertTrue(corr.length==1); + assertTrue(corr[0].equals("paintablepine")); + } else if(orig.toString().equals("good ness")) { + assertTrue(orig.startOffset()==21); + assertTrue(orig.endOffset()==30); + assertTrue(orig.length()==9); + assertTrue(corr.length==1); + assertTrue(corr[0].equals("goodness")); + } else if(orig.toString().equals("pine apple good ness")) { + assertTrue(orig.startOffset()==10); + assertTrue(orig.endOffset()==30); + assertTrue(orig.length()==20); + assertTrue(corr.length==1); + assertTrue(corr[0].equals("pineapplegoodness")); + } else if(orig.toString().equals("pine")) { + assertTrue(orig.startOffset()==10); + assertTrue(orig.endOffset()==14); + assertTrue(orig.length()==4); + assertTrue(corr.length==1); + assertTrue(corr[0].equals("pi ne")); + } else { + fail("Unexpected original result: " + orig); + } + } + } + @Test + public void testInConjunction() throws Exception { + assertQ(req( + "q", "lowerfilt:(paintable pine apple good ness)", + "qt", "spellCheckWithWordbreak", + "indent", "true", + SpellCheckComponent.SPELLCHECK_BUILD, "true", + SpellCheckComponent.COMPONENT_NAME, "true", + SpellCheckComponent.SPELLCHECK_ACCURACY, ".75", + SpellCheckComponent.SPELLCHECK_EXTENDED_RESULTS, "true"), + "//lst[@name='suggestions']/lst[1]/@name='paintable'", + "//lst[@name='suggestions']/lst[2]/@name='pine'", + "//lst[@name='suggestions']/lst[3]/@name='apple'", + "//lst[@name='suggestions']/lst[4]/@name='good'", + "//lst[@name='suggestions']/lst[5]/@name='ness'", + "//lst[@name='paintable']/int[@name='numFound']=8", + "//lst[@name='paintable']/int[@name='startOffset']=11", + "//lst[@name='paintable']/int[@name='endOffset']=20", + "//lst[@name='paintable']/int[@name='origFreq']=0", + "//lst[@name='paintable']/arr[@name='suggestion']/lst[1]/str[@name='word']='printable'", //SolrSpellChecker result interleaved + "//lst[@name='paintable']/arr[@name='suggestion']/lst[1]/int[@name='freq']=3", + "//lst[@name='paintable']/arr[@name='suggestion']/lst[2]/str[@name='word']='paint able'", //1 op + "//lst[@name='paintable']/arr[@name='suggestion']/lst[2]/int[@name='freq']=5", + "//lst[@name='paintable']/arr[@name='suggestion']/lst[3]/str[@name='word']='pintable'", //SolrSpellChecker result interleaved + "//lst[@name='paintable']/arr[@name='suggestion']/lst[3]/int[@name='freq']=1", + "//lst[@name='paintable']/arr[@name='suggestion']/lst[4]/str[@name='word']='pain table'", //1 op + "//lst[@name='paintable']/arr[@name='suggestion']/lst[4]/int[@name='freq']=2", + "//lst[@name='paintable']/arr[@name='suggestion']/lst[5]/str[@name='word']='pointable'", //SolrSpellChecker result interleaved + "//lst[@name='paintable']/arr[@name='suggestion']/lst[5]/int[@name='freq']=1", + "//lst[@name='paintable']/arr[@name='suggestion']/lst[6]/str[@name='word']='pa in table'", //2 ops + "//lst[@name='paintable']/arr[@name='suggestion']/lst[6]/int[@name='freq']=7", + "//lst[@name='paintable']/arr[@name='suggestion']/lst[7]/str[@name='word']='plantable'", //SolrSpellChecker result interleaved + "//lst[@name='paintable']/arr[@name='suggestion']/lst[7]/int[@name='freq']=1", + "//lst[@name='paintable']/arr[@name='suggestion']/lst[8]/str[@name='word']='puntable'", //SolrSpellChecker result interleaved + "//lst[@name='paintable']/arr[@name='suggestion']/lst[8]/int[@name='freq']=1", + "//lst[@name='pine']/int[@name='numFound']=2", + "//lst[@name='pine']/int[@name='startOffset']=21", + "//lst[@name='pine']/int[@name='endOffset']=25", + "//lst[@name='pine']/arr[@name='suggestion']/lst[1]/str[@name='word']='line'", + "//lst[@name='pine']/arr[@name='suggestion']/lst[2]/str[@name='word']='pi ne'", + "//lst[@name='apple']/int[@name='numFound']=1", + "//lst[@name='apple']/arr[@name='suggestion']/lst[1]/str[@name='word']='ample'", + "//lst[@name='good']/int[@name='numFound']=1", + "//lst[@name='good']/arr[@name='suggestion']/lst[1]/str[@name='word']='food'", + "//lst[@name='ness']/int[@name='numFound']=1", + "//lst[@name='ness']/arr[@name='suggestion']/lst[1]/str[@name='word']='mess'", + "//lst[@name='pine apple']/int[@name='numFound']=1", + "//lst[@name='pine apple']/int[@name='startOffset']=21", + "//lst[@name='pine apple']/int[@name='endOffset']=31", + "//lst[@name='pine apple']/arr[@name='suggestion']/lst[1]/str[@name='word']='pineapple'", + "//lst[@name='paintable pine']/int[@name='numFound']=1", + "//lst[@name='paintable pine']/int[@name='startOffset']=11", + "//lst[@name='paintable pine']/int[@name='endOffset']=25", + "//lst[@name='paintable pine']/arr[@name='suggestion']/lst[1]/str[@name='word']='paintablepine'", + "//lst[@name='good ness']/int[@name='numFound']=1", + "//lst[@name='good ness']/int[@name='startOffset']=32", + "//lst[@name='good ness']/int[@name='endOffset']=41", + "//lst[@name='good ness']/arr[@name='suggestion']/lst[1]/str[@name='word']='goodness'", + "//lst[@name='pine apple good ness']/int[@name='numFound']=1", + "//lst[@name='pine apple good ness']/int[@name='startOffset']=21", + "//lst[@name='pine apple good ness']/int[@name='endOffset']=41", + "//lst[@name='pine apple good ness']/arr[@name='suggestion']/lst[1]/str[@name='word']='pineapplegoodness'" + ); + } + @Test + public void testCollate() throws Exception { + assertQ(req( + "q", "lowerfilt:(paintable pine apple godness)", + "qt", "spellCheckWithWordbreak", + "indent", "true", + SpellCheckComponent.SPELLCHECK_BUILD, "true", + SpellCheckComponent.COMPONENT_NAME, "true", + SpellCheckComponent.SPELLCHECK_ACCURACY, ".75", + SpellCheckComponent.SPELLCHECK_EXTENDED_RESULTS, "true", + SpellCheckComponent.SPELLCHECK_COLLATE, "true", + SpellCheckComponent.SPELLCHECK_COLLATE_EXTENDED_RESULTS, "true", + SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "10"), + "//lst[@name='collation'][1 ]/str[@name='collationQuery']='lowerfilt:(printable line ample goodness)'", + "//lst[@name='collation'][2 ]/str[@name='collationQuery']='lowerfilt:(paintablepine ample goodness)'", + "//lst[@name='collation'][3 ]/str[@name='collationQuery']='lowerfilt:(printable pineapple goodness)'", + "//lst[@name='collation'][4 ]/str[@name='collationQuery']='lowerfilt:((paint able) line ample goodness)'", + "//lst[@name='collation'][5 ]/str[@name='collationQuery']='lowerfilt:(printable (pi ne) ample goodness)'", + "//lst[@name='collation'][6 ]/str[@name='collationQuery']='lowerfilt:((paint able) pineapple goodness)'", + "//lst[@name='collation'][7 ]/str[@name='collationQuery']='lowerfilt:((paint able) (pi ne) ample goodness)'", + "//lst[@name='collation'][8 ]/str[@name='collationQuery']='lowerfilt:(pintable line ample goodness)'", + "//lst[@name='collation'][9 ]/str[@name='collationQuery']='lowerfilt:(pintable pineapple goodness)'", + "//lst[@name='collation'][10]/str[@name='collationQuery']='lowerfilt:(pintable (pi ne) ample goodness)'", + "//lst[@name='collation'][10]/lst[@name='misspellingsAndCorrections']/str[@name='paintable']='pintable'", + "//lst[@name='collation'][10]/lst[@name='misspellingsAndCorrections']/str[@name='pine']='pi ne'", + "//lst[@name='collation'][10]/lst[@name='misspellingsAndCorrections']/str[@name='apple']='ample'", + "//lst[@name='collation'][10]/lst[@name='misspellingsAndCorrections']/str[@name='godness']='goodness'" + ); + assertQ(req( + "q", "lowerfilt:(pine AND apple)", + "qt", "spellCheckWithWordbreak", + "indent", "true", + SpellCheckComponent.COMPONENT_NAME, "true", + SpellCheckComponent.SPELLCHECK_ACCURACY, ".75", + SpellCheckComponent.SPELLCHECK_EXTENDED_RESULTS, "true", + SpellCheckComponent.SPELLCHECK_COLLATE, "true", + SpellCheckComponent.SPELLCHECK_COLLATE_EXTENDED_RESULTS, "true", + SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "10"), + "//lst[@name='collation'][1 ]/str[@name='collationQuery']='lowerfilt:(line AND ample)'", + "//lst[@name='collation'][2 ]/str[@name='collationQuery']='lowerfilt:(pineapple)'", + "//lst[@name='collation'][3 ]/str[@name='collationQuery']='lowerfilt:((pi AND ne) AND ample)'" + ); + assertQ(req( + "q", "lowerfilt:pine AND NOT lowerfilt:apple", + "qt", "spellCheckWithWordbreak", + "indent", "true", + SpellCheckComponent.COMPONENT_NAME, "true", + SpellCheckComponent.SPELLCHECK_ACCURACY, ".75", + SpellCheckComponent.SPELLCHECK_EXTENDED_RESULTS, "true", + SpellCheckComponent.SPELLCHECK_COLLATE, "true", + SpellCheckComponent.SPELLCHECK_COLLATE_EXTENDED_RESULTS, "true", + SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "10"), + "//lst[@name='collation'][1 ]/str[@name='collationQuery']='lowerfilt:line AND NOT lowerfilt:ample'", + "//lst[@name='collation'][2 ]/str[@name='collationQuery']='lowerfilt:(pi AND ne) AND NOT lowerfilt:ample'" + ); + assertQ(req( + "q", "lowerfilt:pine NOT lowerfilt:apple", + "qt", "spellCheckWithWordbreak", + "indent", "true", + SpellCheckComponent.COMPONENT_NAME, "true", + SpellCheckComponent.SPELLCHECK_ACCURACY, ".75", + SpellCheckComponent.SPELLCHECK_EXTENDED_RESULTS, "true", + SpellCheckComponent.SPELLCHECK_COLLATE, "true", + SpellCheckComponent.SPELLCHECK_COLLATE_EXTENDED_RESULTS, "true", + SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "10"), + "//lst[@name='collation'][1 ]/str[@name='collationQuery']='lowerfilt:line NOT lowerfilt:ample'", + "//lst[@name='collation'][2 ]/str[@name='collationQuery']='lowerfilt:(pi AND ne) NOT lowerfilt:ample'" + ); + assertQ(req( + "q", "lowerfilt:(+pine -apple)", + "qt", "spellCheckWithWordbreak", + "indent", "true", + SpellCheckComponent.COMPONENT_NAME, "true", + SpellCheckComponent.SPELLCHECK_ACCURACY, ".75", + SpellCheckComponent.SPELLCHECK_EXTENDED_RESULTS, "true", + SpellCheckComponent.SPELLCHECK_COLLATE, "true", + SpellCheckComponent.SPELLCHECK_COLLATE_EXTENDED_RESULTS, "true", + SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "10"), + "//lst[@name='collation'][1 ]/str[@name='collationQuery']='lowerfilt:(+line -ample)'", + "//lst[@name='collation'][2 ]/str[@name='collationQuery']='lowerfilt:((+pi +ne) -ample)'" + ); + assertQ(req( + "q", "lowerfilt:(+printableinpuntableplantable)", + "qt", "spellCheckWithWordbreak", + "indent", "true", + SpellCheckComponent.COMPONENT_NAME, "true", + SpellCheckComponent.SPELLCHECK_ACCURACY, "1", + SpellCheckComponent.SPELLCHECK_EXTENDED_RESULTS, "true", + SpellCheckComponent.SPELLCHECK_COLLATE, "true", + SpellCheckComponent.SPELLCHECK_COLLATE_EXTENDED_RESULTS, "true", + SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "1"), + "//lst[@name='collation'][1 ]/str[@name='collationQuery']='lowerfilt:((+printable +in +puntable +plantable))'" + ); + } +} diff --git a/solr/core/src/test/org/apache/solr/spelling/suggest/SuggesterFSTTest.java b/solr/core/src/test/org/apache/solr/spelling/suggest/SuggesterFSTTest.java index 5c9f7d9..e5d92ea 100644 --- a/solr/core/src/test/org/apache/solr/spelling/suggest/SuggesterFSTTest.java +++ b/solr/core/src/test/org/apache/solr/spelling/suggest/SuggesterFSTTest.java @@ -1,6 +1,6 @@ package org.apache.solr.spelling.suggest; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/spelling/suggest/SuggesterTSTTest.java b/solr/core/src/test/org/apache/solr/spelling/suggest/SuggesterTSTTest.java index 008c99a..c85a3b9 100644 --- a/solr/core/src/test/org/apache/solr/spelling/suggest/SuggesterTSTTest.java +++ b/solr/core/src/test/org/apache/solr/spelling/suggest/SuggesterTSTTest.java @@ -1,6 +1,6 @@ package org.apache.solr.spelling.suggest; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/spelling/suggest/SuggesterTest.java b/solr/core/src/test/org/apache/solr/spelling/suggest/SuggesterTest.java index 212b403..c4c55c4 100644 --- a/solr/core/src/test/org/apache/solr/spelling/suggest/SuggesterTest.java +++ b/solr/core/src/test/org/apache/solr/spelling/suggest/SuggesterTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -43,7 +43,7 @@ public class SuggesterTest extends SolrTestCaseJ4 { } @AfterClass - public static void afterClass() throws Exception { + public static void afterClass() { if (savedFactory == null) { System.clearProperty("solr.directoryFactory"); } else { @@ -51,7 +51,7 @@ public class SuggesterTest extends SolrTestCaseJ4 { } } - public static void addDocs() throws Exception { + public static void addDocs() { assertU(adoc("id", "1", "text", "acceptable accidentally accommodate acquire" )); diff --git a/solr/core/src/test/org/apache/solr/spelling/suggest/SuggesterWFSTTest.java b/solr/core/src/test/org/apache/solr/spelling/suggest/SuggesterWFSTTest.java index 435c071..8b36f67 100644 --- a/solr/core/src/test/org/apache/solr/spelling/suggest/SuggesterWFSTTest.java +++ b/solr/core/src/test/org/apache/solr/spelling/suggest/SuggesterWFSTTest.java @@ -1,6 +1,6 @@ package org.apache.solr.spelling.suggest; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/spelling/suggest/TestPhraseSuggestions.java b/solr/core/src/test/org/apache/solr/spelling/suggest/TestPhraseSuggestions.java index 5641936..28bba18 100644 --- a/solr/core/src/test/org/apache/solr/spelling/suggest/TestPhraseSuggestions.java +++ b/solr/core/src/test/org/apache/solr/spelling/suggest/TestPhraseSuggestions.java @@ -1,6 +1,6 @@ package org.apache.solr.spelling.suggest; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java b/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java index fcf0042..20117a1 100644 --- a/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java +++ b/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.util.ContentStream; import org.apache.solr.common.util.ContentStreamBase; @@ -104,6 +105,7 @@ class NewSearcherListener implements SolrEventListener { } } +@Slow public class AutoCommitTest extends AbstractSolrTestCase { @Override diff --git a/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerOptimizeTest.java b/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerOptimizeTest.java index cabb362..0ebc5f9 100644 --- a/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerOptimizeTest.java +++ b/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerOptimizeTest.java @@ -1,5 +1,5 @@ package org.apache.solr.update; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java b/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java index 2fe7620..a2da229 100644 --- a/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,7 +18,6 @@ package org.apache.solr.update; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -51,7 +50,7 @@ public class DirectUpdateHandlerTest extends SolrTestCaseJ4 { } @AfterClass - public static void afterClass() throws Exception { + public static void afterClass() { if (savedFactory == null) { System.clearProperty("solr.directoryFactory"); } else { diff --git a/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java b/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java index 7a54db4..79be74f 100644 --- a/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java +++ b/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java b/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java index cb3a571..fcdc968 100644 --- a/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java +++ b/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java @@ -1,6 +1,6 @@ package org.apache.solr.update; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/update/SoftAutoCommitTest.java b/solr/core/src/test/org/apache/solr/update/SoftAutoCommitTest.java index 221f95a..52d9a52 100644 --- a/solr/core/src/test/org/apache/solr/update/SoftAutoCommitTest.java +++ b/solr/core/src/test/org/apache/solr/update/SoftAutoCommitTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -25,6 +25,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import org.apache.lucene.util.Constants; +import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrEventListener; @@ -51,6 +52,7 @@ import org.junit.Before; * * */ +@Slow public class SoftAutoCommitTest extends AbstractSolrTestCase { @Override diff --git a/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java b/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java index ae29e0d..730f5c4 100644 --- a/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java +++ b/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java @@ -1,6 +1,6 @@ package org.apache.solr.update; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,6 +17,7 @@ package org.apache.solr.update; * limitations under the License. */ +import java.io.File; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -56,7 +57,7 @@ public class SolrCmdDistributorTest extends BaseDistributedSearchTestCase { // TODO: for now we redefine this method so that it pulls from the above // we don't get helpful override behavior due to the method being static protected void createServers(int numShards) throws Exception { - controlJetty = createJetty(testDir, testDir + "/control/data", null, getSolrConfigFile(), getSchemaFile()); + controlJetty = createJetty(new File(getSolrHome()), testDir + "/control/data", null, getSolrConfigFile(), getSchemaFile()); controlClient = createNewSolrServer(controlJetty.getLocalPort()); @@ -64,7 +65,7 @@ public class SolrCmdDistributorTest extends BaseDistributedSearchTestCase { StringBuilder sb = new StringBuilder(); for (int i = 0; i < numShards; i++) { if (sb.length() > 0) sb.append(','); - JettySolrRunner j = createJetty(testDir, + JettySolrRunner j = createJetty(new File(getSolrHome()), testDir + "/shard" + i + "/data", null, getSolrConfigFile(), getSchemaFile()); jettys.add(j); diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java index dd77aed..fed4dce 100644 --- a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java +++ b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java @@ -17,6 +17,8 @@ package org.apache.solr.update; * limitations under the License. */ +import java.io.File; + import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.core.SolrConfig; import org.apache.solr.schema.IndexSchema; @@ -42,10 +44,13 @@ public class SolrIndexConfigTest extends SolrTestCaseJ4 { @Test public void testTieredMPSolrIndexConfigCreation() throws Exception { - SolrConfig solrConfig = new SolrConfig("solrconfig-mergepolicy.xml"); - SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null, null); + SolrConfig solrConfig = new SolrConfig("solr" + File.separator + + "collection1", "solrconfig-mergepolicy.xml", null); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null, + null); assertNotNull(solrIndexConfig); - assertEquals("org.apache.lucene.index.TieredMergePolicy", solrIndexConfig.defaultMergePolicyClassName); + assertEquals("org.apache.lucene.index.TieredMergePolicy", + solrIndexConfig.defaultMergePolicyClassName); IndexSchema indexSchema = new IndexSchema(solrConfig, "schema.xml", null); solrIndexConfig.toIndexWriterConfig(indexSchema); } diff --git a/solr/core/src/test/org/apache/solr/update/TestIndexingPerformance.java b/solr/core/src/test/org/apache/solr/update/TestIndexingPerformance.java index 23fd687..4e9d16b 100755 --- a/solr/core/src/test/org/apache/solr/update/TestIndexingPerformance.java +++ b/solr/core/src/test/org/apache/solr/update/TestIndexingPerformance.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -41,12 +41,12 @@ public class TestIndexingPerformance extends AbstractSolrTestCase { // TODO: fix this test to not require FSDirectory static String savedFactory; @BeforeClass - public static void beforeClass() throws Exception { + public static void beforeClass() { savedFactory = System.getProperty("solr.DirectoryFactory"); System.setProperty("solr.directoryFactory", "org.apache.solr.core.MockFSDirectoryFactory"); } @AfterClass - public static void afterClass() throws Exception { + public static void afterClass() { if (savedFactory == null) { System.clearProperty("solr.directoryFactory"); } else { diff --git a/solr/core/src/test/org/apache/solr/update/TestUpdate.java b/solr/core/src/test/org/apache/solr/update/TestUpdate.java index f178e65..b9a6575 100644 --- a/solr/core/src/test/org/apache/solr/update/TestUpdate.java +++ b/solr/core/src/test/org/apache/solr/update/TestUpdate.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/update/UpdateParamsTest.java b/solr/core/src/test/org/apache/solr/update/UpdateParamsTest.java index 8b3e537..dc38c84 100644 --- a/solr/core/src/test/org/apache/solr/update/UpdateParamsTest.java +++ b/solr/core/src/test/org/apache/solr/update/UpdateParamsTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/update/processor/CustomUpdateRequestProcessorFactory.java b/solr/core/src/test/org/apache/solr/update/processor/CustomUpdateRequestProcessorFactory.java index 351d23a..e9fa3ad 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/CustomUpdateRequestProcessorFactory.java +++ b/solr/core/src/test/org/apache/solr/update/processor/CustomUpdateRequestProcessorFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/update/processor/DefaultValueUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/DefaultValueUpdateProcessorTest.java new file mode 100644 index 0000000..7099285 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/update/processor/DefaultValueUpdateProcessorTest.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update.processor; + +import java.util.Date; +import java.util.UUID; +import java.util.Arrays; +import java.io.IOException; + +import org.apache.solr.SolrTestCaseJ4; + +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.SolrInputField; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; + +import org.apache.solr.core.SolrCore; + +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.LocalSolrQueryRequest; +import org.apache.solr.request.SolrRequestInfo; +import org.apache.solr.response.SolrQueryResponse; + +import org.apache.solr.update.AddUpdateCommand; +import org.apache.solr.update.processor.UpdateRequestProcessor; +import org.apache.solr.update.processor.UpdateRequestProcessorChain; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class DefaultValueUpdateProcessorTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig-update-processor-chains.xml", "schema12.xml"); + } + + public void testDefaults() throws Exception { + SolrInputDocument d = null; + + Date now = new Date(); + + // get all defaults + d = processAdd("default-values", + doc(f("id", "1111"), + f("name", "Existing", "Values"))); + + assertNotNull(d); + + assertEquals("X", d.getFieldValue("processor_default_s")); + assertEquals(42, d.getFieldValue("processor_default_i")); + assertNotNull(d.getFieldValue("uuid")); + assertNotNull(UUID.fromString(d.getFieldValue("uuid").toString())); + assertNotNull(d.getFieldValue("timestamp")); + assertTrue("timestamp not a date: " + + d.getFieldValue("timestamp").getClass(), + d.getFieldValue("timestamp") instanceof Date); + assertEquals(Arrays.asList("Existing","Values"), + d.getFieldValues("name")); + + // defaults already specified + d = processAdd("default-values", + doc(f("id", "1111"), + f("timestamp", now), + f("uuid", "550e8400-e29b-41d4-a716-446655440000"), + f("processor_default_s", "I HAVE A VALUE"), + f("processor_default_i", 12345), + f("name", "Existing", "Values"))); + + assertNotNull(d); + + assertEquals("I HAVE A VALUE", d.getFieldValue("processor_default_s")); + assertEquals(12345, d.getFieldValue("processor_default_i")); + assertEquals("550e8400-e29b-41d4-a716-446655440000", + d.getFieldValue("uuid")); + assertEquals(now, d.getFieldValue("timestamp")); + assertEquals(Arrays.asList("Existing","Values"), + d.getFieldValues("name")); + } + + + /** + * Convenience method for building up SolrInputDocuments + */ + SolrInputDocument doc(SolrInputField... fields) { + SolrInputDocument d = new SolrInputDocument(); + for (SolrInputField f : fields) { + d.put(f.getName(), f); + } + return d; + } + + /** + * Convenience method for building up SolrInputFields + */ + SolrInputField field(String name, float boost, Object... values) { + SolrInputField f = new SolrInputField(name); + for (Object v : values) { + f.addValue(v, 1.0F); + } + f.setBoost(boost); + return f; + } + + /** + * Convenience method for building up SolrInputFields with default boost + */ + SolrInputField f(String name, Object... values) { + return field(name, 1.0F, values); + } + + + /** + * Runs a document through the specified chain, and returns the final + * document used when the chain is completed (NOTE: some chains may + * modify the document in place + */ + SolrInputDocument processAdd(final String chain, + final SolrInputDocument docIn) + throws IOException { + + SolrCore core = h.getCore(); + UpdateRequestProcessorChain pc = core.getUpdateProcessingChain(chain); + assertNotNull("No Chain named: " + chain, pc); + + SolrQueryResponse rsp = new SolrQueryResponse(); + + SolrQueryRequest req = new LocalSolrQueryRequest + (core, new ModifiableSolrParams()); + try { + SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req,rsp)); + AddUpdateCommand cmd = new AddUpdateCommand(req); + cmd.solrDoc = docIn; + + UpdateRequestProcessor processor = pc.createProcessor(req, rsp); + processor.processAdd(cmd); + + return cmd.solrDoc; + } finally { + SolrRequestInfo.clearRequestInfo(); + req.close(); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/update/processor/FieldMutatingUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/FieldMutatingUpdateProcessorTest.java index 778ff23..de9e21f 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/FieldMutatingUpdateProcessorTest.java +++ b/solr/core/src/test/org/apache/solr/update/processor/FieldMutatingUpdateProcessorTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -34,6 +34,7 @@ import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.core.SolrCore; +import org.apache.solr.schema.IndexSchema; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.LocalSolrQueryRequest; @@ -86,13 +87,8 @@ public class FieldMutatingUpdateProcessorTest extends SolrTestCaseJ4 { ,"//long[@name='first_foo_l'][.='"+count+"']" ,"//long[@name='min_foo_l'][.='-34']" ); - - - } - - public void testTrimAll() throws Exception { SolrInputDocument d = null; @@ -559,6 +555,213 @@ public class FieldMutatingUpdateProcessorTest extends SolrTestCaseJ4 { } + public void testTruncate() throws Exception { + SolrInputDocument d = null; + + d = processAdd("truncate", + doc(f("id", "1111"), + f("trunc", "123456789", "", 42, "abcd"))); + + assertNotNull(d); + + assertEquals(Arrays.asList("12345", "", 42, "abcd"), + d.getFieldValues("trunc")); + } + + public void testIgnore() throws Exception { + + IndexSchema schema = h.getCore().getSchema(); + assertNull("test expects 'foo_giberish' to not be a valid field, looks like schema was changed out from under us", + schema.getFieldTypeNoEx("foo_giberish")); + assertNotNull("test expects 't_raw' to be a valid field, looks like schema was changed out from under us", + schema.getFieldTypeNoEx("t_raw")); + assertNotNull("test expects 'foo_s' to be a valid field, looks like schema was changed out from under us", + schema.getFieldTypeNoEx("foo_s")); + + SolrInputDocument d = null; + + d = processAdd("ignore-not-in-schema", + doc(f("id", "1111"), + f("foo_giberish", "123456789", "", 42, "abcd"), + f("t_raw", "123456789", "", 42, "abcd"), + f("foo_s", "hoss"))); + + assertNotNull(d); + assertFalse(d.containsKey("foo_giberish")); + assertEquals(Arrays.asList("123456789", "", 42, "abcd"), + d.getFieldValues("t_raw")); + assertEquals("hoss", d.getFieldValue("foo_s")); + + d = processAdd("ignore-some", + doc(f("id", "1111"), + f("foo_giberish", "123456789", "", 42, "abcd"), + f("t_raw", "123456789", "", 42, "abcd"), + f("foo_s", "hoss"))); + + assertNotNull(d); + assertEquals(Arrays.asList("123456789", "", 42, "abcd"), + d.getFieldValues("foo_giberish")); + assertFalse(d.containsKey("t_raw")); + assertEquals("hoss", d.getFieldValue("foo_s")); + + + } + + public void testCloneField() throws Exception { + + SolrInputDocument d = null; + + // regardless of chain, all of these should be equivilent + for (String chain : Arrays.asList("clone-single", "clone-multi", + "clone-array","clone-selector" )) { + + // simple clone + d = processAdd(chain, + doc(f("id", "1111"), + f("source0_s", "NOT COPIED"), + f("source1_s", "123456789", "", 42, "abcd"))); + assertNotNull(chain, d); + assertEquals(chain, + Arrays.asList("123456789", "", 42, "abcd"), + d.getFieldValues("source1_s")); + assertEquals(chain, + Arrays.asList("123456789", "", 42, "abcd"), + d.getFieldValues("dest_s")); + + // append to existing values, preserve boost + d = processAdd(chain, + doc(f("id", "1111"), + field("dest_s", 2.3f, "orig1", "orig2"), + f("source0_s", "NOT COPIED"), + f("source1_s", "123456789", "", 42, "abcd"))); + assertNotNull(chain, d); + assertEquals(chain, + Arrays.asList("123456789", "", 42, "abcd"), + d.getFieldValues("source1_s")); + assertEquals(chain, + Arrays.asList("orig1", "orig2", "123456789", "", 42, "abcd"), + d.getFieldValues("dest_s")); + assertEquals(chain + ": dest boost changed", + 2.3f, d.getField("dest_s").getBoost(), 0.0f); + } + + // should be equivilent for any chain matching source1_s and source2_s + for (String chain : Arrays.asList("clone-multi", + "clone-array","clone-selector" )) { + + // simple clone + d = processAdd(chain, + doc(f("id", "1111"), + f("source0_s", "NOT COPIED"), + f("source1_s", "123456789", "", 42, "abcd"), + f("source2_s", "xxx", 999))); + assertNotNull(chain, d); + assertEquals(chain, + Arrays.asList("123456789", "", 42, "abcd"), + d.getFieldValues("source1_s")); + assertEquals(chain, + Arrays.asList("xxx", 999), + d.getFieldValues("source2_s")); + assertEquals(chain, + Arrays.asList("123456789", "", 42, "abcd", "xxx", 999), + d.getFieldValues("dest_s")); + + // append to existing values, preserve boost + d = processAdd(chain, + doc(f("id", "1111"), + field("dest_s", 2.3f, "orig1", "orig2"), + f("source0_s", "NOT COPIED"), + f("source1_s", "123456789", "", 42, "abcd"), + f("source2_s", "xxx", 999))); + assertNotNull(chain, d); + assertEquals(chain, + Arrays.asList("123456789", "", 42, "abcd"), + d.getFieldValues("source1_s")); + assertEquals(chain, + Arrays.asList("xxx", 999), + d.getFieldValues("source2_s")); + assertEquals(chain, + Arrays.asList("orig1", "orig2", + "123456789", "", 42, "abcd", + "xxx", 999), + d.getFieldValues("dest_s")); + assertEquals(chain + ": dest boost changed", + 2.3f, d.getField("dest_s").getBoost(), 0.0f); + } + } + + public void testCloneFieldExample() throws Exception { + + SolrInputDocument d = null; + + // test example from the javadocs + d = processAdd("multiple-clones", + doc(f("id", "1111"), + f("category", "misc"), + f("authors", "Isaac Asimov", "John Brunner"), + f("editors", "John W. Campbell"), + f("store1_price", 87), + f("store2_price", 78), + f("list_price", 1000))); + assertNotNull(d); + assertEquals("misc",d.getFieldValue("category")); + assertEquals("misc",d.getFieldValue("category_s")); + assertEquals(Arrays.asList("Isaac Asimov", "John Brunner"), + d.getFieldValues("authors")); + assertEquals(Arrays.asList("John W. Campbell"), + d.getFieldValues("editors")); + assertEquals(Arrays.asList("Isaac Asimov", "John Brunner", + "John W. Campbell"), + d.getFieldValues("contributors")); + assertEquals(87,d.getFieldValue("store1_price")); + assertEquals(78,d.getFieldValue("store2_price")); + assertEquals(1000,d.getFieldValue("list_price")); + assertEquals(Arrays.asList(87, 78), + d.getFieldValues("all_prices")); + + } + + public void testCloneCombinations() throws Exception { + + SolrInputDocument d = null; + + // maxChars + d = processAdd("clone-max-chars", + doc(f("id", "1111"), + f("field1", "text"))); + assertNotNull(d); + assertEquals("text",d.getFieldValue("field1")); + assertEquals("tex",d.getFieldValue("toField")); + + // move + d = processAdd("clone-move", + doc(f("id", "1111"), + f("field1", "text"))); + assertNotNull(d); + assertEquals("text",d.getFieldValue("toField")); + assertFalse(d.containsKey("field1")); + + // replace + d = processAdd("clone-replace", + doc(f("id", "1111"), + f("toField", "IGNORED"), + f("field1", "text"))); + assertNotNull(d); + assertEquals("text", d.getFieldValue("field1")); + assertEquals("text", d.getFieldValue("toField")); + + // append + d = processAdd("clone-append", + doc(f("id", "1111"), + f("toField", "aaa"), + f("field1", "bbb"), + f("field2", "ccc"))); + assertNotNull(d); + assertEquals("bbb", d.getFieldValue("field1")); + assertEquals("ccc", d.getFieldValue("field2")); + assertEquals("aaa; bbb; ccc", d.getFieldValue("toField")); + } + public void testConcatDefaults() throws Exception { SolrInputDocument d = null; d = processAdd("concat-defaults", diff --git a/solr/core/src/test/org/apache/solr/update/processor/SignatureUpdateProcessorFactoryTest.java b/solr/core/src/test/org/apache/solr/update/processor/SignatureUpdateProcessorFactoryTest.java index 9c5b018..1f2a7c3 100755 --- a/solr/core/src/test/org/apache/solr/update/processor/SignatureUpdateProcessorFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/update/processor/SignatureUpdateProcessorFactoryTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/update/processor/URLClassifyProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/URLClassifyProcessorTest.java index df92fc9..a3eae49 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/URLClassifyProcessorTest.java +++ b/solr/core/src/test/org/apache/solr/update/processor/URLClassifyProcessorTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/update/processor/UniqFieldsUpdateProcessorFactoryTest.java b/solr/core/src/test/org/apache/solr/update/processor/UniqFieldsUpdateProcessorFactoryTest.java index e503dc1..30ecbde 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/UniqFieldsUpdateProcessorFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/update/processor/UniqFieldsUpdateProcessorFactoryTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/update/processor/UpdateRequestProcessorFactoryTest.java b/solr/core/src/test/org/apache/solr/update/processor/UpdateRequestProcessorFactoryTest.java index e3ed6e5..e032612 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/UpdateRequestProcessorFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/update/processor/UpdateRequestProcessorFactoryTest.java @@ -43,10 +43,17 @@ public class UpdateRequestProcessorFactoryTest extends AbstractSolrTestCase { // make sure it loaded the factories UpdateRequestProcessorChain chained = core.getUpdateProcessingChain( "standard" ); - // Make sure it got 3 items and configured the Log chain ok - assertEquals( 3, chained.getFactories().length ); - LogUpdateProcessorFactory log = (LogUpdateProcessorFactory)chained.getFactories()[0]; - assertEquals( 100, log.maxNumToLog ); + // Make sure it got 3 items (4 configured, 1 is enable=false) + assertEquals("wrong number of (enabled) factories in chain", + 3, chained.getFactories().length ); + + // first one should be log, and it should be configured properly + UpdateRequestProcessorFactory first = chained.getFactories()[0]; + assertEquals("wrong factory at front of chain", + LogUpdateProcessorFactory.class, first.getClass()); + LogUpdateProcessorFactory log = (LogUpdateProcessorFactory)first; + assertEquals("wrong config for LogUpdateProcessorFactory", + 100, log.maxNumToLog ); UpdateRequestProcessorChain custom = core.getUpdateProcessingChain( null ); diff --git a/solr/core/src/test/org/apache/solr/util/BitSetPerf.java b/solr/core/src/test/org/apache/solr/util/BitSetPerf.java index f7324f4..7fc7089 100644 --- a/solr/core/src/test/org/apache/solr/util/BitSetPerf.java +++ b/solr/core/src/test/org/apache/solr/util/BitSetPerf.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/util/CircularListTest.java b/solr/core/src/test/org/apache/solr/util/CircularListTest.java index c781ca8..b0c6c06 100644 --- a/solr/core/src/test/org/apache/solr/util/CircularListTest.java +++ b/solr/core/src/test/org/apache/solr/util/CircularListTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/util/DOMUtilTest.java b/solr/core/src/test/org/apache/solr/util/DOMUtilTest.java index e7c7333..8b4fc9c 100644 --- a/solr/core/src/test/org/apache/solr/util/DOMUtilTest.java +++ b/solr/core/src/test/org/apache/solr/util/DOMUtilTest.java @@ -19,7 +19,6 @@ package org.apache.solr.util; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; -import org.apache.solr.util.DOMUtilTestBase; public class DOMUtilTest extends DOMUtilTestBase { @@ -49,7 +48,7 @@ public class DOMUtilTest extends DOMUtilTestBase { assertTypeAndValue( namedList, "Boolean", false ); } - private void assertTypeAndValue( NamedList namedList, String key, Object value ) throws Exception { + private void assertTypeAndValue( NamedList namedList, String key, Object value ) { Object v = namedList.get( key ); assertNotNull( v ); assertEquals( key, v.getClass().getSimpleName() ); diff --git a/solr/core/src/test/org/apache/solr/util/DateMathParserTest.java b/solr/core/src/test/org/apache/solr/util/DateMathParserTest.java index 3a03580..fd23fa3 100644 --- a/solr/core/src/test/org/apache/solr/util/DateMathParserTest.java +++ b/solr/core/src/test/org/apache/solr/util/DateMathParserTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/util/SolrPluginUtilsTest.java b/solr/core/src/test/org/apache/solr/util/SolrPluginUtilsTest.java index 6be5f37..6449171 100644 --- a/solr/core/src/test/org/apache/solr/util/SolrPluginUtilsTest.java +++ b/solr/core/src/test/org/apache/solr/util/SolrPluginUtilsTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/util/TestNumberUtils.java b/solr/core/src/test/org/apache/solr/util/TestNumberUtils.java index db27b39..96639c6 100644 --- a/solr/core/src/test/org/apache/solr/util/TestNumberUtils.java +++ b/solr/core/src/test/org/apache/solr/util/TestNumberUtils.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/util/TestSystemIdResolver.java b/solr/core/src/test/org/apache/solr/util/TestSystemIdResolver.java index 01ae4b5..98cce4e 100644 --- a/solr/core/src/test/org/apache/solr/util/TestSystemIdResolver.java +++ b/solr/core/src/test/org/apache/solr/util/TestSystemIdResolver.java @@ -38,8 +38,8 @@ public class TestSystemIdResolver extends LuceneTestCase { } public void testResolving() throws Exception { - final String testHome = SolrTestCaseJ4.getFile("solr/conf").getParent(); - final ResourceLoader loader = new SolrResourceLoader(testHome, this.getClass().getClassLoader()); + final String testHome = SolrTestCaseJ4.getFile("solr/collection1").getParent(); + final ResourceLoader loader = new SolrResourceLoader(testHome + "/collection1", this.getClass().getClassLoader()); final SystemIdResolver resolver = new SystemIdResolver(loader); final String fileUri = new File(testHome+"/crazy-path-to-config.xml").toURI().toASCIIString(); @@ -61,8 +61,8 @@ public class TestSystemIdResolver extends LuceneTestCase { assertEntityResolving(resolver, "solrres:/schema.xml", "solrres:/solrconfig.xml", "schema.xml"); assertEntityResolving(resolver, "solrres:/org/apache/solr/util/TestSystemIdResolver.class", "solrres:/org/apache/solr/util/RTimer.class", "TestSystemIdResolver.class"); - assertEntityResolving(resolver, SystemIdResolver.createSystemIdFromResourceName(testHome+"/conf/schema.xml"), - SystemIdResolver.createSystemIdFromResourceName(testHome+"/conf/solrconfig.xml"), "schema.xml"); + assertEntityResolving(resolver, SystemIdResolver.createSystemIdFromResourceName(testHome+"/collection1/conf/schema.xml"), + SystemIdResolver.createSystemIdFromResourceName(testHome+"/collection1/conf/solrconfig.xml"), "schema.xml"); assertEntityResolving(resolver, SystemIdResolver.createSystemIdFromResourceName(testHome+"/crazy-path-to-schema.xml"), SystemIdResolver.createSystemIdFromResourceName(testHome+"/crazy-path-to-config.xml"), "crazy-path-to-schema.xml"); diff --git a/solr/core/src/test/org/apache/solr/util/TestUtils.java b/solr/core/src/test/org/apache/solr/util/TestUtils.java index 4d3d38c..4c06f0b 100755 --- a/solr/core/src/test/org/apache/solr/util/TestUtils.java +++ b/solr/core/src/test/org/apache/solr/util/TestUtils.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/core/src/test/org/apache/solr/util/TimeZoneUtilsTest.java b/solr/core/src/test/org/apache/solr/util/TimeZoneUtilsTest.java index 5eae30b..033ebb6 100644 --- a/solr/core/src/test/org/apache/solr/util/TimeZoneUtilsTest.java +++ b/solr/core/src/test/org/apache/solr/util/TimeZoneUtilsTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/solr/example/example-DIH/solr/db/conf/db-data-config.xml b/solr/example/example-DIH/solr/db/conf/db-data-config.xml index 9fb10ea..096377d 100644 --- a/solr/example/example-DIH/solr/db/conf/db-data-config.xml +++ b/solr/example/example-DIH/solr/db/conf/db-data-config.xml @@ -4,8 +4,6 @@ - - - LUCENE_40 + LUCENE_50 @@ -258,7 +258,7 @@ - LUCENE_40 + LUCENE_50 @@ -283,7 +283,7 @@ diff --git a/solr/example/example-DIH/solr/rss/conf/solrconfig.xml b/solr/example/example-DIH/solr/rss/conf/solrconfig.xml index 08d8628..0a3b4f7 100644 --- a/solr/example/example-DIH/solr/rss/conf/solrconfig.xml +++ b/solr/example/example-DIH/solr/rss/conf/solrconfig.xml @@ -24,7 +24,7 @@ that you fully re-index after changing this setting as it can affect both how text is indexed and queried. --> - LUCENE_40 + LUCENE_50 @@ -258,7 +258,7 @@ - LUCENE_40 + LUCENE_50 @@ -258,7 +258,7 @@ - LUCENE_40 + LUCENE_50 @@ -222,7 +222,7 @@ - LUCENE_40 + LUCENE_50 diff --git a/solr/example/multicore/core1/conf/solrconfig.xml b/solr/example/multicore/core1/conf/solrconfig.xml index 13c59fb..2cda4d0 100644 --- a/solr/example/multicore/core1/conf/solrconfig.xml +++ b/solr/example/multicore/core1/conf/solrconfig.xml @@ -21,7 +21,7 @@ It is *not* a good example to work from. --> - LUCENE_40 + LUCENE_50 diff --git a/solr/example/solr/collection1/conf/admin-extra.html b/solr/example/solr/collection1/conf/admin-extra.html new file mode 100644 index 0000000..aa739da --- /dev/null +++ b/solr/example/solr/collection1/conf/admin-extra.html @@ -0,0 +1,31 @@ + + + diff --git a/solr/example/solr/collection1/conf/admin-extra.menu-bottom.html b/solr/example/solr/collection1/conf/admin-extra.menu-bottom.html new file mode 100644 index 0000000..82404ae --- /dev/null +++ b/solr/example/solr/collection1/conf/admin-extra.menu-bottom.html @@ -0,0 +1 @@ + diff --git a/solr/example/solr/collection1/conf/admin-extra.menu-top.html b/solr/example/solr/collection1/conf/admin-extra.menu-top.html new file mode 100644 index 0000000..9807205 --- /dev/null +++ b/solr/example/solr/collection1/conf/admin-extra.menu-top.html @@ -0,0 +1 @@ + diff --git a/solr/example/solr/collection1/conf/currency.xml b/solr/example/solr/collection1/conf/currency.xml new file mode 100644 index 0000000..3a9c58a --- /dev/null +++ b/solr/example/solr/collection1/conf/currency.xml @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/solr/example/solr/collection1/conf/elevate.xml b/solr/example/solr/collection1/conf/elevate.xml new file mode 100644 index 0000000..25d5ceb --- /dev/null +++ b/solr/example/solr/collection1/conf/elevate.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + diff --git a/solr/example/solr/collection1/conf/lang/contractions_ca.txt b/solr/example/solr/collection1/conf/lang/contractions_ca.txt new file mode 100644 index 0000000..307a85f --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/contractions_ca.txt @@ -0,0 +1,8 @@ +# Set of Catalan contractions for ElisionFilter +# TODO: load this as a resource from the analyzer and sync it in build.xml +d +l +m +n +s +t diff --git a/solr/example/solr/collection1/conf/lang/contractions_fr.txt b/solr/example/solr/collection1/conf/lang/contractions_fr.txt new file mode 100644 index 0000000..722db58 --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/contractions_fr.txt @@ -0,0 +1,9 @@ +# Set of French contractions for ElisionFilter +# TODO: load this as a resource from the analyzer and sync it in build.xml +l +m +t +qu +n +s +j diff --git a/solr/example/solr/collection1/conf/lang/contractions_ga.txt b/solr/example/solr/collection1/conf/lang/contractions_ga.txt new file mode 100644 index 0000000..9ebe7fa --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/contractions_ga.txt @@ -0,0 +1,5 @@ +# Set of Irish contractions for ElisionFilter +# TODO: load this as a resource from the analyzer and sync it in build.xml +d +m +b diff --git a/solr/example/solr/collection1/conf/lang/contractions_it.txt b/solr/example/solr/collection1/conf/lang/contractions_it.txt new file mode 100644 index 0000000..cac0409 --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/contractions_it.txt @@ -0,0 +1,23 @@ +# Set of Italian contractions for ElisionFilter +# TODO: load this as a resource from the analyzer and sync it in build.xml +c +l +all +dall +dell +nell +sull +coll +pell +gl +agl +dagl +degl +negl +sugl +un +m +t +s +v +d diff --git a/solr/example/solr/collection1/conf/lang/hyphenations_ga.txt b/solr/example/solr/collection1/conf/lang/hyphenations_ga.txt new file mode 100644 index 0000000..4d2642c --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/hyphenations_ga.txt @@ -0,0 +1,5 @@ +# Set of Irish hyphenations for StopFilter +# TODO: load this as a resource from the analyzer and sync it in build.xml +h +n +t diff --git a/solr/example/solr/collection1/conf/lang/stemdict_nl.txt b/solr/example/solr/collection1/conf/lang/stemdict_nl.txt new file mode 100644 index 0000000..4410729 --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stemdict_nl.txt @@ -0,0 +1,6 @@ +# Set of overrides for the dutch stemmer +# TODO: load this as a resource from the analyzer and sync it in build.xml +fiets fiets +bromfiets bromfiets +ei eier +kind kinder diff --git a/solr/example/solr/collection1/conf/lang/stoptags_ja.txt b/solr/example/solr/collection1/conf/lang/stoptags_ja.txt new file mode 100644 index 0000000..71b7508 --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stoptags_ja.txt @@ -0,0 +1,420 @@ +# +# This file defines a Japanese stoptag set for JapanesePartOfSpeechStopFilter. +# +# Any token with a part-of-speech tag that exactly matches those defined in this +# file are removed from the token stream. +# +# Set your own stoptags by uncommenting the lines below. Note that comments are +# not allowed on the same line as a stoptag. See LUCENE-3745 for frequency lists, +# etc. that can be useful for building you own stoptag set. +# +# The entire possible tagset is provided below for convenience. +# +##### +# noun: unclassified nouns +#åè©ž +# +# noun-common: Common nouns or nouns where the sub-classification is undefined +#åè©ž-一般 +# +# noun-proper: Proper nouns where the sub-classification is undefined +#åè©ž-固有åè©ž +# +# noun-proper-misc: miscellaneous proper nouns +#åè©ž-固有åè©ž-一般 +# +# noun-proper-person: Personal names where the sub-classification is undefined +#åè©ž-固有åè©ž-人å +# +# noun-proper-person-misc: names that cannot be divided into surname and +# given name; foreign names; names where the surname or given name is unknown. +# e.g. ãŠå¸‚ã®æ–¹ +#åè©ž-固有åè©ž-人å-一般 +# +# noun-proper-person-surname: Mainly Japanese surnames. +# e.g. 山田 +#åè©ž-固有åè©ž-人å-姓 +# +# noun-proper-person-given_name: Mainly Japanese given names. +# e.g. 太郎 +#åè©ž-固有åè©ž-人å-å +# +# noun-proper-organization: Names representing organizations. +# e.g. 通産çœ, NHK +#åè©ž-固有åè©ž-組織 +# +# noun-proper-place: Place names where the sub-classification is undefined +#åè©ž-固有åè©ž-地域 +# +# noun-proper-place-misc: Place names excluding countries. +# e.g. アジア, ãƒãƒ«ã‚»ãƒ­ãƒŠ, 京都 +#åè©ž-固有åè©ž-地域-一般 +# +# noun-proper-place-country: Country names. +# e.g. 日本, オーストラリア +#åè©ž-固有åè©ž-地域-国 +# +# noun-pronoun: Pronouns where the sub-classification is undefined +#åè©ž-代åè©ž +# +# noun-pronoun-misc: miscellaneous pronouns: +# e.g. ãã‚Œ, ã“ã“, ã‚ã„ã¤, ã‚ãªãŸ, ã‚ã¡ã“ã¡, ã„ãã¤, ã©ã“ã‹, ãªã«, ã¿ãªã•ã‚“, ã¿ã‚“ãª, ã‚ãŸãã—, ã‚ã‚Œã‚ã‚Œ +#åè©ž-代åè©ž-一般 +# +# noun-pronoun-contraction: Spoken language contraction made by combining a +# pronoun and the particle 'wa'. +# e.g. ã‚りゃ, ã“りゃ, ã“りゃã‚, ãりゃ, ãりゃ゠+#åè©ž-代åè©ž-縮約 +# +# noun-adverbial: Temporal nouns such as names of days or months that behave +# like adverbs. Nouns that represent amount or ratios and can be used adverbially, +# e.g. 金曜, 一月, åˆå¾Œ, å°‘é‡ +#åè©ž-副詞å¯èƒ½ +# +# noun-verbal: Nouns that take arguments with case and can appear followed by +# 'suru' and related verbs (ã™ã‚‹, ã§ãã‚‹, ãªã•ã‚‹, ãã ã•ã‚‹) +# e.g. インプット, æ„›ç€, 悪化, 悪戦苦闘, 一安心, 下å–ã‚Š +#åè©ž-サ変接続 +# +# noun-adjective-base: The base form of adjectives, words that appear before 㪠("na") +# e.g. å¥åº·, 安易, 駄目, ã ã‚ +#åè©ž-形容動詞語幹 +# +# noun-numeric: Arabic numbers, Chinese numerals, and counters like 何 (回), æ•°. +# e.g. 0, 1, 2, 何, æ•°, å¹¾ +#åè©ž-æ•° +# +# noun-affix: noun affixes where the sub-classification is undefined +#åè©ž-éžè‡ªç«‹ +# +# noun-affix-misc: Of adnominalizers, the case-marker ã® ("no"), and words that +# attach to the base form of inflectional words, words that cannot be classified +# into any of the other categories below. This category includes indefinite nouns. +# e.g. ã‚ã‹ã¤ã, æš, ã‹ã„, 甲æ–, æ°—, ãらã„, å«Œã„, ãã›, ç™–, ã“ã¨, 事, ã”ã¨, 毎, ã—ã ã„, 次第, +# é †, ã›ã„, 所為, ã¤ã„ã§, åºã§, ã¤ã‚‚ã‚Š, ç©ã‚‚ã‚Š, 点, ã©ã“ã‚, ã®, ã¯ãš, ç­ˆ, ã¯ãšã¿, å¼¾ã¿, +# æ‹å­, ãµã†, ãµã‚Š, 振り, ã»ã†, æ–¹, æ—¨, ã‚‚ã®, 物, 者, ゆãˆ, æ•…, ゆãˆã‚“, 所以, ã‚ã‘, 訳, +# ã‚ã‚Š, 割り, 割, ã‚“-å£èªž/, ã‚‚ã‚“-å£èªž/ +#åè©ž-éžè‡ªç«‹-一般 +# +# noun-affix-adverbial: noun affixes that that can behave as adverbs. +# e.g. ã‚ã„ã , é–“, ã‚ã’ã, 挙ã’å¥, ã‚ã¨, 後, 余り, 以外, 以é™, 以後, 以上, 以å‰, 一方, ã†ãˆ, +# 上, ã†ã¡, 内, ãŠã‚Š, 折り, ã‹ãŽã‚Š, é™ã‚Š, ãã‚Š, ã£ãã‚Š, çµæžœ, ã“ã‚, é ƒ, ã•ã„, éš›, 最中, ã•ãªã‹, +# 最中, ã˜ãŸã„, 自体, ãŸã³, 度, ãŸã‚, 為, ã¤ã©, 都度, ã¨ãŠã‚Š, 通り, ã¨ã, 時, ã¨ã“ã‚, 所, +# ã¨ãŸã‚“, 途端, ãªã‹, 中, ã®ã¡, 後, ã°ã‚ã„, å ´åˆ, æ—¥, ã¶ã‚“, 分, ã»ã‹, ä»–, ã¾ãˆ, å‰, ã¾ã¾, +# 儘, ä¾­, ã¿ãŽã‚Š, 矢先 +#åè©ž-éžè‡ªç«‹-副詞å¯èƒ½ +# +# noun-affix-aux: noun affixes treated as 助動詞 ("auxiliary verb") in school grammars +# with the stem よã†(ã ) ("you(da)"). +# e.g. よã†, ã‚„ã†, 様 (よã†) +#åè©ž-éžè‡ªç«‹-助動詞語幹 +# +# noun-affix-adjective-base: noun affixes that can connect to the indeclinable +# connection form 㪠(aux "da"). +# e.g. ã¿ãŸã„, ãµã† +#åè©ž-éžè‡ªç«‹-形容動詞語幹 +# +# noun-special: special nouns where the sub-classification is undefined. +#åè©ž-特殊 +# +# noun-special-aux: The ãã†ã  ("souda") stem form that is used for reporting news, is +# treated as 助動詞 ("auxiliary verb") in school grammars, and attach to the base +# form of inflectional words. +# e.g. ãㆠ+#åè©ž-特殊-助動詞語幹 +# +# noun-suffix: noun suffixes where the sub-classification is undefined. +#åè©ž-接尾 +# +# noun-suffix-misc: Of the nouns or stem forms of other parts of speech that connect +# to ガル or タイ and can combine into compound nouns, words that cannot be classified into +# any of the other categories below. In general, this category is more inclusive than +# 接尾語 ("suffix") and is usually the last element in a compound noun. +# e.g. ãŠã, ã‹ãŸ, æ–¹, ç”²æ– (ãŒã„), ãŒã‹ã‚Š, ãŽã¿, 気味, ãã‚‹ã¿, (～ã—ãŸ) ã•, 次第, 済 (ãš) ã¿, +# よã†, (ã§ã)ã£ã“, æ„Ÿ, 観, 性, å­¦, é¡ž, é¢, 用 +#åè©ž-接尾-一般 +# +# noun-suffix-person: Suffixes that form nouns and attach to person names more often +# than other nouns. +# e.g. å›, 様, è‘— +#åè©ž-接尾-人å +# +# noun-suffix-place: Suffixes that form nouns and attach to place names more often +# than other nouns. +# e.g. 町, 市, 県 +#åè©ž-接尾-地域 +# +# noun-suffix-verbal: Of the suffixes that attach to nouns and form nouns, those that +# can appear before スル ("suru"). +# e.g. 化, 視, 分ã‘, 入り, è½ã¡, è²·ã„ +#åè©ž-接尾-サ変接続 +# +# noun-suffix-aux: The stem form of ãã†ã  (様態) that is used to indicate conditions, +# is treated as 助動詞 ("auxiliary verb") in school grammars, and attach to the +# conjunctive form of inflectional words. +# e.g. ãㆠ+#åè©ž-接尾-助動詞語幹 +# +# noun-suffix-adjective-base: Suffixes that attach to other nouns or the conjunctive +# form of inflectional words and appear before the copula ã  ("da"). +# e.g. çš„, ã’, ãŒã¡ +#åè©ž-接尾-形容動詞語幹 +# +# noun-suffix-adverbial: Suffixes that attach to other nouns and can behave as adverbs. +# e.g. 後 (ã”), 以後, 以é™, 以å‰, å‰å¾Œ, 中, 末, 上, 時 (ã˜) +#åè©ž-接尾-副詞å¯èƒ½ +# +# noun-suffix-classifier: Suffixes that attach to numbers and form nouns. This category +# is more inclusive than 助数詞 ("classifier") and includes common nouns that attach +# to numbers. +# e.g. 個, ã¤, 本, 冊, パーセント, cm, kg, カ月, ã‹å›½, 区画, 時間, æ™‚åŠ +#åè©ž-接尾-助数詞 +# +# noun-suffix-special: Special suffixes that mainly attach to inflecting words. +# e.g. (楽ã—) ã•, (考ãˆ) æ–¹ +#åè©ž-接尾-特殊 +# +# noun-suffix-conjunctive: Nouns that behave like conjunctions and join two words +# together. +# e.g. (日本) 対 (アメリカ), 対 (アメリカ), (3) 対 (5), (女優) å…¼ (主婦) +#åè©ž-接続詞的 +# +# noun-verbal_aux: Nouns that attach to the conjunctive particle 㦠("te") and are +# semantically verb-like. +# e.g. ã”らん, ã”覧, 御覧, 頂戴 +#åè©ž-å‹•è©žéžè‡ªç«‹çš„ +# +# noun-quotation: text that cannot be segmented into words, proverbs, Chinese poetry, +# dialects, English, etc. Currently, the only entry for åè©ž 引用文字列 ("noun quotation") +# is ã„ã‚ã ("iwaku"). +#åè©ž-引用文字列 +# +# noun-nai_adjective: Words that appear before the auxiliary verb ãªã„ ("nai") and +# behave like an adjective. +# e.g. 申ã—訳, 仕方, ã¨ã‚“ã§ã‚‚, é•ã„ +#åè©ž-ナイ形容詞語幹 +# +##### +# prefix: unclassified prefixes +#接頭詞 +# +# prefix-nominal: Prefixes that attach to nouns (including adjective stem forms) +# excluding numerical expressions. +# e.g. ㊠(æ°´), æŸ (æ°), åŒ (社), æ•… (～æ°), 高 (å“質), ㊠(見事), ã” (ç«‹æ´¾) +#接頭詞-å詞接続 +# +# prefix-verbal: Prefixes that attach to the imperative form of a verb or a verb +# in conjunctive form followed by ãªã‚‹/ãªã•ã‚‹/ãã ã•ã‚‹. +# e.g. ㊠(読ã¿ãªã•ã„), ㊠(座り) +#接頭詞-動詞接続 +# +# prefix-adjectival: Prefixes that attach to adjectives. +# e.g. ㊠(寒ã„ã§ã™ã­ãˆ), ãƒã‚« (ã§ã‹ã„) +#接頭詞-形容詞接続 +# +# prefix-numerical: Prefixes that attach to numerical expressions. +# e.g. ç´„, ãŠã‚ˆã, 毎時 +#接頭詞-数接続 +# +##### +# verb: unclassified verbs +#å‹•è©ž +# +# verb-main: +#å‹•è©ž-自立 +# +# verb-auxiliary: +#å‹•è©ž-éžè‡ªç«‹ +# +# verb-suffix: +#å‹•è©ž-接尾 +# +##### +# adjective: unclassified adjectives +#形容詞 +# +# adjective-main: +#形容詞-自立 +# +# adjective-auxiliary: +#形容詞-éžè‡ªç«‹ +# +# adjective-suffix: +#形容詞-接尾 +# +##### +# adverb: unclassified adverbs +#副詞 +# +# adverb-misc: Words that can be segmented into one unit and where adnominal +# modification is not possible. +# e.g. ã‚ã„ã‹ã‚らãš, 多分 +#副詞-一般 +# +# adverb-particle_conjunction: Adverbs that can be followed by ã®, ã¯, ã«, +# ãª, ã™ã‚‹, ã , etc. +# e.g. ã“ã‚“ãªã«, ãã‚“ãªã«, ã‚ã‚“ãªã«, ãªã«ã‹, ãªã‚“ã§ã‚‚ +#副詞-助詞類接続 +# +##### +# adnominal: Words that only have noun-modifying forms. +# e.g. ã“ã®, ãã®, ã‚ã®, ã©ã®, ã„ã‚ゆる, ãªã‚“らã‹ã®, 何らã‹ã®, ã„ã‚ã‚“ãª, ã“ã†ã„ã†, ãã†ã„ã†, ã‚ã‚ã„ã†, +# ã©ã†ã„ã†, ã“ã‚“ãª, ãã‚“ãª, ã‚ã‚“ãª, ã©ã‚“ãª, 大ããª, å°ã•ãª, ãŠã‹ã—ãª, ã»ã‚“ã®, ãŸã„ã—ãŸ, +# 「(, ã‚‚) ã•ã‚‹ (ã“ã¨ãªãŒã‚‰)ã€, 微々ãŸã‚‹, 堂々ãŸã‚‹, å˜ãªã‚‹, ã„ã‹ãªã‚‹, 我ãŒã€ã€ŒåŒã˜, 亡ã +#連体詞 +# +##### +# conjunction: Conjunctions that can occur independently. +# e.g. ãŒ, ã‘ã‚Œã©ã‚‚, ãã—ã¦, ã˜ã‚ƒã‚, ãã‚Œã©ã“ã‚ã‹ +接続詞 +# +##### +# particle: unclassified particles. +助詞 +# +# particle-case: case particles where the subclassification is undefined. +助詞-格助詞 +# +# particle-case-misc: Case particles. +# e.g. ã‹ã‚‰, ãŒ, ã§, ã¨, ã«, ã¸, より, ã‚’, ã®, ã«ã¦ +助詞-格助詞-一般 +# +# particle-case-quote: the "to" that appears after nouns, a person’s speech, +# quotation marks, expressions of decisions from a meeting, reasons, judgements, +# conjectures, etc. +# e.g. ( ã ) 㨠(è¿°ã¹ãŸ.), ( ã§ã‚ã‚‹) 㨠(ã—ã¦åŸ·è¡ŒçŒ¶äºˆ...) +助詞-格助詞-引用 +# +# particle-case-compound: Compounds of particles and verbs that mainly behave +# like case particles. +# e.g. ã¨ã„ã†, ã¨ã„ã£ãŸ, ã¨ã‹ã„ã†, ã¨ã—ã¦, ã¨ã¨ã‚‚ã«, ã¨å…±ã«, ã§ã‚‚ã£ã¦, ã«ã‚ãŸã£ã¦, ã«å½“ãŸã£ã¦, ã«å½“ã£ã¦, +# ã«ã‚ãŸã‚Š, ã«å½“ãŸã‚Š, ã«å½“ã‚Š, ã«å½“ãŸã‚‹, ã«ã‚ãŸã‚‹, ã«ãŠã„ã¦, ã«æ–¼ã„ã¦,ã«æ–¼ã¦, ã«ãŠã‘ã‚‹, ã«æ–¼ã‘ã‚‹, +# ã«ã‹ã‘, ã«ã‹ã‘ã¦, ã«ã‹ã‚“ã—, ã«é–¢ã—, ã«ã‹ã‚“ã—ã¦, ã«é–¢ã—ã¦, ã«ã‹ã‚“ã™ã‚‹, ã«é–¢ã™ã‚‹, ã«éš›ã—, +# ã«éš›ã—ã¦, ã«ã—ãŸãŒã„, ã«å¾“ã„, ã«å¾“ã†, ã«ã—ãŸãŒã£ã¦, ã«å¾“ã£ã¦, ã«ãŸã„ã—, ã«å¯¾ã—, ã«ãŸã„ã—ã¦, +# ã«å¯¾ã—ã¦, ã«ãŸã„ã™ã‚‹, ã«å¯¾ã™ã‚‹, ã«ã¤ã„ã¦, ã«ã¤ã, ã«ã¤ã‘, ã«ã¤ã‘ã¦, ã«ã¤ã‚Œ, ã«ã¤ã‚Œã¦, ã«ã¨ã£ã¦, +# ã«ã¨ã‚Š, ã«ã¾ã¤ã‚ã‚‹, ã«ã‚ˆã£ã¦, ã«ä¾ã£ã¦, ã«å› ã£ã¦, ã«ã‚ˆã‚Š, ã«ä¾ã‚Š, ã«å› ã‚Š, ã«ã‚ˆã‚‹, ã«ä¾ã‚‹, ã«å› ã‚‹, +# ã«ã‚ãŸã£ã¦, ã«ã‚ãŸã‚‹, ã‚’ã‚‚ã£ã¦, を以ã£ã¦, を通ã˜, を通ã˜ã¦, を通ã—ã¦, ã‚’ã‚ãã£ã¦, ã‚’ã‚ãã‚Š, ã‚’ã‚ãã‚‹, +# ã£ã¦-å£èªž/, ã¡ã‚…ã†-関西å¼ã€Œã¨ã„ã†ã€/, (何) ã¦ã„ㆠ(人)-å£èªž/, ã£ã¦ã„ã†-å£èªž/, ã¨ã„ãµ, ã¨ã‹ã„ãµ +助詞-格助詞-連語 +# +# particle-conjunctive: +# e.g. ã‹ã‚‰, ã‹ã‚‰ã«ã¯, ãŒ, ã‘ã‚Œã©, ã‘ã‚Œã©ã‚‚, ã‘ã©, ã—, ã¤ã¤, ã¦, ã§, ã¨, ã¨ã“ã‚ãŒ, ã©ã“ã‚ã‹, ã¨ã‚‚, ã©ã‚‚, +# ãªãŒã‚‰, ãªã‚Š, ã®ã§, ã®ã«, ã°, ã‚‚ã®ã®, ã‚„ ( ã—ãŸ), ã‚„ã„ãªã‚„, (ã“ã‚ã‚“) ã˜ã‚ƒ(ã„ã‘ãªã„)-å£èªž/, +# (è¡Œã£) ã¡ã‚ƒ(ã„ã‘ãªã„)-å£èªž/, (言ã£) ãŸã£ã¦ (ã—ã‹ãŸãŒãªã„)-å£èªž/, (ãã‚ŒãŒãªã)ã£ãŸã£ã¦ (平気)-å£èªž/ +助詞-接続助詞 +# +# particle-dependency: +# e.g. ã“ã, ã•ãˆ, ã—ã‹, ã™ã‚‰, ã¯, ã‚‚, ãž +助詞-係助詞 +# +# particle-adverbial: +# e.g. ãŒã¦ã‚‰, ã‹ã‚‚, ãらã„, ä½, ãらã„, ã—ã‚‚, (学校) ã˜ã‚ƒ(ã“ã‚ŒãŒæµè¡Œã£ã¦ã„ã‚‹)-å£èªž/, +# (ãã‚Œ)ã˜ã‚ƒã‚ (よããªã„)-å£èªž/, ãšã¤, (ç§) ãªãž, ãªã©, (ç§) ãªã‚Š (ã«), (先生) ãªã‚“ã‹ (大嫌ã„)-å£èªž/, +# (ç§) ãªã‚“ãž, (先生) ãªã‚“㦠(大嫌ã„)-å£èªž/, ã®ã¿, ã ã‘, (ç§) ã ã£ã¦-å£èªž/, ã ã«, +# (å½¼)ã£ãŸã‚‰-å£èªž/, (ãŠèŒ¶) ã§ã‚‚ (ã„ã‹ãŒ), ç­‰ (ã¨ã†), (今後) ã¨ã‚‚, ã°ã‹ã‚Š, ã°ã£ã‹-å£èªž/, ã°ã£ã‹ã‚Š-å£èªž/, +# ã»ã©, 程, ã¾ã§, è¿„, (誰) ã‚‚ (ãŒ)([助詞-格助詞] ãŠã‚ˆã³ [助詞-係助詞] ã®å‰ã«ä½ç½®ã™ã‚‹ã€Œã‚‚ã€) +助詞-副助詞 +# +# particle-interjective: particles with interjective grammatical roles. +# e.g. (æ¾å³¶) ã‚„ +助詞-間投助詞 +# +# particle-coordinate: +# e.g. ã¨, ãŸã‚Š, ã ã®, ã ã‚Š, ã¨ã‹, ãªã‚Š, ã‚„, やら +助詞-並立助詞 +# +# particle-final: +# e.g. ã‹ã„, ã‹ã—ら, ã•, ãœ, (ã )ã£ã‘-å£èªž/, (ã¨ã¾ã£ã¦ã‚‹) ã§-方言/, ãª, ナ, ãªã‚-å£èªž/, ãž, ã­, ãƒ, +# ã­ã‡-å£èªž/, ã­ãˆ-å£èªž/, ã­ã‚“-方言/, ã®, ã®ã†-å£èªž/, ã‚„, よ, ヨ, よã‰-å£èªž/, ã‚, ã‚ã„-å£èªž/ +助詞-終助詞 +# +# particle-adverbial/conjunctive/final: The particle "ka" when unknown whether it is +# adverbial, conjunctive, or sentence final. For example: +# (a) 「A ã‹ B ã‹ã€. Ex:「(国内ã§é‹ç”¨ã™ã‚‹) ã‹,(海外ã§é‹ç”¨ã™ã‚‹) ã‹ (.)〠+# (b) Inside an adverb phrase. Ex:「(幸ã„ã¨ã„ã†) ã‹ (, 死者ã¯ã„ãªã‹ã£ãŸ.)〠+# 「(祈りãŒå±Šã„ãŸã›ã„) ã‹ (, 試験ã«åˆæ ¼ã—ãŸ.)〠+# (c) 「ã‹ã®ã‚ˆã†ã«ã€. Ex:「(何もãªã‹ã£ãŸ) ã‹ (ã®ã‚ˆã†ã«æŒ¯ã‚‹èˆžã£ãŸ.)〠+# e.g. ã‹ +助詞-副助詞ï¼ä¸¦ç«‹åŠ©è©žï¼çµ‚助詞 +# +# particle-adnominalizer: The "no" that attaches to nouns and modifies +# non-inflectional words. +助詞-連体化 +# +# particle-adnominalizer: The "ni" and "to" that appear following nouns and adverbs +# that are giongo, giseigo, or gitaigo. +# e.g. ã«, 㨠+助詞-副詞化 +# +# particle-special: A particle that does not fit into one of the above classifications. +# This includes particles that are used in Tanka, Haiku, and other poetry. +# e.g. ã‹ãª, ã‘ã‚€, ( ã—ãŸã ã‚ã†) ã«, (ã‚ã‚“ãŸ) ã«ã‚ƒ(ã‚ã‹ã‚‰ã‚“), (俺) ã‚“ (家) +助詞-特殊 +# +##### +# auxiliary-verb: +助動詞 +# +##### +# interjection: Greetings and other exclamations. +# e.g. ãŠã¯ã‚ˆã†, ãŠã¯ã‚ˆã†ã”ã–ã„ã¾ã™, ã“ã‚“ã«ã¡ã¯, ã“ã‚“ã°ã‚“ã¯, ã‚ã‚ŠãŒã¨ã†, ã©ã†ã‚‚ã‚ã‚ŠãŒã¨ã†, ã‚ã‚ŠãŒã¨ã†ã”ã–ã„ã¾ã™, +# ã„ãŸã ãã¾ã™, ã”ã¡ãã†ã•ã¾, ã•ã‚ˆãªã‚‰, ã•ã‚ˆã†ãªã‚‰, ã¯ã„, ã„ã„ãˆ, ã”ã‚ã‚“, ã”ã‚ã‚“ãªã•ã„ +#æ„Ÿå‹•è©ž +# +##### +# symbol: unclassified Symbols. +è¨˜å· +# +# symbol-misc: A general symbol not in one of the categories below. +# e.g. [â—‹â—Ž@$〒→+] +記å·-一般 +# +# symbol-comma: Commas +# e.g. [,ã€] +記å·-読点 +# +# symbol-period: Periods and full stops. +# e.g. [.．。] +記å·-å¥ç‚¹ +# +# symbol-space: Full-width whitespace. +記å·-空白 +# +# symbol-open_bracket: +# e.g. [({‘“『ã€] +記å·-括弧開 +# +# symbol-close_bracket: +# e.g. [)}’â€ã€ã€ã€‘] +記å·-括弧閉 +# +# symbol-alphabetic: +#記å·-アルファベット +# +##### +# other: unclassified other +#ãã®ä»– +# +# other-interjection: Words that are hard to classify as noun-suffixes or +# sentence-final particles. +# e.g. (ã )ã‚¡ +ãã®ä»–-間投 +# +##### +# filler: Aizuchi that occurs during a conversation or sounds inserted as filler. +# e.g. ã‚ã®, ã†ã‚“ã¨, ãˆã¨ +フィラー +# +##### +# non-verbal: non-verbal sound. +éžè¨€èªžéŸ³ +# +##### +# fragment: +#語断片 +# +##### +# unknown: unknown part of speech. +#未知語 +# +##### End of file diff --git a/solr/example/solr/collection1/conf/lang/stopwords_ar.txt b/solr/example/solr/collection1/conf/lang/stopwords_ar.txt new file mode 100644 index 0000000..046829d --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_ar.txt @@ -0,0 +1,125 @@ +# This file was created by Jacques Savoy and is distributed under the BSD license. +# See http://members.unine.ch/jacques.savoy/clef/index.html. +# Also see http://www.opensource.org/licenses/bsd-license.html +# Cleaned on October 11, 2009 (not normalized, so use before normalization) +# This means that when modifying this list, you might need to add some +# redundant entries, for example containing forms with both Ø£ and ا +من +ومن +منها +منه +ÙÙŠ +ÙˆÙÙŠ +Ùيها +Ùيه +Ùˆ +Ù +ثم +او +أو +ب +بها +به +ا +Ø£ +اى +اي +أي +أى +لا +ولا +الا +ألا +إلا +لكن +ما +وما +كما +Ùما +عن +مع +اذا +إذا +ان +أن +إن +انها +أنها +إنها +انه +أنه +إنه +بان +بأن +Ùان +Ùأن +وان +وأن +وإن +التى +التي +الذى +الذي +الذين +الى +الي +إلى +إلي +على +عليها +عليه +اما +أما +إما +ايضا +أيضا +كل +وكل +لم +ولم +لن +ولن +هى +هي +هو +وهى +وهي +وهو +Ùهى +Ùهي +Ùهو +انت +أنت +لك +لها +له +هذه +هذا +تلك +ذلك +هناك +كانت +كان +يكون +تكون +وكانت +وكان +غير +بعض +قد +نحو +بين +بينما +منذ +ضمن +حيث +الان +الآن +خلال +بعد +قبل +حتى +عند +عندما +لدى +جميع diff --git a/solr/example/solr/collection1/conf/lang/stopwords_bg.txt b/solr/example/solr/collection1/conf/lang/stopwords_bg.txt new file mode 100644 index 0000000..1ae4ba2 --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_bg.txt @@ -0,0 +1,193 @@ +# This file was created by Jacques Savoy and is distributed under the BSD license. +# See http://members.unine.ch/jacques.savoy/clef/index.html. +# Also see http://www.opensource.org/licenses/bsd-license.html +а +аз +ако +ала +бе +без +беше +би +бил +била +били +било +близо +бъдат +бъде +бÑха +в +Ð²Ð°Ñ +ваш +ваша +вероÑтно +вече +взема +ви +вие +винаги +вÑе +вÑеки +вÑички +вÑичко +вÑÑка +във +въпреки +върху +г +ги +главно +го +д +да +дали +до +докато +докога +дори +доÑега +доÑта +е +едва +един +ето +за +зад +заедно +заради +заÑега +затова +защо +защото +и +из +или +им +има +имат +иÑка +й +каза +как +каква +какво +както +какъв +като +кога +когато +което +които +кой +който +колко +коÑто +къде +където +към +ли +м +ме +между +мен +ми +мнозина +мога +могат +може +Ð¼Ð¾Ð»Ñ +момента +му +н +на +над +назад +най +направи +напред +например +Ð½Ð°Ñ +не +него +Ð½ÐµÑ +ни +ние +никой +нито +но +нÑкои +нÑкой +нÑма +обаче +около +оÑвен +оÑобено +от +отгоре +отново +още +пак +по +повече +повечето +под +поне +поради +поÑле +почти +прави +пред +преди +през +при +пък +първо +Ñ +Ñа +Ñамо +Ñе +Ñега +Ñи +Ñкоро +Ñлед +Ñме +Ñпоред +Ñред +Ñрещу +Ñте +Ñъм +ÑÑŠÑ +Ñъщо +Ñ‚ +тази +така +такива +такъв +там +твой +те +тези +ти +тн +то +това +тогава +този +той +толкова +точно +Ñ‚Ñ€Ñбва +тук +тъй +Ñ‚Ñ +Ñ‚ÑÑ… +у +хареÑва +ч +че +чеÑто +чрез +ще +щом +Ñ diff --git a/solr/example/solr/collection1/conf/lang/stopwords_ca.txt b/solr/example/solr/collection1/conf/lang/stopwords_ca.txt new file mode 100644 index 0000000..3da65de --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_ca.txt @@ -0,0 +1,220 @@ +# Catalan stopwords from http://github.com/vcl/cue.language (Apache 2 Licensed) +a +abans +ací +ah +així +això +al +als +aleshores +algun +alguna +algunes +alguns +alhora +allà +allí +allò +altra +altre +altres +amb +ambdós +ambdues +apa +aquell +aquella +aquelles +aquells +aquest +aquesta +aquestes +aquests +aquí +baix +cada +cadascú +cadascuna +cadascunes +cadascuns +com +contra +d'un +d'una +d'unes +d'uns +dalt +de +del +dels +des +després +dins +dintre +donat +doncs +durant +e +eh +el +els +em +en +encara +ens +entre +érem +eren +éreu +es +és +esta +està +estàvem +estaven +estàveu +esteu +et +etc +ets +fins +fora +gairebé +ha +han +has +havia +he +hem +heu +hi +ho +i +igual +iguals +ja +l'hi +la +les +li +li'n +llavors +m'he +ma +mal +malgrat +mateix +mateixa +mateixes +mateixos +me +mentre +més +meu +meus +meva +meves +molt +molta +moltes +molts +mon +mons +n'he +n'hi +ne +ni +no +nogensmenys +només +nosaltres +nostra +nostre +nostres +o +oh +oi +on +pas +pel +pels +per +però +perquè +poc +poca +pocs +poques +potser +propi +qual +quals +quan +quant +que +què +quelcom +qui +quin +quina +quines +quins +s'ha +s'han +sa +semblant +semblants +ses +seu +seus +seva +seva +seves +si +sobre +sobretot +sóc +solament +sols +son +són +sons +sota +sou +t'ha +t'han +t'he +ta +tal +també +tampoc +tan +tant +tanta +tantes +teu +teus +teva +teves +ton +tons +tot +tota +totes +tots +un +una +unes +uns +us +va +vaig +vam +van +vas +veu +vosaltres +vostra +vostre +vostres diff --git a/solr/example/solr/collection1/conf/lang/stopwords_cz.txt b/solr/example/solr/collection1/conf/lang/stopwords_cz.txt new file mode 100644 index 0000000..53c6097 --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_cz.txt @@ -0,0 +1,172 @@ +a +s +k +o +i +u +v +z +dnes +cz +tímto +budeÅ¡ +budem +byli +jseÅ¡ +můj +svým +ta +tomto +tohle +tuto +tyto +jej +zda +proÄ +máte +tato +kam +tohoto +kdo +kteří +mi +nám +tom +tomuto +mít +nic +proto +kterou +byla +toho +protože +asi +ho +naÅ¡i +napiÅ¡te +re +což +tím +takže +svých +její +svými +jste +aj +tu +tedy +teto +bylo +kde +ke +pravé +ji +nad +nejsou +Äi +pod +téma +mezi +pÅ™es +ty +pak +vám +ani +když +vÅ¡ak +neg +jsem +tento +Älánku +Älánky +aby +jsme +pÅ™ed +pta +jejich +byl +jeÅ¡tÄ› +až +bez +také +pouze +první +vaÅ¡e +která +nás +nový +tipy +pokud +může +strana +jeho +své +jiné +zprávy +nové +není +vás +jen +podle +zde +už +být +více +bude +již +než +který +by +které +co +nebo +ten +tak +má +pÅ™i +od +po +jsou +jak +další +ale +si +se +ve +to +jako +za +zpÄ›t +ze +do +pro +je +na +atd +atp +jakmile +pÅ™iÄemž +já +on +ona +ono +oni +ony +my +vy +jí +ji +mÄ› +mne +jemu +tomu +tÄ›m +tÄ›mu +nÄ›mu +nÄ›muž +jehož +jíž +jelikož +jež +jakož +naÄež diff --git a/solr/example/solr/collection1/conf/lang/stopwords_da.txt b/solr/example/solr/collection1/conf/lang/stopwords_da.txt new file mode 100644 index 0000000..a3ff5fe --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_da.txt @@ -0,0 +1,108 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/danish/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + + | A Danish stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | This is a ranked list (commonest to rarest) of stopwords derived from + | a large text sample. + + +og | and +i | in +jeg | I +det | that (dem. pronoun)/it (pers. pronoun) +at | that (in front of a sentence)/to (with infinitive) +en | a/an +den | it (pers. pronoun)/that (dem. pronoun) +til | to/at/for/until/against/by/of/into, more +er | present tense of "to be" +som | who, as +pÃ¥ | on/upon/in/on/at/to/after/of/with/for, on +de | they +med | with/by/in, along +han | he +af | of/by/from/off/for/in/with/on, off +for | at/for/to/from/by/of/ago, in front/before, because +ikke | not +der | who/which, there/those +var | past tense of "to be" +mig | me/myself +sig | oneself/himself/herself/itself/themselves +men | but +et | a/an/one, one (number), someone/somebody/one +har | present tense of "to have" +om | round/about/for/in/a, about/around/down, if +vi | we +min | my +havde | past tense of "to have" +ham | him +hun | she +nu | now +over | over/above/across/by/beyond/past/on/about, over/past +da | then, when/as/since +fra | from/off/since, off, since +du | you +ud | out +sin | his/her/its/one's +dem | them +os | us/ourselves +op | up +man | you/one +hans | his +hvor | where +eller | or +hvad | what +skal | must/shall etc. +selv | myself/youself/herself/ourselves etc., even +her | here +alle | all/everyone/everybody etc. +vil | will (verb) +blev | past tense of "to stay/to remain/to get/to become" +kunne | could +ind | in +nÃ¥r | when +være | present tense of "to be" +dog | however/yet/after all +noget | something +ville | would +jo | you know/you see (adv), yes +deres | their/theirs +efter | after/behind/according to/for/by/from, later/afterwards +ned | down +skulle | should +denne | this +end | than +dette | this +mit | my/mine +ogsÃ¥ | also +under | under/beneath/below/during, below/underneath +have | have +dig | you +anden | other +hende | her +mine | my +alt | everything +meget | much/very, plenty of +sit | his, her, its, one's +sine | his, her, its, one's +vor | our +mod | against +disse | these +hvis | if +din | your/yours +nogle | some +hos | by/at +blive | be/become +mange | many +ad | by/through +bliver | present tense of "to be/to become" +hendes | her/hers +været | be +thi | for (conj) +jer | you +sÃ¥dan | such, like this/like that diff --git a/solr/example/solr/collection1/conf/lang/stopwords_de.txt b/solr/example/solr/collection1/conf/lang/stopwords_de.txt new file mode 100644 index 0000000..f770384 --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_de.txt @@ -0,0 +1,292 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/german/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + + | A German stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | The number of forms in this list is reduced significantly by passing it + | through the German stemmer. + + +aber | but + +alle | all +allem +allen +aller +alles + +als | than, as +also | so +am | an + dem +an | at + +ander | other +andere +anderem +anderen +anderer +anderes +anderm +andern +anderr +anders + +auch | also +auf | on +aus | out of +bei | by +bin | am +bis | until +bist | art +da | there +damit | with it +dann | then + +der | the +den +des +dem +die +das + +daß | that + +derselbe | the same +derselben +denselben +desselben +demselben +dieselbe +dieselben +dasselbe + +dazu | to that + +dein | thy +deine +deinem +deinen +deiner +deines + +denn | because + +derer | of those +dessen | of him + +dich | thee +dir | to thee +du | thou + +dies | this +diese +diesem +diesen +dieser +dieses + + +doch | (several meanings) +dort | (over) there + + +durch | through + +ein | a +eine +einem +einen +einer +eines + +einig | some +einige +einigem +einigen +einiger +einiges + +einmal | once + +er | he +ihn | him +ihm | to him + +es | it +etwas | something + +euer | your +eure +eurem +euren +eurer +eures + +für | for +gegen | towards +gewesen | p.p. of sein +hab | have +habe | have +haben | have +hat | has +hatte | had +hatten | had +hier | here +hin | there +hinter | behind + +ich | I +mich | me +mir | to me + + +ihr | you, to her +ihre +ihrem +ihren +ihrer +ihres +euch | to you + +im | in + dem +in | in +indem | while +ins | in + das +ist | is + +jede | each, every +jedem +jeden +jeder +jedes + +jene | that +jenem +jenen +jener +jenes + +jetzt | now +kann | can + +kein | no +keine +keinem +keinen +keiner +keines + +können | can +könnte | could +machen | do +man | one + +manche | some, many a +manchem +manchen +mancher +manches + +mein | my +meine +meinem +meinen +meiner +meines + +mit | with +muss | must +musste | had to +nach | to(wards) +nicht | not +nichts | nothing +noch | still, yet +nun | now +nur | only +ob | whether +oder | or +ohne | without +sehr | very + +sein | his +seine +seinem +seinen +seiner +seines + +selbst | self +sich | herself + +sie | they, she +ihnen | to them + +sind | are +so | so + +solche | such +solchem +solchen +solcher +solches + +soll | shall +sollte | should +sondern | but +sonst | else +über | over +um | about, around +und | and + +uns | us +unse +unsem +unsen +unser +unses + +unter | under +viel | much +vom | von + dem +von | from +vor | before +während | while +war | was +waren | were +warst | wast +was | what +weg | away, off +weil | because +weiter | further + +welche | which +welchem +welchen +welcher +welches + +wenn | when +werde | will +werden | will +wie | how +wieder | again +will | want +wir | we +wird | will +wirst | willst +wo | where +wollen | want +wollte | wanted +würde | would +würden | would +zu | to +zum | zu + dem +zur | zu + der +zwar | indeed +zwischen | between + diff --git a/solr/example/solr/collection1/conf/lang/stopwords_el.txt b/solr/example/solr/collection1/conf/lang/stopwords_el.txt new file mode 100644 index 0000000..232681f --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_el.txt @@ -0,0 +1,78 @@ +# Lucene Greek Stopwords list +# Note: by default this file is used after GreekLowerCaseFilter, +# so when modifying this file use 'σ' instead of 'Ï‚' +ο +η +το +οι +τα +του +τησ +των +τον +την +και +κι +κ +ειμαι +εισαι +ειναι +ειμαστε +ειστε +στο +στον +στη +στην +μα +αλλα +απο +για +Ï€Ïοσ +με +σε +ωσ +παÏα +αντι +κατα +μετα +θα +να +δε +δεν +μη +μην +επι +ενω +εαν +αν +τοτε +που +πωσ +ποιοσ +ποια +ποιο +ποιοι +ποιεσ +ποιων +ποιουσ +αυτοσ +αυτη +αυτο +αυτοι +αυτων +αυτουσ +αυτεσ +αυτα +εκεινοσ +εκεινη +εκεινο +εκεινοι +εκεινεσ +εκεινα +εκεινων +εκεινουσ +οπωσ +ομωσ +ισωσ +οσο +οτι diff --git a/solr/example/solr/collection1/conf/lang/stopwords_en.txt b/solr/example/solr/collection1/conf/lang/stopwords_en.txt new file mode 100644 index 0000000..2c164c0 --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_en.txt @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +# Standard english stop words taken from Lucene's StopAnalyzer +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +such +that +the +their +then +there +these +they +this +to +was +will +with diff --git a/solr/example/solr/collection1/conf/lang/stopwords_es.txt b/solr/example/solr/collection1/conf/lang/stopwords_es.txt new file mode 100644 index 0000000..2db1476 --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_es.txt @@ -0,0 +1,354 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/spanish/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + + | A Spanish stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + + | The following is a ranked list (commonest to rarest) of stopwords + | deriving from a large sample of text. + + | Extra words have been added at the end. + +de | from, of +la | the, her +que | who, that +el | the +en | in +y | and +a | to +los | the, them +del | de + el +se | himself, from him etc +las | the, them +por | for, by, etc +un | a +para | for +con | with +no | no +una | a +su | his, her +al | a + el + | es from SER +lo | him +como | how +más | more +pero | pero +sus | su plural +le | to him, her +ya | already +o | or + | fue from SER +este | this + | ha from HABER +sí | himself etc +porque | because +esta | this + | son from SER +entre | between + | está from ESTAR +cuando | when +muy | very +sin | without +sobre | on + | ser from SER + | tiene from TENER +también | also +me | me +hasta | until +hay | there is/are +donde | where + | han from HABER +quien | whom, that + | están from ESTAR + | estado from ESTAR +desde | from +todo | all +nos | us +durante | during + | estados from ESTAR +todos | all +uno | a +les | to them +ni | nor +contra | against +otros | other + | fueron from SER +ese | that +eso | that + | había from HABER +ante | before +ellos | they +e | and (variant of y) +esto | this +mí | me +antes | before +algunos | some +qué | what? +unos | a +yo | I +otro | other +otras | other +otra | other +él | he +tanto | so much, many +esa | that +estos | these +mucho | much, many +quienes | who +nada | nothing +muchos | many +cual | who + | sea from SER +poco | few +ella | she +estar | to be + | haber from HABER +estas | these + | estaba from ESTAR + | estamos from ESTAR +algunas | some +algo | something +nosotros | we + + | other forms + +mi | me +mis | mi plural +tú | thou +te | thee +ti | thee +tu | thy +tus | tu plural +ellas | they +nosotras | we +vosotros | you +vosotras | you +os | you +mío | mine +mía | +míos | +mías | +tuyo | thine +tuya | +tuyos | +tuyas | +suyo | his, hers, theirs +suya | +suyos | +suyas | +nuestro | ours +nuestra | +nuestros | +nuestras | +vuestro | yours +vuestra | +vuestros | +vuestras | +esos | those +esas | those + + | forms of estar, to be (not including the infinitive): +estoy +estás +está +estamos +estáis +están +esté +estés +estemos +estéis +estén +estaré +estarás +estará +estaremos +estaréis +estarán +estaría +estarías +estaríamos +estaríais +estarían +estaba +estabas +estábamos +estabais +estaban +estuve +estuviste +estuvo +estuvimos +estuvisteis +estuvieron +estuviera +estuvieras +estuviéramos +estuvierais +estuvieran +estuviese +estuvieses +estuviésemos +estuvieseis +estuviesen +estando +estado +estada +estados +estadas +estad + + | forms of haber, to have (not including the infinitive): +he +has +ha +hemos +habéis +han +haya +hayas +hayamos +hayáis +hayan +habré +habrás +habrá +habremos +habréis +habrán +habría +habrías +habríamos +habríais +habrían +había +habías +habíamos +habíais +habían +hube +hubiste +hubo +hubimos +hubisteis +hubieron +hubiera +hubieras +hubiéramos +hubierais +hubieran +hubiese +hubieses +hubiésemos +hubieseis +hubiesen +habiendo +habido +habida +habidos +habidas + + | forms of ser, to be (not including the infinitive): +soy +eres +es +somos +sois +son +sea +seas +seamos +seáis +sean +seré +serás +será +seremos +seréis +serán +sería +serías +seríamos +seríais +serían +era +eras +éramos +erais +eran +fui +fuiste +fue +fuimos +fuisteis +fueron +fuera +fueras +fuéramos +fuerais +fueran +fuese +fueses +fuésemos +fueseis +fuesen +siendo +sido + | sed also means 'thirst' + + | forms of tener, to have (not including the infinitive): +tengo +tienes +tiene +tenemos +tenéis +tienen +tenga +tengas +tengamos +tengáis +tengan +tendré +tendrás +tendrá +tendremos +tendréis +tendrán +tendría +tendrías +tendríamos +tendríais +tendrían +tenía +tenías +teníamos +teníais +tenían +tuve +tuviste +tuvo +tuvimos +tuvisteis +tuvieron +tuviera +tuvieras +tuviéramos +tuvierais +tuvieran +tuviese +tuvieses +tuviésemos +tuvieseis +tuviesen +teniendo +tenido +tenida +tenidos +tenidas +tened + diff --git a/solr/example/solr/collection1/conf/lang/stopwords_eu.txt b/solr/example/solr/collection1/conf/lang/stopwords_eu.txt new file mode 100644 index 0000000..25f1db9 --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_eu.txt @@ -0,0 +1,99 @@ +# example set of basque stopwords +al +anitz +arabera +asko +baina +bat +batean +batek +bati +batzuei +batzuek +batzuetan +batzuk +bera +beraiek +berau +berauek +bere +berori +beroriek +beste +bezala +da +dago +dira +ditu +du +dute +edo +egin +ere +eta +eurak +ez +gainera +gu +gutxi +guzti +haiei +haiek +haietan +hainbeste +hala +han +handik +hango +hara +hari +hark +hartan +hau +hauei +hauek +hauetan +hemen +hemendik +hemengo +hi +hona +honek +honela +honetan +honi +hor +hori +horiei +horiek +horietan +horko +horra +horrek +horrela +horretan +horri +hortik +hura +izan +ni +noiz +nola +non +nondik +nongo +nor +nora +ze +zein +zen +zenbait +zenbat +zer +zergatik +ziren +zituen +zu +zuek +zuen +zuten diff --git a/solr/example/solr/collection1/conf/lang/stopwords_fa.txt b/solr/example/solr/collection1/conf/lang/stopwords_fa.txt new file mode 100644 index 0000000..723641c --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_fa.txt @@ -0,0 +1,313 @@ +# This file was created by Jacques Savoy and is distributed under the BSD license. +# See http://members.unine.ch/jacques.savoy/clef/index.html. +# Also see http://www.opensource.org/licenses/bsd-license.html +# Note: by default this file is used after normalization, so when adding entries +# to this file, use the arabic 'ÙŠ' instead of 'ÛŒ' +انان +نداشته +سراسر +خياه +ايشان +وي +تاكنون +بيشتري +دوم +پس +ناشي +ÙˆÚ¯Ùˆ +يا +داشتند +سپس +هنگام +هرگز +پنج +نشان +امسال +ديگر +گروهي +شدند +چطور +ده +Ùˆ +دو +نخستين +ولي +چرا +Ú†Ù‡ +وسط +Ù‡ +كدام +قابل +يك +رÙت +Ù‡Ùت +همچنين +در +هزار +بله +بلي +شايد +اما +شناسي +گرÙته +دهد +داشته +دانست +داشتن +خواهيم +ميليارد +وقتيكه +امد +خواهد +جز +اورده +شده +بلكه +خدمات +شدن +برخي +نبود +بسياري +جلوگيري +حق +كردند +نوعي +بعري +نكرده +نظير +نبايد +بوده +بودن +داد +اورد +هست +جايي +شود +دنبال +داده +بايد +سابق +هيچ +همان +انجا +كمتر +كجاست +گردد +كسي +تر +مردم +تان +دادن +بودند +سري +جدا +ندارند +مگر +يكديگر +دارد +دهند +بنابراين +هنگامي +سمت +جا +انچه +خود +دادند +زياد +دارند +اثر +بدون +بهترين +بيشتر +البته +به +براساس +بيرون +كرد +بعضي +گرÙت +توي +اي +ميليون +او +جريان +تول +بر +مانند +برابر +باشيم +مدتي +گويند +اكنون +تا +تنها +جديد +چند +بي +نشده +كردن +كردم +گويد +كرده +كنيم +نمي +نزد +روي +قصد +Ùقط +بالاي +ديگران +اين +ديروز +توسط +سوم +ايم +دانند +سوي +استÙاده +شما +كنار +داريم +ساخته +طور +امده +رÙته +نخست +بيست +نزديك +طي +كنيد +از +انها +تمامي +داشت +يكي +طريق +اش +چيست +روب +نمايد +Ú¯Ùت +چندين +چيزي +تواند +ام +ايا +با +ان +ايد +ترين +اينكه +ديگري +راه +هايي +بروز +همچنان +پاعين +كس +حدود +مختل٠+مقابل +چيز +گيرد +ندارد +ضد +همچون +سازي +شان +مورد +باره +مرسي +خويش +برخوردار +چون +خارج +شش +هنوز +تحت +ضمن +هستيم +Ú¯Ùته +Ùكر +بسيار +پيش +براي +روزهاي +انكه +نخواهد +بالا +كل +وقتي +كي +چنين +كه +گيري +نيست +است +كجا +كند +نيز +يابد +بندي +حتي +توانند +عقب +خواست +كنند +بين +تمام +همه +ما +باشند +مثل +شد +اري +باشد +اره +طبق +بعد +اگر +صورت +غير +جاي +بيش +ريزي +اند +زيرا +چگونه +بار +لطÙا +مي +درباره +من +ديده +همين +گذاري +برداري +علت +گذاشته +هم +Ùوق +نه +ها +شوند +اباد +همواره +هر +اول +خواهند +چهار +نام +امروز +مان +هاي +قبل +كنم +سعي +تازه +را +هستند +زير +جلوي +عنوان +بود diff --git a/solr/example/solr/collection1/conf/lang/stopwords_fi.txt b/solr/example/solr/collection1/conf/lang/stopwords_fi.txt new file mode 100644 index 0000000..addad79 --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_fi.txt @@ -0,0 +1,95 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/finnish/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + +| forms of BE + +olla +olen +olet +on +olemme +olette +ovat +ole | negative form + +oli +olisi +olisit +olisin +olisimme +olisitte +olisivat +olit +olin +olimme +olitte +olivat +ollut +olleet + +en | negation +et +ei +emme +ette +eivät + +|Nom Gen Acc Part Iness Elat Illat Adess Ablat Allat Ess Trans +minä minun minut minua minussa minusta minuun minulla minulta minulle | I +sinä sinun sinut sinua sinussa sinusta sinuun sinulla sinulta sinulle | you +hän hänen hänet häntä hänessä hänestä häneen hänellä häneltä hänelle | he she +me meidän meidät meitä meissä meistä meihin meillä meiltä meille | we +te teidän teidät teitä teissä teistä teihin teillä teiltä teille | you +he heidän heidät heitä heissä heistä heihin heillä heiltä heille | they + +tämä tämän tätä tässä tästä tähän tallä tältä tälle tänä täksi | this +tuo tuon tuotä tuossa tuosta tuohon tuolla tuolta tuolle tuona tuoksi | that +se sen sitä siinä siitä siihen sillä siltä sille sinä siksi | it +nämä näiden näitä näissä näistä näihin näillä näiltä näille näinä näiksi | these +nuo noiden noita noissa noista noihin noilla noilta noille noina noiksi | those +ne niiden niitä niissä niistä niihin niillä niiltä niille niinä niiksi | they + +kuka kenen kenet ketä kenessä kenestä keneen kenellä keneltä kenelle kenenä keneksi| who +ketkä keiden ketkä keitä keissä keistä keihin keillä keiltä keille keinä keiksi | (pl) +mikä minkä minkä mitä missä mistä mihin millä miltä mille minä miksi | which what +mitkä | (pl) + +joka jonka jota jossa josta johon jolla jolta jolle jona joksi | who which +jotka joiden joita joissa joista joihin joilla joilta joille joina joiksi | (pl) + +| conjunctions + +että | that +ja | and +jos | if +koska | because +kuin | than +mutta | but +niin | so +sekä | and +sillä | for +tai | or +vaan | but +vai | or +vaikka | although + + +| prepositions + +kanssa | with +mukaan | according to +noin | about +poikki | across +yli | over, across + +| other + +kun | when +niin | so +nyt | now +itse | self + diff --git a/solr/example/solr/collection1/conf/lang/stopwords_fr.txt b/solr/example/solr/collection1/conf/lang/stopwords_fr.txt new file mode 100644 index 0000000..c00837e --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_fr.txt @@ -0,0 +1,183 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/french/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + + | A French stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + +au | a + le +aux | a + les +avec | with +ce | this +ces | these +dans | with +de | of +des | de + les +du | de + le +elle | she +en | `of them' etc +et | and +eux | them +il | he +je | I +la | the +le | the +leur | their +lui | him +ma | my (fem) +mais | but +me | me +même | same; as in moi-même (myself) etc +mes | me (pl) +moi | me +mon | my (masc) +ne | not +nos | our (pl) +notre | our +nous | we +on | one +ou | where +par | by +pas | not +pour | for +qu | que before vowel +que | that +qui | who +sa | his, her (fem) +se | oneself +ses | his (pl) +son | his, her (masc) +sur | on +ta | thy (fem) +te | thee +tes | thy (pl) +toi | thee +ton | thy (masc) +tu | thou +un | a +une | a +vos | your (pl) +votre | your +vous | you + + | single letter forms + +c | c' +d | d' +j | j' +l | l' +à | to, at +m | m' +n | n' +s | s' +t | t' +y | there + + | forms of être (not including the infinitive): +été +étée +étées +étés +étant +suis +es +est +sommes +êtes +sont +serai +seras +sera +serons +serez +seront +serais +serait +serions +seriez +seraient +étais +était +étions +étiez +étaient +fus +fut +fûmes +fûtes +furent +sois +soit +soyons +soyez +soient +fusse +fusses +fût +fussions +fussiez +fussent + + | forms of avoir (not including the infinitive): +ayant +eu +eue +eues +eus +ai +as +avons +avez +ont +aurai +auras +aura +aurons +aurez +auront +aurais +aurait +aurions +auriez +auraient +avais +avait +avions +aviez +avaient +eut +eûmes +eûtes +eurent +aie +aies +ait +ayons +ayez +aient +eusse +eusses +eût +eussions +eussiez +eussent + + | Later additions (from Jean-Christophe Deschamps) +ceci | this +celà  | that +cet | this +cette | this +ici | here +ils | they +les | the (pl) +leurs | their (pl) +quel | which +quels | which +quelle | which +quelles | which +sans | without +soi | oneself + diff --git a/solr/example/solr/collection1/conf/lang/stopwords_ga.txt b/solr/example/solr/collection1/conf/lang/stopwords_ga.txt new file mode 100644 index 0000000..9ff88d7 --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_ga.txt @@ -0,0 +1,110 @@ + +a +ach +ag +agus +an +aon +ar +arna +as +b' +ba +beirt +bhúr +caoga +ceathair +ceathrar +chomh +chtó +chuig +chun +cois +céad +cúig +cúigear +d' +daichead +dar +de +deich +deichniúr +den +dhá +do +don +dtí +dá +dár +dó +faoi +faoin +faoina +faoinár +fara +fiche +gach +gan +go +gur +haon +hocht +i +iad +idir +in +ina +ins +inár +is +le +leis +lena +lenár +m' +mar +mo +mé +na +nach +naoi +naonúr +ná +ní +níor +nó +nócha +ocht +ochtar +os +roimh +sa +seacht +seachtar +seachtó +seasca +seisear +siad +sibh +sinn +sna +sé +sí +tar +thar +thú +triúr +trí +trína +trínár +tríocha +tú +um +ár +é +éis +í +ó +ón +óna +ónár diff --git a/solr/example/solr/collection1/conf/lang/stopwords_gl.txt b/solr/example/solr/collection1/conf/lang/stopwords_gl.txt new file mode 100644 index 0000000..d8760b1 --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_gl.txt @@ -0,0 +1,161 @@ +# galican stopwords +a +aínda +alí +aquel +aquela +aquelas +aqueles +aquilo +aquí +ao +aos +as +así +á +ben +cando +che +co +coa +comigo +con +connosco +contigo +convosco +coas +cos +cun +cuns +cunha +cunhas +da +dalgunha +dalgunhas +dalgún +dalgúns +das +de +del +dela +delas +deles +desde +deste +do +dos +dun +duns +dunha +dunhas +e +el +ela +elas +eles +en +era +eran +esa +esas +ese +eses +esta +estar +estaba +está +están +este +estes +estiven +estou +eu +é +facer +foi +foron +fun +había +hai +iso +isto +la +las +lle +lles +lo +los +mais +me +meu +meus +min +miña +miñas +moi +na +nas +neste +nin +no +non +nos +nosa +nosas +noso +nosos +nós +nun +nunha +nuns +nunhas +o +os +ou +ó +ós +para +pero +pode +pois +pola +polas +polo +polos +por +que +se +senón +ser +seu +seus +sexa +sido +sobre +súa +súas +tamén +tan +te +ten +teñen +teño +ter +teu +teus +ti +tido +tiña +tiven +túa +túas +un +unha +unhas +uns +vos +vosa +vosas +voso +vosos +vós diff --git a/solr/example/solr/collection1/conf/lang/stopwords_hi.txt b/solr/example/solr/collection1/conf/lang/stopwords_hi.txt new file mode 100644 index 0000000..86286bb --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_hi.txt @@ -0,0 +1,235 @@ +# Also see http://www.opensource.org/licenses/bsd-license.html +# See http://members.unine.ch/jacques.savoy/clef/index.html. +# This file was created by Jacques Savoy and is distributed under the BSD license. +# Note: by default this file also contains forms normalized by HindiNormalizer +# for spelling variation (see section below), such that it can be used whether or +# not you enable that feature. When adding additional entries to this list, +# please add the normalized form as well. +अंदर +अत +अपना +अपनी +अपने +अभी +आदि +आप +इतà¥à¤¯à¤¾à¤¦à¤¿ +इन +इनका +इनà¥à¤¹à¥€à¤‚ +इनà¥à¤¹à¥‡à¤‚ +इनà¥à¤¹à¥‹à¤‚ +इस +इसका +इसकी +इसके +इसमें +इसी +इसे +उन +उनका +उनकी +उनके +उनको +उनà¥à¤¹à¥€à¤‚ +उनà¥à¤¹à¥‡à¤‚ +उनà¥à¤¹à¥‹à¤‚ +उस +उसके +उसी +उसे +à¤à¤• +à¤à¤µà¤‚ +à¤à¤¸ +à¤à¤¸à¥‡ +और +कई +कर +करता +करते +करना +करने +करें +कहते +कहा +का +काफ़ी +कि +कितना +किनà¥à¤¹à¥‡à¤‚ +किनà¥à¤¹à¥‹à¤‚ +किया +किर +किस +किसी +किसे +की +कà¥à¤› +कà¥à¤² +के +को +कोई +कौन +कौनसा +गया +घर +जब +जहाठ+जा +जितना +जिन +जिनà¥à¤¹à¥‡à¤‚ +जिनà¥à¤¹à¥‹à¤‚ +जिस +जिसे +जीधर +जैसा +जैसे +जो +तक +तब +तरह +तिन +तिनà¥à¤¹à¥‡à¤‚ +तिनà¥à¤¹à¥‹à¤‚ +तिस +तिसे +तो +था +थी +थे +दबारा +दिया +दà¥à¤¸à¤°à¤¾ +दूसरे +दो +दà¥à¤µà¤¾à¤°à¤¾ +न +नहीं +ना +निहायत +नीचे +ने +पर +पर +पहले +पूरा +पे +फिर +बनी +बही +बहà¥à¤¤ +बाद +बाला +बिलकà¥à¤² +भी +भीतर +मगर +मानो +मे +में +यदि +यह +यहाठ+यही +या +यिह +ये +रखें +रहा +रहे +ऱà¥à¤µà¤¾à¤¸à¤¾ +लिठ+लिये +लेकिन +व +वरà¥à¤— +वह +वह +वहाठ+वहीं +वाले +वà¥à¤¹ +वे +वग़ैरह +संग +सकता +सकते +सबसे +सभी +साथ +साबà¥à¤¤ +साभ +सारा +से +सो +ही +हà¥à¤† +हà¥à¤ˆ +हà¥à¤ +है +हैं +हो +होता +होती +होते +होना +होने +# additional normalized forms of the above +अपनि +जेसे +होति +सभि +तिंहों +इंहों +दवारा +इसि +किंहें +थि +उंहों +ओर +जिंहें +वहिं +अभि +बनि +हि +उंहिं +उंहें +हें +वगेरह +à¤à¤¸à¥‡ +रवासा +कोन +निचे +काफि +उसि +पà¥à¤°à¤¾ +भितर +हे +बहि +वहां +कोइ +यहां +जिंहों +तिंहें +किसि +कइ +यहि +इंहिं +जिधर +इंहें +अदि +इतयादि +हà¥à¤‡ +कोनसा +इसकि +दà¥à¤¸à¤°à¥‡ +जहां +अप +किंहों +उनकि +भि +वरग +हà¥à¤… +जेसा +नहिं diff --git a/solr/example/solr/collection1/conf/lang/stopwords_hu.txt b/solr/example/solr/collection1/conf/lang/stopwords_hu.txt new file mode 100644 index 0000000..1a96f1d --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_hu.txt @@ -0,0 +1,209 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/hungarian/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + +| Hungarian stop word list +| prepared by Anna Tordai + +a +ahogy +ahol +aki +akik +akkor +alatt +által +általában +amely +amelyek +amelyekben +amelyeket +amelyet +amelynek +ami +amit +amolyan +amíg +amikor +át +abban +ahhoz +annak +arra +arról +az +azok +azon +azt +azzal +azért +aztán +azután +azonban +bár +be +belül +benne +cikk +cikkek +cikkeket +csak +de +e +eddig +egész +egy +egyes +egyetlen +egyéb +egyik +egyre +ekkor +el +elég +ellen +elÅ‘ +elÅ‘ször +elÅ‘tt +elsÅ‘ +én +éppen +ebben +ehhez +emilyen +ennek +erre +ez +ezt +ezek +ezen +ezzel +ezért +és +fel +felé +hanem +hiszen +hogy +hogyan +igen +így +illetve +ill. +ill +ilyen +ilyenkor +ison +ismét +itt +jó +jól +jobban +kell +kellett +keresztül +keressünk +ki +kívül +között +közül +legalább +lehet +lehetett +legyen +lenne +lenni +lesz +lett +maga +magát +majd +majd +már +más +másik +meg +még +mellett +mert +mely +melyek +mi +mit +míg +miért +milyen +mikor +minden +mindent +mindenki +mindig +mint +mintha +mivel +most +nagy +nagyobb +nagyon +ne +néha +nekem +neki +nem +néhány +nélkül +nincs +olyan +ott +össze +Å‘ +Å‘k +Å‘ket +pedig +persze +rá +s +saját +sem +semmi +sok +sokat +sokkal +számára +szemben +szerint +szinte +talán +tehát +teljes +tovább +továbbá +több +úgy +ugyanis +új +újabb +újra +után +utána +utolsó +vagy +vagyis +valaki +valami +valamint +való +vagyok +van +vannak +volt +voltam +voltak +voltunk +vissza +vele +viszont +volna diff --git a/solr/example/solr/collection1/conf/lang/stopwords_hy.txt b/solr/example/solr/collection1/conf/lang/stopwords_hy.txt new file mode 100644 index 0000000..60c1c50 --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_hy.txt @@ -0,0 +1,46 @@ +# example set of Armenian stopwords. +Õ¡ÕµÕ¤ +Õ¡ÕµÕ¬ +Õ¡ÕµÕ¶ +Õ¡ÕµÕ½ +Õ¤Õ¸Ö‚ +Õ¤Õ¸Ö‚Ö„ +Õ¥Õ´ +Õ¥Õ¶ +Õ¥Õ¶Ö„ +Õ¥Õ½ +Õ¥Ö„ +Õ§ +Õ§Õ« +Õ§Õ«Õ¶ +Õ§Õ«Õ¶Ö„ +Õ§Õ«Ö€ +Õ§Õ«Ö„ +Õ§Ö€ +Õ¨Õ½Õ¿ +Õ© +Õ« +Õ«Õ¶ +Õ«Õ½Õ¯ +Õ«Ö€ +Õ¯Õ¡Õ´ +Õ°Õ¡Õ´Õ¡Ö€ +Õ°Õ¥Õ¿ +Õ°Õ¥Õ¿Õ¸ +Õ´Õ¥Õ¶Ö„ +Õ´Õ¥Õ» +Õ´Õ« +Õ¶ +Õ¶Õ¡ +Õ¶Õ¡Ö‡ +Õ¶Ö€Õ¡ +Õ¶Ö€Õ¡Õ¶Ö„ +Õ¸Ö€ +Õ¸Ö€Õ¨ +Õ¸Ö€Õ¸Õ¶Ö„ +Õ¸Ö€ÕºÕ¥Õ½ +Õ¸Ö‚ +Õ¸Ö‚Õ´ +ÕºÕ«Õ¿Õ« +Õ¾Ö€Õ¡ +Ö‡ diff --git a/solr/example/solr/collection1/conf/lang/stopwords_id.txt b/solr/example/solr/collection1/conf/lang/stopwords_id.txt new file mode 100644 index 0000000..4617f83 --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_id.txt @@ -0,0 +1,359 @@ +# from appendix D of: A Study of Stemming Effects on Information +# Retrieval in Bahasa Indonesia +ada +adanya +adalah +adapun +agak +agaknya +agar +akan +akankah +akhirnya +aku +akulah +amat +amatlah +anda +andalah +antar +diantaranya +antara +antaranya +diantara +apa +apaan +mengapa +apabila +apakah +apalagi +apatah +atau +ataukah +ataupun +bagai +bagaikan +sebagai +sebagainya +bagaimana +bagaimanapun +sebagaimana +bagaimanakah +bagi +bahkan +bahwa +bahwasanya +sebaliknya +banyak +sebanyak +beberapa +seberapa +begini +beginian +beginikah +beginilah +sebegini +begitu +begitukah +begitulah +begitupun +sebegitu +belum +belumlah +sebelum +sebelumnya +sebenarnya +berapa +berapakah +berapalah +berapapun +betulkah +sebetulnya +biasa +biasanya +bila +bilakah +bisa +bisakah +sebisanya +boleh +bolehkah +bolehlah +buat +bukan +bukankah +bukanlah +bukannya +cuma +percuma +dahulu +dalam +dan +dapat +dari +daripada +dekat +demi +demikian +demikianlah +sedemikian +dengan +depan +di +dia +dialah +dini +diri +dirinya +terdiri +dong +dulu +enggak +enggaknya +entah +entahlah +terhadap +terhadapnya +hal +hampir +hanya +hanyalah +harus +haruslah +harusnya +seharusnya +hendak +hendaklah +hendaknya +hingga +sehingga +ia +ialah +ibarat +ingin +inginkah +inginkan +ini +inikah +inilah +itu +itukah +itulah +jangan +jangankan +janganlah +jika +jikalau +juga +justru +kala +kalau +kalaulah +kalaupun +kalian +kami +kamilah +kamu +kamulah +kan +kapan +kapankah +kapanpun +dikarenakan +karena +karenanya +ke +kecil +kemudian +kenapa +kepada +kepadanya +ketika +seketika +khususnya +kini +kinilah +kiranya +sekiranya +kita +kitalah +kok +lagi +lagian +selagi +lah +lain +lainnya +melainkan +selaku +lalu +melalui +terlalu +lama +lamanya +selama +selama +selamanya +lebih +terlebih +bermacam +macam +semacam +maka +makanya +makin +malah +malahan +mampu +mampukah +mana +manakala +manalagi +masih +masihkah +semasih +masing +mau +maupun +semaunya +memang +mereka +merekalah +meski +meskipun +semula +mungkin +mungkinkah +nah +namun +nanti +nantinya +nyaris +oleh +olehnya +seorang +seseorang +pada +padanya +padahal +paling +sepanjang +pantas +sepantasnya +sepantasnyalah +para +pasti +pastilah +per +pernah +pula +pun +merupakan +rupanya +serupa +saat +saatnya +sesaat +saja +sajalah +saling +bersama +sama +sesama +sambil +sampai +sana +sangat +sangatlah +saya +sayalah +se +sebab +sebabnya +sebuah +tersebut +tersebutlah +sedang +sedangkan +sedikit +sedikitnya +segala +segalanya +segera +sesegera +sejak +sejenak +sekali +sekalian +sekalipun +sesekali +sekaligus +sekarang +sekarang +sekitar +sekitarnya +sela +selain +selalu +seluruh +seluruhnya +semakin +sementara +sempat +semua +semuanya +sendiri +sendirinya +seolah +seperti +sepertinya +sering +seringnya +serta +siapa +siapakah +siapapun +disini +disinilah +sini +sinilah +sesuatu +sesuatunya +suatu +sesudah +sesudahnya +sudah +sudahkah +sudahlah +supaya +tadi +tadinya +tak +tanpa +setelah +telah +tentang +tentu +tentulah +tentunya +tertentu +seterusnya +tapi +tetapi +setiap +tiap +setidaknya +tidak +tidakkah +tidaklah +toh +waduh +wah +wahai +sewaktu +walau +walaupun +wong +yaitu +yakni +yang diff --git a/solr/example/solr/collection1/conf/lang/stopwords_it.txt b/solr/example/solr/collection1/conf/lang/stopwords_it.txt new file mode 100644 index 0000000..4cb5b08 --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_it.txt @@ -0,0 +1,301 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/italian/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + + | An Italian stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + +ad | a (to) before vowel +al | a + il +allo | a + lo +ai | a + i +agli | a + gli +all | a + l' +agl | a + gl' +alla | a + la +alle | a + le +con | with +col | con + il +coi | con + i (forms collo, cogli etc are now very rare) +da | from +dal | da + il +dallo | da + lo +dai | da + i +dagli | da + gli +dall | da + l' +dagl | da + gll' +dalla | da + la +dalle | da + le +di | of +del | di + il +dello | di + lo +dei | di + i +degli | di + gli +dell | di + l' +degl | di + gl' +della | di + la +delle | di + le +in | in +nel | in + el +nello | in + lo +nei | in + i +negli | in + gli +nell | in + l' +negl | in + gl' +nella | in + la +nelle | in + le +su | on +sul | su + il +sullo | su + lo +sui | su + i +sugli | su + gli +sull | su + l' +sugl | su + gl' +sulla | su + la +sulle | su + le +per | through, by +tra | among +contro | against +io | I +tu | thou +lui | he +lei | she +noi | we +voi | you +loro | they +mio | my +mia | +miei | +mie | +tuo | +tua | +tuoi | thy +tue | +suo | +sua | +suoi | his, her +sue | +nostro | our +nostra | +nostri | +nostre | +vostro | your +vostra | +vostri | +vostre | +mi | me +ti | thee +ci | us, there +vi | you, there +lo | him, the +la | her, the +li | them +le | them, the +gli | to him, the +ne | from there etc +il | the +un | a +uno | a +una | a +ma | but +ed | and +se | if +perché | why, because +anche | also +come | how +dov | where (as dov') +dove | where +che | who, that +chi | who +cui | whom +non | not +più | more +quale | who, that +quanto | how much +quanti | +quanta | +quante | +quello | that +quelli | +quella | +quelle | +questo | this +questi | +questa | +queste | +si | yes +tutto | all +tutti | all + + | single letter forms: + +a | at +c | as c' for ce or ci +e | and +i | the +l | as l' +o | or + + | forms of avere, to have (not including the infinitive): + +ho +hai +ha +abbiamo +avete +hanno +abbia +abbiate +abbiano +avrò +avrai +avrà +avremo +avrete +avranno +avrei +avresti +avrebbe +avremmo +avreste +avrebbero +avevo +avevi +aveva +avevamo +avevate +avevano +ebbi +avesti +ebbe +avemmo +aveste +ebbero +avessi +avesse +avessimo +avessero +avendo +avuto +avuta +avuti +avute + + | forms of essere, to be (not including the infinitive): +sono +sei +è +siamo +siete +sia +siate +siano +sarò +sarai +sarà +saremo +sarete +saranno +sarei +saresti +sarebbe +saremmo +sareste +sarebbero +ero +eri +era +eravamo +eravate +erano +fui +fosti +fu +fummo +foste +furono +fossi +fosse +fossimo +fossero +essendo + + | forms of fare, to do (not including the infinitive, fa, fat-): +faccio +fai +facciamo +fanno +faccia +facciate +facciano +farò +farai +farà +faremo +farete +faranno +farei +faresti +farebbe +faremmo +fareste +farebbero +facevo +facevi +faceva +facevamo +facevate +facevano +feci +facesti +fece +facemmo +faceste +fecero +facessi +facesse +facessimo +facessero +facendo + + | forms of stare, to be (not including the infinitive): +sto +stai +sta +stiamo +stanno +stia +stiate +stiano +starò +starai +starà +staremo +starete +staranno +starei +staresti +starebbe +staremmo +stareste +starebbero +stavo +stavi +stava +stavamo +stavate +stavano +stetti +stesti +stette +stemmo +steste +stettero +stessi +stesse +stessimo +stessero +stando diff --git a/solr/example/solr/collection1/conf/lang/stopwords_ja.txt b/solr/example/solr/collection1/conf/lang/stopwords_ja.txt new file mode 100644 index 0000000..d4321be --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_ja.txt @@ -0,0 +1,127 @@ +# +# This file defines a stopword set for Japanese. +# +# This set is made up of hand-picked frequent terms from segmented Japanese Wikipedia. +# Punctuation characters and frequent kanji have mostly been left out. See LUCENE-3745 +# for frequency lists, etc. that can be useful for making your own set (if desired) +# +# Note that there is an overlap between these stopwords and the terms stopped when used +# in combination with the JapanesePartOfSpeechStopFilter. When editing this file, note +# that comments are not allowed on the same line as stopwords. +# +# Also note that stopping is done in a case-insensitive manner. Change your StopFilter +# configuration if you need case-sensitive stopping. Lastly, note that stopping is done +# using the same character width as the entries in this file. Since this StopFilter is +# normally done after a CJKWidthFilter in your chain, you would usually want your romaji +# entries to be in half-width and your kana entries to be in full-width. +# +ã® +ã« +㯠+ã‚’ +㟠+㌠+㧠+㦠+㨠+ã— +ã‚Œ +ã• +ã‚ã‚‹ +ã„ã‚‹ +ã‚‚ +ã™ã‚‹ +ã‹ã‚‰ +㪠+ã“㨠+ã¨ã—㦠+ã„ +ã‚„ +れる +ãªã© +ãªã£ +ãªã„ +ã“ã® +ãŸã‚ +ãã® +ã‚㣠+よㆠ+ã¾ãŸ +ã‚‚ã® +ã¨ã„ㆠ+ã‚ã‚Š +ã¾ã§ +られ +ãªã‚‹ +㸠+ã‹ +ã  +ã“ã‚Œ +ã«ã‚ˆã£ã¦ +ã«ã‚ˆã‚Š +ãŠã‚Š +より +ã«ã‚ˆã‚‹ +ãš +ãªã‚Š +られる +ã«ãŠã„㦠+ã° +ãªã‹ã£ +ãªã +ã—ã‹ã— +ã«ã¤ã„㦠+ã› +ã ã£ +ãã®å¾Œ +ã§ãã‚‹ +ãã‚Œ +ㆠ+ã®ã§ +ãªãŠ +ã®ã¿ +ã§ã +ã +㤠+ã«ãŠã‘ã‚‹ +ãŠã‚ˆã³ +ã„ㆠ+ã•ã‚‰ã« +ã§ã‚‚ +ら +ãŸã‚Š +ãã®ä»– +ã«é–¢ã™ã‚‹ +ãŸã¡ +ã¾ã™ +ã‚“ +ãªã‚‰ +ã«å¯¾ã—㦠+特㫠+ã›ã‚‹ +åŠã³ +ã“れら +ã¨ã +ã§ã¯ +ã«ã¦ +ã»ã‹ +ãªãŒã‚‰ +ã†ã¡ +ãã—㦠+ã¨ã¨ã‚‚ã« +ãŸã ã— +ã‹ã¤ã¦ +ãã‚Œãžã‚Œ +ã¾ãŸã¯ +㊠+ã»ã© +ã‚‚ã®ã® +ã«å¯¾ã™ã‚‹ +ã»ã¨ã‚“ã© +ã¨å…±ã« +ã¨ã„ã£ãŸ +ã§ã™ +ã¨ã‚‚ +ã¨ã“ã‚ +ã“ã“ +##### End of file diff --git a/solr/example/solr/collection1/conf/lang/stopwords_lv.txt b/solr/example/solr/collection1/conf/lang/stopwords_lv.txt new file mode 100644 index 0000000..e21a23c --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_lv.txt @@ -0,0 +1,172 @@ +# Set of Latvian stopwords from A Stemming Algorithm for Latvian, Karlis Kreslins +# the original list of over 800 forms was refined: +# pronouns, adverbs, interjections were removed +# +# prepositions +aiz +ap +ar +apakÅ¡ +Ärpus +augÅ¡pus +bez +caur +dēļ +gar +iekÅ¡ +iz +kopÅ¡ +labad +lejpus +lÄ«dz +no +otrpus +pa +par +pÄr +pÄ“c +pie +pirms +pret +priekÅ¡ +starp +Å¡aipus +uz +viņpus +virs +virspus +zem +apakÅ¡pus +# Conjunctions +un +bet +jo +ja +ka +lai +tomÄ“r +tikko +turpretÄ« +arÄ« +kaut +gan +tÄdēļ +tÄ +ne +tikvien +vien +kÄ +ir +te +vai +kamÄ“r +# Particles +ar +diezin +droÅ¡i +diemžēl +nebÅ«t +ik +it +taÄu +nu +pat +tiklab +iekÅ¡pus +nedz +tik +nevis +turpretim +jeb +iekam +iekÄm +iekÄms +kolÄ«dz +lÄ«dzko +tiklÄ«dz +jebÅ¡u +tÄlab +tÄpÄ“c +nekÄ +itin +jÄ +jau +jel +nÄ“ +nezin +tad +tikai +vis +tak +iekams +vien +# modal verbs +bÅ«t +biju +biji +bija +bijÄm +bijÄt +esmu +esi +esam +esat +bÅ«Å¡u +bÅ«si +bÅ«s +bÅ«sim +bÅ«siet +tikt +tiku +tiki +tika +tikÄm +tikÄt +tieku +tiec +tiek +tiekam +tiekat +tikÅ¡u +tiks +tiksim +tiksiet +tapt +tapi +tapÄt +topat +tapÅ¡u +tapsi +taps +tapsim +tapsiet +kļūt +kļuvu +kļuvi +kļuva +kļuvÄm +kļuvÄt +kļūstu +kļūsti +kļūst +kļūstam +kļūstat +kļūšu +kļūsi +kļūs +kļūsim +kļūsiet +# verbs +varÄ“t +varÄ“ju +varÄ“jÄm +varÄ“Å¡u +varÄ“sim +var +varÄ“ji +varÄ“jÄt +varÄ“si +varÄ“siet +varat +varÄ“ja +varÄ“s diff --git a/solr/example/solr/collection1/conf/lang/stopwords_nl.txt b/solr/example/solr/collection1/conf/lang/stopwords_nl.txt new file mode 100644 index 0000000..f4d61f5 --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_nl.txt @@ -0,0 +1,117 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/dutch/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + + | A Dutch stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | This is a ranked list (commonest to rarest) of stopwords derived from + | a large sample of Dutch text. + + | Dutch stop words frequently exhibit homonym clashes. These are indicated + | clearly below. + +de | the +en | and +van | of, from +ik | I, the ego +te | (1) chez, at etc, (2) to, (3) too +dat | that, which +die | that, those, who, which +in | in, inside +een | a, an, one +hij | he +het | the, it +niet | not, nothing, naught +zijn | (1) to be, being, (2) his, one's, its +is | is +was | (1) was, past tense of all persons sing. of 'zijn' (to be) (2) wax, (3) the washing, (4) rise of river +op | on, upon, at, in, up, used up +aan | on, upon, to (as dative) +met | with, by +als | like, such as, when +voor | (1) before, in front of, (2) furrow +had | had, past tense all persons sing. of 'hebben' (have) +er | there +maar | but, only +om | round, about, for etc +hem | him +dan | then +zou | should/would, past tense all persons sing. of 'zullen' +of | or, whether, if +wat | what, something, anything +mijn | possessive and noun 'mine' +men | people, 'one' +dit | this +zo | so, thus, in this way +door | through by +over | over, across +ze | she, her, they, them +zich | oneself +bij | (1) a bee, (2) by, near, at +ook | also, too +tot | till, until +je | you +mij | me +uit | out of, from +der | Old Dutch form of 'van der' still found in surnames +daar | (1) there, (2) because +haar | (1) her, their, them, (2) hair +naar | (1) unpleasant, unwell etc, (2) towards, (3) as +heb | present first person sing. of 'to have' +hoe | how, why +heeft | present third person sing. of 'to have' +hebben | 'to have' and various parts thereof +deze | this +u | you +want | (1) for, (2) mitten, (3) rigging +nog | yet, still +zal | 'shall', first and third person sing. of verb 'zullen' (will) +me | me +zij | she, they +nu | now +ge | 'thou', still used in Belgium and south Netherlands +geen | none +omdat | because +iets | something, somewhat +worden | to become, grow, get +toch | yet, still +al | all, every, each +waren | (1) 'were' (2) to wander, (3) wares, (3) +veel | much, many +meer | (1) more, (2) lake +doen | to do, to make +toen | then, when +moet | noun 'spot/mote' and present form of 'to must' +ben | (1) am, (2) 'are' in interrogative second person singular of 'to be' +zonder | without +kan | noun 'can' and present form of 'to be able' +hun | their, them +dus | so, consequently +alles | all, everything, anything +onder | under, beneath +ja | yes, of course +eens | once, one day +hier | here +wie | who +werd | imperfect third person sing. of 'become' +altijd | always +doch | yet, but etc +wordt | present third person sing. of 'become' +wezen | (1) to be, (2) 'been' as in 'been fishing', (3) orphans +kunnen | to be able +ons | us/our +zelf | self +tegen | against, towards, at +na | after, near +reeds | already +wil | (1) present tense of 'want', (2) 'will', noun, (3) fender +kon | could; past tense of 'to be able' +niets | nothing +uw | your +iemand | somebody +geweest | been; past participle of 'be' +andere | other diff --git a/solr/example/solr/collection1/conf/lang/stopwords_no.txt b/solr/example/solr/collection1/conf/lang/stopwords_no.txt new file mode 100644 index 0000000..e76f36e --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_no.txt @@ -0,0 +1,192 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/norwegian/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + + | A Norwegian stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | This stop word list is for the dominant bokmÃ¥l dialect. Words unique + | to nynorsk are marked *. + + | Revised by Jan Bruusgaard , Jan 2005 + +og | and +i | in +jeg | I +det | it/this/that +at | to (w. inf.) +en | a/an +et | a/an +den | it/this/that +til | to +er | is/am/are +som | who/that +pÃ¥ | on +de | they / you(formal) +med | with +han | he +av | of +ikke | not +ikkje | not * +der | there +sÃ¥ | so +var | was/were +meg | me +seg | you +men | but +ett | one +har | have +om | about +vi | we +min | my +mitt | my +ha | have +hadde | had +hun | she +nÃ¥ | now +over | over +da | when/as +ved | by/know +fra | from +du | you +ut | out +sin | your +dem | them +oss | us +opp | up +man | you/one +kan | can +hans | his +hvor | where +eller | or +hva | what +skal | shall/must +selv | self (reflective) +sjøl | self (reflective) +her | here +alle | all +vil | will +bli | become +ble | became +blei | became * +blitt | have become +kunne | could +inn | in +nÃ¥r | when +være | be +kom | come +noen | some +noe | some +ville | would +dere | you +som | who/which/that +deres | their/theirs +kun | only/just +ja | yes +etter | after +ned | down +skulle | should +denne | this +for | for/because +deg | you +si | hers/his +sine | hers/his +sitt | hers/his +mot | against +Ã¥ | to +meget | much +hvorfor | why +dette | this +disse | these/those +uten | without +hvordan | how +ingen | none +din | your +ditt | your +blir | become +samme | same +hvilken | which +hvilke | which (plural) +sÃ¥nn | such a +inni | inside/within +mellom | between +vÃ¥r | our +hver | each +hvem | who +vors | us/ours +hvis | whose +bÃ¥de | both +bare | only/just +enn | than +fordi | as/because +før | before +mange | many +ogsÃ¥ | also +slik | just +vært | been +være | to be +bÃ¥e | both * +begge | both +siden | since +dykk | your * +dykkar | yours * +dei | they * +deira | them * +deires | theirs * +deim | them * +di | your (fem.) * +dÃ¥ | as/when * +eg | I * +ein | a/an * +eit | a/an * +eitt | a/an * +elles | or * +honom | he * +hjÃ¥ | at * +ho | she * +hoe | she * +henne | her +hennar | her/hers +hennes | hers +hoss | how * +hossen | how * +ikkje | not * +ingi | noone * +inkje | noone * +korleis | how * +korso | how * +kva | what/which * +kvar | where * +kvarhelst | where * +kven | who/whom * +kvi | why * +kvifor | why * +me | we * +medan | while * +mi | my * +mine | my * +mykje | much * +no | now * +nokon | some (masc./neut.) * +noka | some (fem.) * +nokor | some * +noko | some * +nokre | some * +si | his/hers * +sia | since * +sidan | since * +so | so * +somt | some * +somme | some * +um | about* +upp | up * +vere | be * +vore | was * +verte | become * +vort | become * +varte | became * +vart | became * + diff --git a/solr/example/solr/collection1/conf/lang/stopwords_pt.txt b/solr/example/solr/collection1/conf/lang/stopwords_pt.txt new file mode 100644 index 0000000..276c1b4 --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_pt.txt @@ -0,0 +1,251 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/portuguese/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + + | A Portuguese stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + + | The following is a ranked list (commonest to rarest) of stopwords + | deriving from a large sample of text. + + | Extra words have been added at the end. + +de | of, from +a | the; to, at; her +o | the; him +que | who, that +e | and +do | de + o +da | de + a +em | in +um | a +para | for + | é from SER +com | with +não | not, no +uma | a +os | the; them +no | em + o +se | himself etc +na | em + a +por | for +mais | more +as | the; them +dos | de + os +como | as, like +mas | but + | foi from SER +ao | a + o +ele | he +das | de + as + | tem from TER +à | a + a +seu | his +sua | her +ou | or + | ser from SER +quando | when +muito | much + | há from HAV +nos | em + os; us +já | already, now + | está from EST +eu | I +também | also +só | only, just +pelo | per + o +pela | per + a +até | up to +isso | that +ela | he +entre | between + | era from SER +depois | after +sem | without +mesmo | same +aos | a + os + | ter from TER +seus | his +quem | whom +nas | em + as +me | me +esse | that +eles | they + | estão from EST +você | you + | tinha from TER + | foram from SER +essa | that +num | em + um +nem | nor +suas | her +meu | my +às | a + as +minha | my + | têm from TER +numa | em + uma +pelos | per + os +elas | they + | havia from HAV + | seja from SER +qual | which + | será from SER +nós | we + | tenho from TER +lhe | to him, her +deles | of them +essas | those +esses | those +pelas | per + as +este | this + | fosse from SER +dele | of him + + | other words. There are many contractions such as naquele = em+aquele, + | mo = me+o, but they are rare. + | Indefinite article plural forms are also rare. + +tu | thou +te | thee +vocês | you (plural) +vos | you +lhes | to them +meus | my +minhas +teu | thy +tua +teus +tuas +nosso | our +nossa +nossos +nossas + +dela | of her +delas | of them + +esta | this +estes | these +estas | these +aquele | that +aquela | that +aqueles | those +aquelas | those +isto | this +aquilo | that + + | forms of estar, to be (not including the infinitive): +estou +está +estamos +estão +estive +esteve +estivemos +estiveram +estava +estávamos +estavam +estivera +estivéramos +esteja +estejamos +estejam +estivesse +estivéssemos +estivessem +estiver +estivermos +estiverem + + | forms of haver, to have (not including the infinitive): +hei +há +havemos +hão +houve +houvemos +houveram +houvera +houvéramos +haja +hajamos +hajam +houvesse +houvéssemos +houvessem +houver +houvermos +houverem +houverei +houverá +houveremos +houverão +houveria +houveríamos +houveriam + + | forms of ser, to be (not including the infinitive): +sou +somos +são +era +éramos +eram +fui +foi +fomos +foram +fora +fôramos +seja +sejamos +sejam +fosse +fôssemos +fossem +for +formos +forem +serei +será +seremos +serão +seria +seríamos +seriam + + | forms of ter, to have (not including the infinitive): +tenho +tem +temos +tém +tinha +tínhamos +tinham +tive +teve +tivemos +tiveram +tivera +tivéramos +tenha +tenhamos +tenham +tivesse +tivéssemos +tivessem +tiver +tivermos +tiverem +terei +terá +teremos +terão +teria +teríamos +teriam diff --git a/solr/example/solr/collection1/conf/lang/stopwords_ro.txt b/solr/example/solr/collection1/conf/lang/stopwords_ro.txt new file mode 100644 index 0000000..4fdee90 --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_ro.txt @@ -0,0 +1,233 @@ +# This file was created by Jacques Savoy and is distributed under the BSD license. +# See http://members.unine.ch/jacques.savoy/clef/index.html. +# Also see http://www.opensource.org/licenses/bsd-license.html +acea +aceasta +această +aceea +acei +aceia +acel +acela +acele +acelea +acest +acesta +aceste +acestea +aceÅŸti +aceÅŸtia +acolo +acum +ai +aia +aibă +aici +al +ăla +ale +alea +ălea +altceva +altcineva +am +ar +are +aÅŸ +aÅŸadar +asemenea +asta +ăsta +astăzi +astea +ăstea +ăştia +asupra +aÅ£i +au +avea +avem +aveÅ£i +azi +bine +bucur +bună +ca +că +căci +când +care +cărei +căror +cărui +cât +câte +câţi +către +câtva +ce +cel +ceva +chiar +cînd +cine +cineva +cît +cîte +cîţi +cîtva +contra +cu +cum +cumva +curând +curînd +da +dă +dacă +dar +datorită +de +deci +deja +deoarece +departe +deÅŸi +din +dinaintea +dintr +dintre +drept +după +ea +ei +el +ele +eram +este +eÅŸti +eu +face +fără +fi +fie +fiecare +fii +fim +fiÅ£i +iar +ieri +îi +îl +îmi +împotriva +în +înainte +înaintea +încât +încît +încotro +între +întrucât +întrucît +îţi +la +lângă +le +li +lîngă +lor +lui +mă +mâine +mea +mei +mele +mereu +meu +mi +mine +mult +multă +mulÅ£i +ne +nicăieri +nici +nimeni +niÅŸte +noastră +noastre +noi +noÅŸtri +nostru +nu +ori +oricând +oricare +oricât +orice +oricînd +oricine +oricît +oricum +oriunde +până +pe +pentru +peste +pînă +poate +pot +prea +prima +primul +prin +printr +sa +să +săi +sale +sau +său +se +ÅŸi +sînt +sîntem +sînteÅ£i +spre +sub +sunt +suntem +sunteÅ£i +ta +tăi +tale +tău +te +Å£i +Å£ie +tine +toată +toate +tot +toÅ£i +totuÅŸi +tu +un +una +unde +undeva +unei +unele +uneori +unor +vă +vi +voastră +voastre +voi +voÅŸtri +vostru +vouă +vreo +vreun diff --git a/solr/example/solr/collection1/conf/lang/stopwords_ru.txt b/solr/example/solr/collection1/conf/lang/stopwords_ru.txt new file mode 100644 index 0000000..6430769 --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_ru.txt @@ -0,0 +1,241 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/russian/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + + | a russian stop word list. comments begin with vertical bar. each stop + | word is at the start of a line. + + | this is a ranked list (commonest to rarest) of stopwords derived from + | a large text sample. + + | letter `Ñ‘' is translated to `е'. + +и | and +в | in/into +во | alternative form +не | not +что | what/that +он | he +на | on/onto +Ñ | i +Ñ | from +Ñо | alternative form +как | how +а | milder form of `no' (but) +то | conjunction and form of `that' +вÑе | all +она | she +так | so, thus +его | him +но | but +да | yes/and +Ñ‚Ñ‹ | thou +к | towards, by +у | around, chez +же | intensifier particle +вы | you +за | beyond, behind +бы | conditional/subj. particle +по | up to, along +только | only +ее | her +мне | to me +было | it was +вот | here is/are, particle +от | away from +Ð¼ÐµÐ½Ñ | me +еще | still, yet, more +нет | no, there isnt/arent +о | about +из | out of +ему | to him +теперь | now +когда | when +даже | even +ну | so, well +вдруг | suddenly +ли | interrogative particle +еÑли | if +уже | already, but homonym of `narrower' +или | or +ни | neither +быть | to be +был | he was +него | prepositional form of его +до | up to +Ð²Ð°Ñ | you accusative +нибудь | indef. suffix preceded by hyphen +опÑÑ‚ÑŒ | again +уж | already, but homonym of `adder' +вам | to you +Ñказал | he said +ведь | particle `after all' +там | there +потом | then +ÑÐµÐ±Ñ | oneself +ничего | nothing +ей | to her +может | usually with `быть' as `maybe' +они | they +тут | here +где | where +еÑÑ‚ÑŒ | there is/are +надо | got to, must +ней | prepositional form of ей +Ð´Ð»Ñ | for +мы | we +Ñ‚ÐµÐ±Ñ | thee +их | them, their +чем | than +была | she was +Ñам | self +чтоб | in order to +без | without +будто | as if +человек | man, person, one +чего | genitive form of `what' +раз | once +тоже | also +Ñебе | to oneself +под | beneath +жизнь | life +будет | will be +ж | short form of intensifer particle `же' +тогда | then +кто | who +Ñтот | this +говорил | was saying +того | genitive form of `that' +потому | for that reason +Ñтого | genitive form of `this' +какой | which +ÑовÑем | altogether +ним | prepositional form of `его', `они' +здеÑÑŒ | here +Ñтом | prepositional form of `Ñтот' +один | one +почти | almost +мой | my +тем | instrumental/dative plural of `тот', `то' +чтобы | full form of `in order that' +нее | her (acc.) +кажетÑÑ | it seems +ÑÐµÐ¹Ñ‡Ð°Ñ | now +были | they were +куда | where to +зачем | why +Ñказать | to say +вÑех | all (acc., gen. preposn. plural) +никогда | never +ÑÐµÐ³Ð¾Ð´Ð½Ñ | today +можно | possible, one can +при | by +наконец | finally +два | two +об | alternative form of `о', about +другой | another +хоть | even +поÑле | after +над | above +больше | more +тот | that one (masc.) +через | across, in +Ñти | these +Ð½Ð°Ñ | us +про | about +вÑего | in all, only, of all +них | prepositional form of `они' (they) +ÐºÐ°ÐºÐ°Ñ | which, feminine +много | lots +разве | interrogative particle +Ñказала | she said +три | three +Ñту | this, acc. fem. sing. +Ð¼Ð¾Ñ | my, feminine +впрочем | moreover, besides +хорошо | good +Ñвою | ones own, acc. fem. sing. +Ñтой | oblique form of `Ñта', fem. `this' +перед | in front of +иногда | sometimes +лучше | better +чуть | a little +том | preposn. form of `that one' +Ð½ÐµÐ»ÑŒÐ·Ñ | one must not +такой | such a one +им | to them +более | more +вÑегда | always +конечно | of course +вÑÑŽ | acc. fem. sing of `all' +между | between + + + | b: some paradigms + | + | personal pronouns + | + | Ñ Ð¼ÐµÐ½Ñ Ð¼Ð½Ðµ мной [мною] + | Ñ‚Ñ‹ Ñ‚ÐµÐ±Ñ Ñ‚ÐµÐ±Ðµ тобой [тобою] + | он его ему им [него, нему, ним] + | она ее Ñи ею [нее, нÑи, нею] + | оно его ему им [него, нему, ним] + | + | мы Ð½Ð°Ñ Ð½Ð°Ð¼ нами + | вы Ð²Ð°Ñ Ð²Ð°Ð¼ вами + | они их им ими [них, ним, ними] + | + | ÑÐµÐ±Ñ Ñебе Ñобой [Ñобою] + | + | demonstrative pronouns: Ñтот (this), тот (that) + | + | Ñтот Ñта Ñто Ñти + | Ñтого ÑÑ‚Ñ‹ Ñто Ñти + | Ñтого Ñтой Ñтого Ñтих + | Ñтому Ñтой Ñтому Ñтим + | Ñтим Ñтой Ñтим [Ñтою] Ñтими + | Ñтом Ñтой Ñтом Ñтих + | + | тот та то те + | того ту то те + | того той того тех + | тому той тому тем + | тем той тем [тою] теми + | том той том тех + | + | determinative pronouns + | + | (a) веÑÑŒ (all) + | + | веÑÑŒ вÑÑ Ð²Ñе вÑе + | вÑего вÑÑŽ вÑе вÑе + | вÑего вÑей вÑего вÑех + | вÑему вÑей вÑему вÑем + | вÑем вÑей вÑем [вÑею] вÑеми + | вÑем вÑей вÑем вÑех + | + | (b) Ñам (himself etc) + | + | Ñам Ñама Ñамо Ñами + | Ñамого Ñаму Ñамо Ñамих + | Ñамого Ñамой Ñамого Ñамих + | Ñамому Ñамой Ñамому Ñамим + | Ñамим Ñамой Ñамим [Ñамою] Ñамими + | Ñамом Ñамой Ñамом Ñамих + | + | stems of verbs `to be', `to have', `to do' and modal + | + | быть бы буд быв еÑÑ‚ÑŒ Ñуть + | име + | дел + | мог мож мочь + | уме + | хоч хот + | долж + | можн + | нужн + | Ð½ÐµÐ»ÑŒÐ·Ñ + diff --git a/solr/example/solr/collection1/conf/lang/stopwords_sv.txt b/solr/example/solr/collection1/conf/lang/stopwords_sv.txt new file mode 100644 index 0000000..22bddfd --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_sv.txt @@ -0,0 +1,131 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/swedish/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + + | A Swedish stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | This is a ranked list (commonest to rarest) of stopwords derived from + | a large text sample. + + | Swedish stop words occasionally exhibit homonym clashes. For example + | sÃ¥ = so, but also seed. These are indicated clearly below. + +och | and +det | it, this/that +att | to (with infinitive) +i | in, at +en | a +jag | I +hon | she +som | who, that +han | he +pÃ¥ | on +den | it, this/that +med | with +var | where, each +sig | him(self) etc +för | for +sÃ¥ | so (also: seed) +till | to +är | is +men | but +ett | a +om | if; around, about +hade | had +de | they, these/those +av | of +icke | not, no +mig | me +du | you +henne | her +dÃ¥ | then, when +sin | his +nu | now +har | have +inte | inte nÃ¥gon = no one +hans | his +honom | him +skulle | 'sake' +hennes | her +där | there +min | my +man | one (pronoun) +ej | nor +vid | at, by, on (also: vast) +kunde | could +nÃ¥got | some etc +frÃ¥n | from, off +ut | out +när | when +efter | after, behind +upp | up +vi | we +dem | them +vara | be +vad | what +över | over +än | than +dig | you +kan | can +sina | his +här | here +ha | have +mot | towards +alla | all +under | under (also: wonder) +nÃ¥gon | some etc +eller | or (else) +allt | all +mycket | much +sedan | since +ju | why +denna | this/that +själv | myself, yourself etc +detta | this/that +Ã¥t | to +utan | without +varit | was +hur | how +ingen | no +mitt | my +ni | you +bli | to be, become +blev | from bli +oss | us +din | thy +dessa | these/those +nÃ¥gra | some etc +deras | their +blir | from bli +mina | my +samma | (the) same +vilken | who, that +er | you, your +sÃ¥dan | such a +vÃ¥r | our +blivit | from bli +dess | its +inom | within +mellan | between +sÃ¥dant | such a +varför | why +varje | each +vilka | who, that +ditt | thy +vem | who +vilket | who, that +sitta | his +sÃ¥dana | such a +vart | each +dina | thy +vars | whose +vÃ¥rt | our +vÃ¥ra | our +ert | your +era | your +vilkas | whose + diff --git a/solr/example/solr/collection1/conf/lang/stopwords_th.txt b/solr/example/solr/collection1/conf/lang/stopwords_th.txt new file mode 100644 index 0000000..07f0fab --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_th.txt @@ -0,0 +1,119 @@ +# Thai stopwords from: +# "Opinion Detection in Thai Political News Columns +# Based on Subjectivity Analysis" +# Khampol Sukhum, Supot Nitsuwat, and Choochart Haruechaiyasak +ไว้ +ไม่ +ไป +ได้ +ให้ +ใน +โดย +à¹à¸«à¹ˆà¸‡ +à¹à¸¥à¹‰à¸§ +à¹à¸¥à¸° +à¹à¸£à¸ +à¹à¸šà¸š +à¹à¸•à¹ˆ +เอง +เห็น +เลย +เริ่ม +เรา +เมื่อ +เพื่อ +เพราะ +เป็นà¸à¸²à¸£ +เป็น +เปิดเผย +เปิด +เนื่องจาภ+เดียวà¸à¸±à¸™ +เดียว +เช่น +เฉพาะ +เคย +เข้า +เขา +อีภ+อาจ +อะไร +ออภ+อย่าง +อยู่ +อยาภ+หาภ+หลาย +หลังจาภ+หลัง +หรือ +หนึ่ง +ส่วน +ส่ง +สุด +สà¹à¸²à¸«à¸£à¸±à¸š +ว่า +วัน +ลง +ร่วม +ราย +รับ +ระหว่าง +รวม +ยัง +มี +มาภ+มา +พร้อม +พบ +ผ่าน +ผล +บาง +น่า +นี้ +นà¹à¸² +นั้น +นัภ+นอà¸à¸ˆà¸²à¸ +ทุภ+ที่สุด +ที่ +ทà¹à¸²à¹ƒà¸«à¹‰ +ทà¹à¸² +ทาง +ทั้งนี้ +ทั้ง +ถ้า +ถูภ+ถึง +ต้อง +ต่างๆ +ต่าง +ต่อ +ตาม +ตั้งà¹à¸•à¹ˆ +ตั้ง +ด้าน +ด้วย +ดัง +ซึ่ง +ช่วง +จึง +จาภ+จัด +จะ +คือ +ความ +ครั้ง +คง +ขึ้น +ของ +ขอ +ขณะ +à¸à¹ˆà¸­à¸™ +à¸à¹‡ +à¸à¸²à¸£ +à¸à¸±à¸š +à¸à¸±à¸™ +à¸à¸§à¹ˆà¸² +à¸à¸¥à¹ˆà¸²à¸§ diff --git a/solr/example/solr/collection1/conf/lang/stopwords_tr.txt b/solr/example/solr/collection1/conf/lang/stopwords_tr.txt new file mode 100644 index 0000000..84d9408 --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/stopwords_tr.txt @@ -0,0 +1,212 @@ +# Turkish stopwords from LUCENE-559 +# merged with the list from "Information Retrieval on Turkish Texts" +# (http://www.users.muohio.edu/canf/papers/JASIST2008offPrint.pdf) +acaba +altmış +altı +ama +ancak +arada +aslında +ayrıca +bana +bazı +belki +ben +benden +beni +benim +beri +beÅŸ +bile +bin +bir +birçok +biri +birkaç +birkez +birÅŸey +birÅŸeyi +biz +bize +bizden +bizi +bizim +böyle +böylece +bu +buna +bunda +bundan +bunlar +bunları +bunların +bunu +bunun +burada +çok +çünkü +da +daha +dahi +de +defa +deÄŸil +diÄŸer +diye +doksan +dokuz +dolayı +dolayısıyla +dört +edecek +eden +ederek +edilecek +ediliyor +edilmesi +ediyor +eÄŸer +elli +en +etmesi +etti +ettiÄŸi +ettiÄŸini +gibi +göre +halen +hangi +hatta +hem +henüz +hep +hepsi +her +herhangi +herkesin +hiç +hiçbir +için +iki +ile +ilgili +ise +iÅŸte +itibaren +itibariyle +kadar +karşın +katrilyon +kendi +kendilerine +kendini +kendisi +kendisine +kendisini +kez +ki +kim +kimden +kime +kimi +kimse +kırk +milyar +milyon +mu +mü +mı +nasıl +ne +neden +nedenle +nerde +nerede +nereye +niye +niçin +o +olan +olarak +oldu +olduÄŸu +olduÄŸunu +olduklarını +olmadı +olmadığı +olmak +olması +olmayan +olmaz +olsa +olsun +olup +olur +olursa +oluyor +on +ona +ondan +onlar +onlardan +onları +onların +onu +onun +otuz +oysa +öyle +pek +raÄŸmen +sadece +sanki +sekiz +seksen +sen +senden +seni +senin +siz +sizden +sizi +sizin +ÅŸey +ÅŸeyden +ÅŸeyi +ÅŸeyler +şöyle +ÅŸu +ÅŸuna +ÅŸunda +ÅŸundan +ÅŸunları +ÅŸunu +tarafından +trilyon +tüm +üç +üzere +var +vardı +ve +veya +ya +yani +yapacak +yapılan +yapılması +yapıyor +yapmak +yaptı +yaptığı +yaptığını +yaptıkları +yedi +yerine +yetmiÅŸ +yine +yirmi +yoksa +yüz +zaten diff --git a/solr/example/solr/collection1/conf/lang/userdict_ja.txt b/solr/example/solr/collection1/conf/lang/userdict_ja.txt new file mode 100644 index 0000000..6f0368e --- /dev/null +++ b/solr/example/solr/collection1/conf/lang/userdict_ja.txt @@ -0,0 +1,29 @@ +# +# This is a sample user dictionary for Kuromoji (JapaneseTokenizer) +# +# Add entries to this file in order to override the statistical model in terms +# of segmentation, readings and part-of-speech tags. Notice that entries do +# not have weights since they are always used when found. This is by-design +# in order to maximize ease-of-use. +# +# Entries are defined using the following CSV format: +# , ... , ... , +# +# Notice that a single half-width space separates tokens and readings, and +# that the number tokens and readings must match exactly. +# +# Also notice that multiple entries with the same is undefined. +# +# Whitespace only lines are ignored. Comments are not allowed on entry lines. +# + +# Custom segmentation for kanji compounds +日本経済新èž,日本 経済 æ–°èž,ニホン ケイザイ シンブン,カスタムåè©ž +関西国際空港,関西 国際 空港,カンサイ コクサイ クウコウ,カスタムåè©ž + +# Custom segmentation for compound katakana +トートãƒãƒƒã‚°,トート ãƒãƒƒã‚°,トート ãƒãƒƒã‚°,ã‹ãšã‚«ãƒŠåè©ž +ショルダーãƒãƒƒã‚°,ショルダー ãƒãƒƒã‚°,ショルダー ãƒãƒƒã‚°,ã‹ãšã‚«ãƒŠåè©ž + +# Custom reading for former sumo wrestler +æœé’é¾,æœé’é¾,アサショウリュウ,カスタム人å diff --git a/solr/example/solr/collection1/conf/mapping-FoldToASCII.txt b/solr/example/solr/collection1/conf/mapping-FoldToASCII.txt new file mode 100644 index 0000000..9a84b6e --- /dev/null +++ b/solr/example/solr/collection1/conf/mapping-FoldToASCII.txt @@ -0,0 +1,3813 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This map converts alphabetic, numeric, and symbolic Unicode characters +# which are not in the first 127 ASCII characters (the "Basic Latin" Unicode +# block) into their ASCII equivalents, if one exists. +# +# Characters from the following Unicode blocks are converted; however, only +# those characters with reasonable ASCII alternatives are converted: +# +# - C1 Controls and Latin-1 Supplement: http://www.unicode.org/charts/PDF/U0080.pdf +# - Latin Extended-A: http://www.unicode.org/charts/PDF/U0100.pdf +# - Latin Extended-B: http://www.unicode.org/charts/PDF/U0180.pdf +# - Latin Extended Additional: http://www.unicode.org/charts/PDF/U1E00.pdf +# - Latin Extended-C: http://www.unicode.org/charts/PDF/U2C60.pdf +# - Latin Extended-D: http://www.unicode.org/charts/PDF/UA720.pdf +# - IPA Extensions: http://www.unicode.org/charts/PDF/U0250.pdf +# - Phonetic Extensions: http://www.unicode.org/charts/PDF/U1D00.pdf +# - Phonetic Extensions Supplement: http://www.unicode.org/charts/PDF/U1D80.pdf +# - General Punctuation: http://www.unicode.org/charts/PDF/U2000.pdf +# - Superscripts and Subscripts: http://www.unicode.org/charts/PDF/U2070.pdf +# - Enclosed Alphanumerics: http://www.unicode.org/charts/PDF/U2460.pdf +# - Dingbats: http://www.unicode.org/charts/PDF/U2700.pdf +# - Supplemental Punctuation: http://www.unicode.org/charts/PDF/U2E00.pdf +# - Alphabetic Presentation Forms: http://www.unicode.org/charts/PDF/UFB00.pdf +# - Halfwidth and Fullwidth Forms: http://www.unicode.org/charts/PDF/UFF00.pdf +# +# See: http://en.wikipedia.org/wiki/Latin_characters_in_Unicode +# +# The set of character conversions supported by this map is a superset of +# those supported by the map represented by mapping-ISOLatin1Accent.txt. +# +# See the bottom of this file for the Perl script used to generate the contents +# of this file (without this header) from ASCIIFoldingFilter.java. + + +# Syntax: +# "source" => "target" +# "source".length() > 0 (source cannot be empty.) +# "target".length() >= 0 (target can be empty.) + + +# À [LATIN CAPITAL LETTER A WITH GRAVE] +"\u00C0" => "A" + +# à [LATIN CAPITAL LETTER A WITH ACUTE] +"\u00C1" => "A" + +#  [LATIN CAPITAL LETTER A WITH CIRCUMFLEX] +"\u00C2" => "A" + +# à [LATIN CAPITAL LETTER A WITH TILDE] +"\u00C3" => "A" + +# Ä [LATIN CAPITAL LETTER A WITH DIAERESIS] +"\u00C4" => "A" + +# Ã… [LATIN CAPITAL LETTER A WITH RING ABOVE] +"\u00C5" => "A" + +# Ä€ [LATIN CAPITAL LETTER A WITH MACRON] +"\u0100" => "A" + +# Ä‚ [LATIN CAPITAL LETTER A WITH BREVE] +"\u0102" => "A" + +# Ä„ [LATIN CAPITAL LETTER A WITH OGONEK] +"\u0104" => "A" + +# Æ http://en.wikipedia.org/wiki/Schwa [LATIN CAPITAL LETTER SCHWA] +"\u018F" => "A" + +# Ç [LATIN CAPITAL LETTER A WITH CARON] +"\u01CD" => "A" + +# Çž [LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON] +"\u01DE" => "A" + +# Ç  [LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON] +"\u01E0" => "A" + +# Ǻ [LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE] +"\u01FA" => "A" + +# È€ [LATIN CAPITAL LETTER A WITH DOUBLE GRAVE] +"\u0200" => "A" + +# È‚ [LATIN CAPITAL LETTER A WITH INVERTED BREVE] +"\u0202" => "A" + +# Ȧ [LATIN CAPITAL LETTER A WITH DOT ABOVE] +"\u0226" => "A" + +# Ⱥ [LATIN CAPITAL LETTER A WITH STROKE] +"\u023A" => "A" + +# á´€ [LATIN LETTER SMALL CAPITAL A] +"\u1D00" => "A" + +# Ḁ [LATIN CAPITAL LETTER A WITH RING BELOW] +"\u1E00" => "A" + +# Ạ [LATIN CAPITAL LETTER A WITH DOT BELOW] +"\u1EA0" => "A" + +# Ả [LATIN CAPITAL LETTER A WITH HOOK ABOVE] +"\u1EA2" => "A" + +# Ấ [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE] +"\u1EA4" => "A" + +# Ầ [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE] +"\u1EA6" => "A" + +# Ẩ [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE] +"\u1EA8" => "A" + +# Ẫ [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE] +"\u1EAA" => "A" + +# Ậ [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW] +"\u1EAC" => "A" + +# Ắ [LATIN CAPITAL LETTER A WITH BREVE AND ACUTE] +"\u1EAE" => "A" + +# Ằ [LATIN CAPITAL LETTER A WITH BREVE AND GRAVE] +"\u1EB0" => "A" + +# Ẳ [LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE] +"\u1EB2" => "A" + +# Ẵ [LATIN CAPITAL LETTER A WITH BREVE AND TILDE] +"\u1EB4" => "A" + +# Ặ [LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW] +"\u1EB6" => "A" + +# â’¶ [CIRCLED LATIN CAPITAL LETTER A] +"\u24B6" => "A" + +# Ａ [FULLWIDTH LATIN CAPITAL LETTER A] +"\uFF21" => "A" + +# à [LATIN SMALL LETTER A WITH GRAVE] +"\u00E0" => "a" + +# á [LATIN SMALL LETTER A WITH ACUTE] +"\u00E1" => "a" + +# â [LATIN SMALL LETTER A WITH CIRCUMFLEX] +"\u00E2" => "a" + +# ã [LATIN SMALL LETTER A WITH TILDE] +"\u00E3" => "a" + +# ä [LATIN SMALL LETTER A WITH DIAERESIS] +"\u00E4" => "a" + +# Ã¥ [LATIN SMALL LETTER A WITH RING ABOVE] +"\u00E5" => "a" + +# Ä [LATIN SMALL LETTER A WITH MACRON] +"\u0101" => "a" + +# ă [LATIN SMALL LETTER A WITH BREVE] +"\u0103" => "a" + +# Ä… [LATIN SMALL LETTER A WITH OGONEK] +"\u0105" => "a" + +# ÇŽ [LATIN SMALL LETTER A WITH CARON] +"\u01CE" => "a" + +# ÇŸ [LATIN SMALL LETTER A WITH DIAERESIS AND MACRON] +"\u01DF" => "a" + +# Ç¡ [LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON] +"\u01E1" => "a" + +# Ç» [LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE] +"\u01FB" => "a" + +# È [LATIN SMALL LETTER A WITH DOUBLE GRAVE] +"\u0201" => "a" + +# ȃ [LATIN SMALL LETTER A WITH INVERTED BREVE] +"\u0203" => "a" + +# ȧ [LATIN SMALL LETTER A WITH DOT ABOVE] +"\u0227" => "a" + +# É [LATIN SMALL LETTER TURNED A] +"\u0250" => "a" + +# É™ [LATIN SMALL LETTER SCHWA] +"\u0259" => "a" + +# Éš [LATIN SMALL LETTER SCHWA WITH HOOK] +"\u025A" => "a" + +# ᶠ[LATIN SMALL LETTER A WITH RETROFLEX HOOK] +"\u1D8F" => "a" + +# ᶕ [LATIN SMALL LETTER SCHWA WITH RETROFLEX HOOK] +"\u1D95" => "a" + +# ạ [LATIN SMALL LETTER A WITH RING BELOW] +"\u1E01" => "a" + +# ả [LATIN SMALL LETTER A WITH RIGHT HALF RING] +"\u1E9A" => "a" + +# ạ [LATIN SMALL LETTER A WITH DOT BELOW] +"\u1EA1" => "a" + +# ả [LATIN SMALL LETTER A WITH HOOK ABOVE] +"\u1EA3" => "a" + +# ấ [LATIN SMALL LETTER A WITH CIRCUMFLEX AND ACUTE] +"\u1EA5" => "a" + +# ầ [LATIN SMALL LETTER A WITH CIRCUMFLEX AND GRAVE] +"\u1EA7" => "a" + +# ẩ [LATIN SMALL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE] +"\u1EA9" => "a" + +# ẫ [LATIN SMALL LETTER A WITH CIRCUMFLEX AND TILDE] +"\u1EAB" => "a" + +# ậ [LATIN SMALL LETTER A WITH CIRCUMFLEX AND DOT BELOW] +"\u1EAD" => "a" + +# ắ [LATIN SMALL LETTER A WITH BREVE AND ACUTE] +"\u1EAF" => "a" + +# ằ [LATIN SMALL LETTER A WITH BREVE AND GRAVE] +"\u1EB1" => "a" + +# ẳ [LATIN SMALL LETTER A WITH BREVE AND HOOK ABOVE] +"\u1EB3" => "a" + +# ẵ [LATIN SMALL LETTER A WITH BREVE AND TILDE] +"\u1EB5" => "a" + +# ặ [LATIN SMALL LETTER A WITH BREVE AND DOT BELOW] +"\u1EB7" => "a" + +# â‚ [LATIN SUBSCRIPT SMALL LETTER A] +"\u2090" => "a" + +# â‚” [LATIN SUBSCRIPT SMALL LETTER SCHWA] +"\u2094" => "a" + +# â“ [CIRCLED LATIN SMALL LETTER A] +"\u24D0" => "a" + +# â±¥ [LATIN SMALL LETTER A WITH STROKE] +"\u2C65" => "a" + +# Ɐ [LATIN CAPITAL LETTER TURNED A] +"\u2C6F" => "a" + +# ï½ [FULLWIDTH LATIN SMALL LETTER A] +"\uFF41" => "a" + +# Ꜳ [LATIN CAPITAL LETTER AA] +"\uA732" => "AA" + +# Æ [LATIN CAPITAL LETTER AE] +"\u00C6" => "AE" + +# Ç¢ [LATIN CAPITAL LETTER AE WITH MACRON] +"\u01E2" => "AE" + +# Ǽ [LATIN CAPITAL LETTER AE WITH ACUTE] +"\u01FC" => "AE" + +# á´ [LATIN LETTER SMALL CAPITAL AE] +"\u1D01" => "AE" + +# Ꜵ [LATIN CAPITAL LETTER AO] +"\uA734" => "AO" + +# Ꜷ [LATIN CAPITAL LETTER AU] +"\uA736" => "AU" + +# Ꜹ [LATIN CAPITAL LETTER AV] +"\uA738" => "AV" + +# Ꜻ [LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR] +"\uA73A" => "AV" + +# Ꜽ [LATIN CAPITAL LETTER AY] +"\uA73C" => "AY" + +# â’œ [PARENTHESIZED LATIN SMALL LETTER A] +"\u249C" => "(a)" + +# ꜳ [LATIN SMALL LETTER AA] +"\uA733" => "aa" + +# æ [LATIN SMALL LETTER AE] +"\u00E6" => "ae" + +# Ç£ [LATIN SMALL LETTER AE WITH MACRON] +"\u01E3" => "ae" + +# ǽ [LATIN SMALL LETTER AE WITH ACUTE] +"\u01FD" => "ae" + +# á´‚ [LATIN SMALL LETTER TURNED AE] +"\u1D02" => "ae" + +# ꜵ [LATIN SMALL LETTER AO] +"\uA735" => "ao" + +# ꜷ [LATIN SMALL LETTER AU] +"\uA737" => "au" + +# ꜹ [LATIN SMALL LETTER AV] +"\uA739" => "av" + +# ꜻ [LATIN SMALL LETTER AV WITH HORIZONTAL BAR] +"\uA73B" => "av" + +# ꜽ [LATIN SMALL LETTER AY] +"\uA73D" => "ay" + +# Æ [LATIN CAPITAL LETTER B WITH HOOK] +"\u0181" => "B" + +# Æ‚ [LATIN CAPITAL LETTER B WITH TOPBAR] +"\u0182" => "B" + +# Ƀ [LATIN CAPITAL LETTER B WITH STROKE] +"\u0243" => "B" + +# Ê™ [LATIN LETTER SMALL CAPITAL B] +"\u0299" => "B" + +# á´ƒ [LATIN LETTER SMALL CAPITAL BARRED B] +"\u1D03" => "B" + +# Ḃ [LATIN CAPITAL LETTER B WITH DOT ABOVE] +"\u1E02" => "B" + +# Ḅ [LATIN CAPITAL LETTER B WITH DOT BELOW] +"\u1E04" => "B" + +# Ḇ [LATIN CAPITAL LETTER B WITH LINE BELOW] +"\u1E06" => "B" + +# â’· [CIRCLED LATIN CAPITAL LETTER B] +"\u24B7" => "B" + +# ï¼¢ [FULLWIDTH LATIN CAPITAL LETTER B] +"\uFF22" => "B" + +# Æ€ [LATIN SMALL LETTER B WITH STROKE] +"\u0180" => "b" + +# ƃ [LATIN SMALL LETTER B WITH TOPBAR] +"\u0183" => "b" + +# É“ [LATIN SMALL LETTER B WITH HOOK] +"\u0253" => "b" + +# ᵬ [LATIN SMALL LETTER B WITH MIDDLE TILDE] +"\u1D6C" => "b" + +# ᶀ [LATIN SMALL LETTER B WITH PALATAL HOOK] +"\u1D80" => "b" + +# ḃ [LATIN SMALL LETTER B WITH DOT ABOVE] +"\u1E03" => "b" + +# ḅ [LATIN SMALL LETTER B WITH DOT BELOW] +"\u1E05" => "b" + +# ḇ [LATIN SMALL LETTER B WITH LINE BELOW] +"\u1E07" => "b" + +# â“‘ [CIRCLED LATIN SMALL LETTER B] +"\u24D1" => "b" + +# ｂ [FULLWIDTH LATIN SMALL LETTER B] +"\uFF42" => "b" + +# â’ [PARENTHESIZED LATIN SMALL LETTER B] +"\u249D" => "(b)" + +# Ç [LATIN CAPITAL LETTER C WITH CEDILLA] +"\u00C7" => "C" + +# Ć [LATIN CAPITAL LETTER C WITH ACUTE] +"\u0106" => "C" + +# Ĉ [LATIN CAPITAL LETTER C WITH CIRCUMFLEX] +"\u0108" => "C" + +# ÄŠ [LATIN CAPITAL LETTER C WITH DOT ABOVE] +"\u010A" => "C" + +# ÄŒ [LATIN CAPITAL LETTER C WITH CARON] +"\u010C" => "C" + +# Ƈ [LATIN CAPITAL LETTER C WITH HOOK] +"\u0187" => "C" + +# È» [LATIN CAPITAL LETTER C WITH STROKE] +"\u023B" => "C" + +# Ê— [LATIN LETTER STRETCHED C] +"\u0297" => "C" + +# á´„ [LATIN LETTER SMALL CAPITAL C] +"\u1D04" => "C" + +# Ḉ [LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE] +"\u1E08" => "C" + +# â’¸ [CIRCLED LATIN CAPITAL LETTER C] +"\u24B8" => "C" + +# ï¼£ [FULLWIDTH LATIN CAPITAL LETTER C] +"\uFF23" => "C" + +# ç [LATIN SMALL LETTER C WITH CEDILLA] +"\u00E7" => "c" + +# ć [LATIN SMALL LETTER C WITH ACUTE] +"\u0107" => "c" + +# ĉ [LATIN SMALL LETTER C WITH CIRCUMFLEX] +"\u0109" => "c" + +# Ä‹ [LATIN SMALL LETTER C WITH DOT ABOVE] +"\u010B" => "c" + +# Ä [LATIN SMALL LETTER C WITH CARON] +"\u010D" => "c" + +# ƈ [LATIN SMALL LETTER C WITH HOOK] +"\u0188" => "c" + +# ȼ [LATIN SMALL LETTER C WITH STROKE] +"\u023C" => "c" + +# É• [LATIN SMALL LETTER C WITH CURL] +"\u0255" => "c" + +# ḉ [LATIN SMALL LETTER C WITH CEDILLA AND ACUTE] +"\u1E09" => "c" + +# ↄ [LATIN SMALL LETTER REVERSED C] +"\u2184" => "c" + +# â“’ [CIRCLED LATIN SMALL LETTER C] +"\u24D2" => "c" + +# Ꜿ [LATIN CAPITAL LETTER REVERSED C WITH DOT] +"\uA73E" => "c" + +# ꜿ [LATIN SMALL LETTER REVERSED C WITH DOT] +"\uA73F" => "c" + +# ｃ [FULLWIDTH LATIN SMALL LETTER C] +"\uFF43" => "c" + +# â’ž [PARENTHESIZED LATIN SMALL LETTER C] +"\u249E" => "(c)" + +# à [LATIN CAPITAL LETTER ETH] +"\u00D0" => "D" + +# ÄŽ [LATIN CAPITAL LETTER D WITH CARON] +"\u010E" => "D" + +# Ä [LATIN CAPITAL LETTER D WITH STROKE] +"\u0110" => "D" + +# Ɖ [LATIN CAPITAL LETTER AFRICAN D] +"\u0189" => "D" + +# ÆŠ [LATIN CAPITAL LETTER D WITH HOOK] +"\u018A" => "D" + +# Æ‹ [LATIN CAPITAL LETTER D WITH TOPBAR] +"\u018B" => "D" + +# á´… [LATIN LETTER SMALL CAPITAL D] +"\u1D05" => "D" + +# á´† [LATIN LETTER SMALL CAPITAL ETH] +"\u1D06" => "D" + +# Ḋ [LATIN CAPITAL LETTER D WITH DOT ABOVE] +"\u1E0A" => "D" + +# Ḍ [LATIN CAPITAL LETTER D WITH DOT BELOW] +"\u1E0C" => "D" + +# Ḏ [LATIN CAPITAL LETTER D WITH LINE BELOW] +"\u1E0E" => "D" + +# Ḡ[LATIN CAPITAL LETTER D WITH CEDILLA] +"\u1E10" => "D" + +# Ḓ [LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW] +"\u1E12" => "D" + +# â’¹ [CIRCLED LATIN CAPITAL LETTER D] +"\u24B9" => "D" + +# ê¹ [LATIN CAPITAL LETTER INSULAR D] +"\uA779" => "D" + +# Ｄ [FULLWIDTH LATIN CAPITAL LETTER D] +"\uFF24" => "D" + +# ð [LATIN SMALL LETTER ETH] +"\u00F0" => "d" + +# Ä [LATIN SMALL LETTER D WITH CARON] +"\u010F" => "d" + +# Ä‘ [LATIN SMALL LETTER D WITH STROKE] +"\u0111" => "d" + +# ÆŒ [LATIN SMALL LETTER D WITH TOPBAR] +"\u018C" => "d" + +# È¡ [LATIN SMALL LETTER D WITH CURL] +"\u0221" => "d" + +# É– [LATIN SMALL LETTER D WITH TAIL] +"\u0256" => "d" + +# É— [LATIN SMALL LETTER D WITH HOOK] +"\u0257" => "d" + +# áµ­ [LATIN SMALL LETTER D WITH MIDDLE TILDE] +"\u1D6D" => "d" + +# ᶠ[LATIN SMALL LETTER D WITH PALATAL HOOK] +"\u1D81" => "d" + +# ᶑ [LATIN SMALL LETTER D WITH HOOK AND TAIL] +"\u1D91" => "d" + +# ḋ [LATIN SMALL LETTER D WITH DOT ABOVE] +"\u1E0B" => "d" + +# Ḡ[LATIN SMALL LETTER D WITH DOT BELOW] +"\u1E0D" => "d" + +# Ḡ[LATIN SMALL LETTER D WITH LINE BELOW] +"\u1E0F" => "d" + +# ḑ [LATIN SMALL LETTER D WITH CEDILLA] +"\u1E11" => "d" + +# ḓ [LATIN SMALL LETTER D WITH CIRCUMFLEX BELOW] +"\u1E13" => "d" + +# â““ [CIRCLED LATIN SMALL LETTER D] +"\u24D3" => "d" + +# êº [LATIN SMALL LETTER INSULAR D] +"\uA77A" => "d" + +# ｄ [FULLWIDTH LATIN SMALL LETTER D] +"\uFF44" => "d" + +# Ç„ [LATIN CAPITAL LETTER DZ WITH CARON] +"\u01C4" => "DZ" + +# DZ [LATIN CAPITAL LETTER DZ] +"\u01F1" => "DZ" + +# Ç… [LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON] +"\u01C5" => "Dz" + +# Dz [LATIN CAPITAL LETTER D WITH SMALL LETTER Z] +"\u01F2" => "Dz" + +# â’Ÿ [PARENTHESIZED LATIN SMALL LETTER D] +"\u249F" => "(d)" + +# ȸ [LATIN SMALL LETTER DB DIGRAPH] +"\u0238" => "db" + +# dž [LATIN SMALL LETTER DZ WITH CARON] +"\u01C6" => "dz" + +# dz [LATIN SMALL LETTER DZ] +"\u01F3" => "dz" + +# Ê£ [LATIN SMALL LETTER DZ DIGRAPH] +"\u02A3" => "dz" + +# Ê¥ [LATIN SMALL LETTER DZ DIGRAPH WITH CURL] +"\u02A5" => "dz" + +# È [LATIN CAPITAL LETTER E WITH GRAVE] +"\u00C8" => "E" + +# É [LATIN CAPITAL LETTER E WITH ACUTE] +"\u00C9" => "E" + +# Ê [LATIN CAPITAL LETTER E WITH CIRCUMFLEX] +"\u00CA" => "E" + +# Ë [LATIN CAPITAL LETTER E WITH DIAERESIS] +"\u00CB" => "E" + +# Ä’ [LATIN CAPITAL LETTER E WITH MACRON] +"\u0112" => "E" + +# Ä” [LATIN CAPITAL LETTER E WITH BREVE] +"\u0114" => "E" + +# Ä– [LATIN CAPITAL LETTER E WITH DOT ABOVE] +"\u0116" => "E" + +# Ę [LATIN CAPITAL LETTER E WITH OGONEK] +"\u0118" => "E" + +# Äš [LATIN CAPITAL LETTER E WITH CARON] +"\u011A" => "E" + +# ÆŽ [LATIN CAPITAL LETTER REVERSED E] +"\u018E" => "E" + +# Æ [LATIN CAPITAL LETTER OPEN E] +"\u0190" => "E" + +# È„ [LATIN CAPITAL LETTER E WITH DOUBLE GRAVE] +"\u0204" => "E" + +# Ȇ [LATIN CAPITAL LETTER E WITH INVERTED BREVE] +"\u0206" => "E" + +# Ȩ [LATIN CAPITAL LETTER E WITH CEDILLA] +"\u0228" => "E" + +# Ɇ [LATIN CAPITAL LETTER E WITH STROKE] +"\u0246" => "E" + +# á´‡ [LATIN LETTER SMALL CAPITAL E] +"\u1D07" => "E" + +# Ḕ [LATIN CAPITAL LETTER E WITH MACRON AND GRAVE] +"\u1E14" => "E" + +# Ḗ [LATIN CAPITAL LETTER E WITH MACRON AND ACUTE] +"\u1E16" => "E" + +# Ḙ [LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW] +"\u1E18" => "E" + +# Ḛ [LATIN CAPITAL LETTER E WITH TILDE BELOW] +"\u1E1A" => "E" + +# Ḝ [LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE] +"\u1E1C" => "E" + +# Ẹ [LATIN CAPITAL LETTER E WITH DOT BELOW] +"\u1EB8" => "E" + +# Ẻ [LATIN CAPITAL LETTER E WITH HOOK ABOVE] +"\u1EBA" => "E" + +# Ẽ [LATIN CAPITAL LETTER E WITH TILDE] +"\u1EBC" => "E" + +# Ế [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE] +"\u1EBE" => "E" + +# Ề [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE] +"\u1EC0" => "E" + +# Ể [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE] +"\u1EC2" => "E" + +# Ễ [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE] +"\u1EC4" => "E" + +# Ệ [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW] +"\u1EC6" => "E" + +# â’º [CIRCLED LATIN CAPITAL LETTER E] +"\u24BA" => "E" + +# â±» [LATIN LETTER SMALL CAPITAL TURNED E] +"\u2C7B" => "E" + +# ï¼¥ [FULLWIDTH LATIN CAPITAL LETTER E] +"\uFF25" => "E" + +# è [LATIN SMALL LETTER E WITH GRAVE] +"\u00E8" => "e" + +# é [LATIN SMALL LETTER E WITH ACUTE] +"\u00E9" => "e" + +# ê [LATIN SMALL LETTER E WITH CIRCUMFLEX] +"\u00EA" => "e" + +# ë [LATIN SMALL LETTER E WITH DIAERESIS] +"\u00EB" => "e" + +# Ä“ [LATIN SMALL LETTER E WITH MACRON] +"\u0113" => "e" + +# Ä• [LATIN SMALL LETTER E WITH BREVE] +"\u0115" => "e" + +# Ä— [LATIN SMALL LETTER E WITH DOT ABOVE] +"\u0117" => "e" + +# Ä™ [LATIN SMALL LETTER E WITH OGONEK] +"\u0119" => "e" + +# Ä› [LATIN SMALL LETTER E WITH CARON] +"\u011B" => "e" + +# Ç [LATIN SMALL LETTER TURNED E] +"\u01DD" => "e" + +# È… [LATIN SMALL LETTER E WITH DOUBLE GRAVE] +"\u0205" => "e" + +# ȇ [LATIN SMALL LETTER E WITH INVERTED BREVE] +"\u0207" => "e" + +# È© [LATIN SMALL LETTER E WITH CEDILLA] +"\u0229" => "e" + +# ɇ [LATIN SMALL LETTER E WITH STROKE] +"\u0247" => "e" + +# ɘ [LATIN SMALL LETTER REVERSED E] +"\u0258" => "e" + +# É› [LATIN SMALL LETTER OPEN E] +"\u025B" => "e" + +# Éœ [LATIN SMALL LETTER REVERSED OPEN E] +"\u025C" => "e" + +# É [LATIN SMALL LETTER REVERSED OPEN E WITH HOOK] +"\u025D" => "e" + +# Éž [LATIN SMALL LETTER CLOSED REVERSED OPEN E] +"\u025E" => "e" + +# Êš [LATIN SMALL LETTER CLOSED OPEN E] +"\u029A" => "e" + +# á´ˆ [LATIN SMALL LETTER TURNED OPEN E] +"\u1D08" => "e" + +# ᶒ [LATIN SMALL LETTER E WITH RETROFLEX HOOK] +"\u1D92" => "e" + +# ᶓ [LATIN SMALL LETTER OPEN E WITH RETROFLEX HOOK] +"\u1D93" => "e" + +# ᶔ [LATIN SMALL LETTER REVERSED OPEN E WITH RETROFLEX HOOK] +"\u1D94" => "e" + +# ḕ [LATIN SMALL LETTER E WITH MACRON AND GRAVE] +"\u1E15" => "e" + +# ḗ [LATIN SMALL LETTER E WITH MACRON AND ACUTE] +"\u1E17" => "e" + +# ḙ [LATIN SMALL LETTER E WITH CIRCUMFLEX BELOW] +"\u1E19" => "e" + +# ḛ [LATIN SMALL LETTER E WITH TILDE BELOW] +"\u1E1B" => "e" + +# Ḡ[LATIN SMALL LETTER E WITH CEDILLA AND BREVE] +"\u1E1D" => "e" + +# ẹ [LATIN SMALL LETTER E WITH DOT BELOW] +"\u1EB9" => "e" + +# ẻ [LATIN SMALL LETTER E WITH HOOK ABOVE] +"\u1EBB" => "e" + +# ẽ [LATIN SMALL LETTER E WITH TILDE] +"\u1EBD" => "e" + +# ế [LATIN SMALL LETTER E WITH CIRCUMFLEX AND ACUTE] +"\u1EBF" => "e" + +# á» [LATIN SMALL LETTER E WITH CIRCUMFLEX AND GRAVE] +"\u1EC1" => "e" + +# ể [LATIN SMALL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE] +"\u1EC3" => "e" + +# á»… [LATIN SMALL LETTER E WITH CIRCUMFLEX AND TILDE] +"\u1EC5" => "e" + +# ệ [LATIN SMALL LETTER E WITH CIRCUMFLEX AND DOT BELOW] +"\u1EC7" => "e" + +# â‚‘ [LATIN SUBSCRIPT SMALL LETTER E] +"\u2091" => "e" + +# â“” [CIRCLED LATIN SMALL LETTER E] +"\u24D4" => "e" + +# ⱸ [LATIN SMALL LETTER E WITH NOTCH] +"\u2C78" => "e" + +# ï½… [FULLWIDTH LATIN SMALL LETTER E] +"\uFF45" => "e" + +# â’  [PARENTHESIZED LATIN SMALL LETTER E] +"\u24A0" => "(e)" + +# Æ‘ [LATIN CAPITAL LETTER F WITH HOOK] +"\u0191" => "F" + +# Ḟ [LATIN CAPITAL LETTER F WITH DOT ABOVE] +"\u1E1E" => "F" + +# â’» [CIRCLED LATIN CAPITAL LETTER F] +"\u24BB" => "F" + +# ꜰ [LATIN LETTER SMALL CAPITAL F] +"\uA730" => "F" + +# ê» [LATIN CAPITAL LETTER INSULAR F] +"\uA77B" => "F" + +# ꟻ [LATIN EPIGRAPHIC LETTER REVERSED F] +"\uA7FB" => "F" + +# Ｆ [FULLWIDTH LATIN CAPITAL LETTER F] +"\uFF26" => "F" + +# Æ’ [LATIN SMALL LETTER F WITH HOOK] +"\u0192" => "f" + +# áµ® [LATIN SMALL LETTER F WITH MIDDLE TILDE] +"\u1D6E" => "f" + +# ᶂ [LATIN SMALL LETTER F WITH PALATAL HOOK] +"\u1D82" => "f" + +# ḟ [LATIN SMALL LETTER F WITH DOT ABOVE] +"\u1E1F" => "f" + +# ẛ [LATIN SMALL LETTER LONG S WITH DOT ABOVE] +"\u1E9B" => "f" + +# â“• [CIRCLED LATIN SMALL LETTER F] +"\u24D5" => "f" + +# ê¼ [LATIN SMALL LETTER INSULAR F] +"\uA77C" => "f" + +# ｆ [FULLWIDTH LATIN SMALL LETTER F] +"\uFF46" => "f" + +# â’¡ [PARENTHESIZED LATIN SMALL LETTER F] +"\u24A1" => "(f)" + +# ff [LATIN SMALL LIGATURE FF] +"\uFB00" => "ff" + +# ffi [LATIN SMALL LIGATURE FFI] +"\uFB03" => "ffi" + +# ffl [LATIN SMALL LIGATURE FFL] +"\uFB04" => "ffl" + +# ï¬ [LATIN SMALL LIGATURE FI] +"\uFB01" => "fi" + +# fl [LATIN SMALL LIGATURE FL] +"\uFB02" => "fl" + +# Äœ [LATIN CAPITAL LETTER G WITH CIRCUMFLEX] +"\u011C" => "G" + +# Äž [LATIN CAPITAL LETTER G WITH BREVE] +"\u011E" => "G" + +# Ä  [LATIN CAPITAL LETTER G WITH DOT ABOVE] +"\u0120" => "G" + +# Ä¢ [LATIN CAPITAL LETTER G WITH CEDILLA] +"\u0122" => "G" + +# Æ“ [LATIN CAPITAL LETTER G WITH HOOK] +"\u0193" => "G" + +# Ǥ [LATIN CAPITAL LETTER G WITH STROKE] +"\u01E4" => "G" + +# Ç¥ [LATIN SMALL LETTER G WITH STROKE] +"\u01E5" => "G" + +# Ǧ [LATIN CAPITAL LETTER G WITH CARON] +"\u01E6" => "G" + +# ǧ [LATIN SMALL LETTER G WITH CARON] +"\u01E7" => "G" + +# Ç´ [LATIN CAPITAL LETTER G WITH ACUTE] +"\u01F4" => "G" + +# É¢ [LATIN LETTER SMALL CAPITAL G] +"\u0262" => "G" + +# Ê› [LATIN LETTER SMALL CAPITAL G WITH HOOK] +"\u029B" => "G" + +# Ḡ [LATIN CAPITAL LETTER G WITH MACRON] +"\u1E20" => "G" + +# â’¼ [CIRCLED LATIN CAPITAL LETTER G] +"\u24BC" => "G" + +# ê½ [LATIN CAPITAL LETTER INSULAR G] +"\uA77D" => "G" + +# ê¾ [LATIN CAPITAL LETTER TURNED INSULAR G] +"\uA77E" => "G" + +# Ｇ [FULLWIDTH LATIN CAPITAL LETTER G] +"\uFF27" => "G" + +# Ä [LATIN SMALL LETTER G WITH CIRCUMFLEX] +"\u011D" => "g" + +# ÄŸ [LATIN SMALL LETTER G WITH BREVE] +"\u011F" => "g" + +# Ä¡ [LATIN SMALL LETTER G WITH DOT ABOVE] +"\u0121" => "g" + +# Ä£ [LATIN SMALL LETTER G WITH CEDILLA] +"\u0123" => "g" + +# ǵ [LATIN SMALL LETTER G WITH ACUTE] +"\u01F5" => "g" + +# É  [LATIN SMALL LETTER G WITH HOOK] +"\u0260" => "g" + +# É¡ [LATIN SMALL LETTER SCRIPT G] +"\u0261" => "g" + +# áµ· [LATIN SMALL LETTER TURNED G] +"\u1D77" => "g" + +# áµ¹ [LATIN SMALL LETTER INSULAR G] +"\u1D79" => "g" + +# ᶃ [LATIN SMALL LETTER G WITH PALATAL HOOK] +"\u1D83" => "g" + +# ḡ [LATIN SMALL LETTER G WITH MACRON] +"\u1E21" => "g" + +# â“– [CIRCLED LATIN SMALL LETTER G] +"\u24D6" => "g" + +# ê¿ [LATIN SMALL LETTER TURNED INSULAR G] +"\uA77F" => "g" + +# ｇ [FULLWIDTH LATIN SMALL LETTER G] +"\uFF47" => "g" + +# â’¢ [PARENTHESIZED LATIN SMALL LETTER G] +"\u24A2" => "(g)" + +# Ĥ [LATIN CAPITAL LETTER H WITH CIRCUMFLEX] +"\u0124" => "H" + +# Ħ [LATIN CAPITAL LETTER H WITH STROKE] +"\u0126" => "H" + +# Èž [LATIN CAPITAL LETTER H WITH CARON] +"\u021E" => "H" + +# Êœ [LATIN LETTER SMALL CAPITAL H] +"\u029C" => "H" + +# Ḣ [LATIN CAPITAL LETTER H WITH DOT ABOVE] +"\u1E22" => "H" + +# Ḥ [LATIN CAPITAL LETTER H WITH DOT BELOW] +"\u1E24" => "H" + +# Ḧ [LATIN CAPITAL LETTER H WITH DIAERESIS] +"\u1E26" => "H" + +# Ḩ [LATIN CAPITAL LETTER H WITH CEDILLA] +"\u1E28" => "H" + +# Ḫ [LATIN CAPITAL LETTER H WITH BREVE BELOW] +"\u1E2A" => "H" + +# â’½ [CIRCLED LATIN CAPITAL LETTER H] +"\u24BD" => "H" + +# Ⱨ [LATIN CAPITAL LETTER H WITH DESCENDER] +"\u2C67" => "H" + +# â±µ [LATIN CAPITAL LETTER HALF H] +"\u2C75" => "H" + +# Ｈ [FULLWIDTH LATIN CAPITAL LETTER H] +"\uFF28" => "H" + +# Ä¥ [LATIN SMALL LETTER H WITH CIRCUMFLEX] +"\u0125" => "h" + +# ħ [LATIN SMALL LETTER H WITH STROKE] +"\u0127" => "h" + +# ÈŸ [LATIN SMALL LETTER H WITH CARON] +"\u021F" => "h" + +# É¥ [LATIN SMALL LETTER TURNED H] +"\u0265" => "h" + +# ɦ [LATIN SMALL LETTER H WITH HOOK] +"\u0266" => "h" + +# Ê® [LATIN SMALL LETTER TURNED H WITH FISHHOOK] +"\u02AE" => "h" + +# ʯ [LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL] +"\u02AF" => "h" + +# ḣ [LATIN SMALL LETTER H WITH DOT ABOVE] +"\u1E23" => "h" + +# ḥ [LATIN SMALL LETTER H WITH DOT BELOW] +"\u1E25" => "h" + +# ḧ [LATIN SMALL LETTER H WITH DIAERESIS] +"\u1E27" => "h" + +# ḩ [LATIN SMALL LETTER H WITH CEDILLA] +"\u1E29" => "h" + +# ḫ [LATIN SMALL LETTER H WITH BREVE BELOW] +"\u1E2B" => "h" + +# ẖ [LATIN SMALL LETTER H WITH LINE BELOW] +"\u1E96" => "h" + +# â“— [CIRCLED LATIN SMALL LETTER H] +"\u24D7" => "h" + +# ⱨ [LATIN SMALL LETTER H WITH DESCENDER] +"\u2C68" => "h" + +# ⱶ [LATIN SMALL LETTER HALF H] +"\u2C76" => "h" + +# ｈ [FULLWIDTH LATIN SMALL LETTER H] +"\uFF48" => "h" + +# Ƕ http://en.wikipedia.org/wiki/Hwair [LATIN CAPITAL LETTER HWAIR] +"\u01F6" => "HV" + +# â’£ [PARENTHESIZED LATIN SMALL LETTER H] +"\u24A3" => "(h)" + +# Æ• [LATIN SMALL LETTER HV] +"\u0195" => "hv" + +# ÃŒ [LATIN CAPITAL LETTER I WITH GRAVE] +"\u00CC" => "I" + +# à [LATIN CAPITAL LETTER I WITH ACUTE] +"\u00CD" => "I" + +# ÃŽ [LATIN CAPITAL LETTER I WITH CIRCUMFLEX] +"\u00CE" => "I" + +# à [LATIN CAPITAL LETTER I WITH DIAERESIS] +"\u00CF" => "I" + +# Ĩ [LATIN CAPITAL LETTER I WITH TILDE] +"\u0128" => "I" + +# Ī [LATIN CAPITAL LETTER I WITH MACRON] +"\u012A" => "I" + +# Ĭ [LATIN CAPITAL LETTER I WITH BREVE] +"\u012C" => "I" + +# Ä® [LATIN CAPITAL LETTER I WITH OGONEK] +"\u012E" => "I" + +# Ä° [LATIN CAPITAL LETTER I WITH DOT ABOVE] +"\u0130" => "I" + +# Æ– [LATIN CAPITAL LETTER IOTA] +"\u0196" => "I" + +# Æ— [LATIN CAPITAL LETTER I WITH STROKE] +"\u0197" => "I" + +# Ç [LATIN CAPITAL LETTER I WITH CARON] +"\u01CF" => "I" + +# Ȉ [LATIN CAPITAL LETTER I WITH DOUBLE GRAVE] +"\u0208" => "I" + +# ÈŠ [LATIN CAPITAL LETTER I WITH INVERTED BREVE] +"\u020A" => "I" + +# ɪ [LATIN LETTER SMALL CAPITAL I] +"\u026A" => "I" + +# áµ» [LATIN SMALL CAPITAL LETTER I WITH STROKE] +"\u1D7B" => "I" + +# Ḭ [LATIN CAPITAL LETTER I WITH TILDE BELOW] +"\u1E2C" => "I" + +# Ḯ [LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE] +"\u1E2E" => "I" + +# Ỉ [LATIN CAPITAL LETTER I WITH HOOK ABOVE] +"\u1EC8" => "I" + +# Ị [LATIN CAPITAL LETTER I WITH DOT BELOW] +"\u1ECA" => "I" + +# â’¾ [CIRCLED LATIN CAPITAL LETTER I] +"\u24BE" => "I" + +# ꟾ [LATIN EPIGRAPHIC LETTER I LONGA] +"\uA7FE" => "I" + +# Ｉ [FULLWIDTH LATIN CAPITAL LETTER I] +"\uFF29" => "I" + +# ì [LATIN SMALL LETTER I WITH GRAVE] +"\u00EC" => "i" + +# í [LATIN SMALL LETTER I WITH ACUTE] +"\u00ED" => "i" + +# î [LATIN SMALL LETTER I WITH CIRCUMFLEX] +"\u00EE" => "i" + +# ï [LATIN SMALL LETTER I WITH DIAERESIS] +"\u00EF" => "i" + +# Ä© [LATIN SMALL LETTER I WITH TILDE] +"\u0129" => "i" + +# Ä« [LATIN SMALL LETTER I WITH MACRON] +"\u012B" => "i" + +# Ä­ [LATIN SMALL LETTER I WITH BREVE] +"\u012D" => "i" + +# į [LATIN SMALL LETTER I WITH OGONEK] +"\u012F" => "i" + +# ı [LATIN SMALL LETTER DOTLESS I] +"\u0131" => "i" + +# Ç [LATIN SMALL LETTER I WITH CARON] +"\u01D0" => "i" + +# ȉ [LATIN SMALL LETTER I WITH DOUBLE GRAVE] +"\u0209" => "i" + +# È‹ [LATIN SMALL LETTER I WITH INVERTED BREVE] +"\u020B" => "i" + +# ɨ [LATIN SMALL LETTER I WITH STROKE] +"\u0268" => "i" + +# á´‰ [LATIN SMALL LETTER TURNED I] +"\u1D09" => "i" + +# áµ¢ [LATIN SUBSCRIPT SMALL LETTER I] +"\u1D62" => "i" + +# áµ¼ [LATIN SMALL LETTER IOTA WITH STROKE] +"\u1D7C" => "i" + +# ᶖ [LATIN SMALL LETTER I WITH RETROFLEX HOOK] +"\u1D96" => "i" + +# ḭ [LATIN SMALL LETTER I WITH TILDE BELOW] +"\u1E2D" => "i" + +# ḯ [LATIN SMALL LETTER I WITH DIAERESIS AND ACUTE] +"\u1E2F" => "i" + +# ỉ [LATIN SMALL LETTER I WITH HOOK ABOVE] +"\u1EC9" => "i" + +# ị [LATIN SMALL LETTER I WITH DOT BELOW] +"\u1ECB" => "i" + +# â± [SUPERSCRIPT LATIN SMALL LETTER I] +"\u2071" => "i" + +# ⓘ [CIRCLED LATIN SMALL LETTER I] +"\u24D8" => "i" + +# ｉ [FULLWIDTH LATIN SMALL LETTER I] +"\uFF49" => "i" + +# IJ [LATIN CAPITAL LIGATURE IJ] +"\u0132" => "IJ" + +# â’¤ [PARENTHESIZED LATIN SMALL LETTER I] +"\u24A4" => "(i)" + +# ij [LATIN SMALL LIGATURE IJ] +"\u0133" => "ij" + +# Ä´ [LATIN CAPITAL LETTER J WITH CIRCUMFLEX] +"\u0134" => "J" + +# Ɉ [LATIN CAPITAL LETTER J WITH STROKE] +"\u0248" => "J" + +# á´Š [LATIN LETTER SMALL CAPITAL J] +"\u1D0A" => "J" + +# â’¿ [CIRCLED LATIN CAPITAL LETTER J] +"\u24BF" => "J" + +# Ｊ [FULLWIDTH LATIN CAPITAL LETTER J] +"\uFF2A" => "J" + +# ĵ [LATIN SMALL LETTER J WITH CIRCUMFLEX] +"\u0135" => "j" + +# Ç° [LATIN SMALL LETTER J WITH CARON] +"\u01F0" => "j" + +# È· [LATIN SMALL LETTER DOTLESS J] +"\u0237" => "j" + +# ɉ [LATIN SMALL LETTER J WITH STROKE] +"\u0249" => "j" + +# ÉŸ [LATIN SMALL LETTER DOTLESS J WITH STROKE] +"\u025F" => "j" + +# Ê„ [LATIN SMALL LETTER DOTLESS J WITH STROKE AND HOOK] +"\u0284" => "j" + +# Ê [LATIN SMALL LETTER J WITH CROSSED-TAIL] +"\u029D" => "j" + +# â“™ [CIRCLED LATIN SMALL LETTER J] +"\u24D9" => "j" + +# â±¼ [LATIN SUBSCRIPT SMALL LETTER J] +"\u2C7C" => "j" + +# ｊ [FULLWIDTH LATIN SMALL LETTER J] +"\uFF4A" => "j" + +# â’¥ [PARENTHESIZED LATIN SMALL LETTER J] +"\u24A5" => "(j)" + +# Ķ [LATIN CAPITAL LETTER K WITH CEDILLA] +"\u0136" => "K" + +# Ƙ [LATIN CAPITAL LETTER K WITH HOOK] +"\u0198" => "K" + +# Ǩ [LATIN CAPITAL LETTER K WITH CARON] +"\u01E8" => "K" + +# á´‹ [LATIN LETTER SMALL CAPITAL K] +"\u1D0B" => "K" + +# Ḱ [LATIN CAPITAL LETTER K WITH ACUTE] +"\u1E30" => "K" + +# Ḳ [LATIN CAPITAL LETTER K WITH DOT BELOW] +"\u1E32" => "K" + +# Ḵ [LATIN CAPITAL LETTER K WITH LINE BELOW] +"\u1E34" => "K" + +# â“€ [CIRCLED LATIN CAPITAL LETTER K] +"\u24C0" => "K" + +# Ⱪ [LATIN CAPITAL LETTER K WITH DESCENDER] +"\u2C69" => "K" + +# ê€ [LATIN CAPITAL LETTER K WITH STROKE] +"\uA740" => "K" + +# ê‚ [LATIN CAPITAL LETTER K WITH DIAGONAL STROKE] +"\uA742" => "K" + +# ê„ [LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE] +"\uA744" => "K" + +# Ｋ [FULLWIDTH LATIN CAPITAL LETTER K] +"\uFF2B" => "K" + +# Ä· [LATIN SMALL LETTER K WITH CEDILLA] +"\u0137" => "k" + +# Æ™ [LATIN SMALL LETTER K WITH HOOK] +"\u0199" => "k" + +# Ç© [LATIN SMALL LETTER K WITH CARON] +"\u01E9" => "k" + +# Êž [LATIN SMALL LETTER TURNED K] +"\u029E" => "k" + +# ᶄ [LATIN SMALL LETTER K WITH PALATAL HOOK] +"\u1D84" => "k" + +# ḱ [LATIN SMALL LETTER K WITH ACUTE] +"\u1E31" => "k" + +# ḳ [LATIN SMALL LETTER K WITH DOT BELOW] +"\u1E33" => "k" + +# ḵ [LATIN SMALL LETTER K WITH LINE BELOW] +"\u1E35" => "k" + +# â“š [CIRCLED LATIN SMALL LETTER K] +"\u24DA" => "k" + +# ⱪ [LATIN SMALL LETTER K WITH DESCENDER] +"\u2C6A" => "k" + +# ê [LATIN SMALL LETTER K WITH STROKE] +"\uA741" => "k" + +# êƒ [LATIN SMALL LETTER K WITH DIAGONAL STROKE] +"\uA743" => "k" + +# ê… [LATIN SMALL LETTER K WITH STROKE AND DIAGONAL STROKE] +"\uA745" => "k" + +# ｋ [FULLWIDTH LATIN SMALL LETTER K] +"\uFF4B" => "k" + +# â’¦ [PARENTHESIZED LATIN SMALL LETTER K] +"\u24A6" => "(k)" + +# Ĺ [LATIN CAPITAL LETTER L WITH ACUTE] +"\u0139" => "L" + +# Ä» [LATIN CAPITAL LETTER L WITH CEDILLA] +"\u013B" => "L" + +# Ľ [LATIN CAPITAL LETTER L WITH CARON] +"\u013D" => "L" + +# Ä¿ [LATIN CAPITAL LETTER L WITH MIDDLE DOT] +"\u013F" => "L" + +# Å [LATIN CAPITAL LETTER L WITH STROKE] +"\u0141" => "L" + +# Ƚ [LATIN CAPITAL LETTER L WITH BAR] +"\u023D" => "L" + +# ÊŸ [LATIN LETTER SMALL CAPITAL L] +"\u029F" => "L" + +# á´Œ [LATIN LETTER SMALL CAPITAL L WITH STROKE] +"\u1D0C" => "L" + +# Ḷ [LATIN CAPITAL LETTER L WITH DOT BELOW] +"\u1E36" => "L" + +# Ḹ [LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON] +"\u1E38" => "L" + +# Ḻ [LATIN CAPITAL LETTER L WITH LINE BELOW] +"\u1E3A" => "L" + +# Ḽ [LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW] +"\u1E3C" => "L" + +# â“ [CIRCLED LATIN CAPITAL LETTER L] +"\u24C1" => "L" + +# â±  [LATIN CAPITAL LETTER L WITH DOUBLE BAR] +"\u2C60" => "L" + +# â±¢ [LATIN CAPITAL LETTER L WITH MIDDLE TILDE] +"\u2C62" => "L" + +# ê† [LATIN CAPITAL LETTER BROKEN L] +"\uA746" => "L" + +# êˆ [LATIN CAPITAL LETTER L WITH HIGH STROKE] +"\uA748" => "L" + +# Ꞁ [LATIN CAPITAL LETTER TURNED L] +"\uA780" => "L" + +# Ｌ [FULLWIDTH LATIN CAPITAL LETTER L] +"\uFF2C" => "L" + +# ĺ [LATIN SMALL LETTER L WITH ACUTE] +"\u013A" => "l" + +# ļ [LATIN SMALL LETTER L WITH CEDILLA] +"\u013C" => "l" + +# ľ [LATIN SMALL LETTER L WITH CARON] +"\u013E" => "l" + +# Å€ [LATIN SMALL LETTER L WITH MIDDLE DOT] +"\u0140" => "l" + +# Å‚ [LATIN SMALL LETTER L WITH STROKE] +"\u0142" => "l" + +# Æš [LATIN SMALL LETTER L WITH BAR] +"\u019A" => "l" + +# È´ [LATIN SMALL LETTER L WITH CURL] +"\u0234" => "l" + +# É« [LATIN SMALL LETTER L WITH MIDDLE TILDE] +"\u026B" => "l" + +# ɬ [LATIN SMALL LETTER L WITH BELT] +"\u026C" => "l" + +# É­ [LATIN SMALL LETTER L WITH RETROFLEX HOOK] +"\u026D" => "l" + +# ᶅ [LATIN SMALL LETTER L WITH PALATAL HOOK] +"\u1D85" => "l" + +# ḷ [LATIN SMALL LETTER L WITH DOT BELOW] +"\u1E37" => "l" + +# ḹ [LATIN SMALL LETTER L WITH DOT BELOW AND MACRON] +"\u1E39" => "l" + +# ḻ [LATIN SMALL LETTER L WITH LINE BELOW] +"\u1E3B" => "l" + +# ḽ [LATIN SMALL LETTER L WITH CIRCUMFLEX BELOW] +"\u1E3D" => "l" + +# â“› [CIRCLED LATIN SMALL LETTER L] +"\u24DB" => "l" + +# ⱡ [LATIN SMALL LETTER L WITH DOUBLE BAR] +"\u2C61" => "l" + +# ê‡ [LATIN SMALL LETTER BROKEN L] +"\uA747" => "l" + +# ê‰ [LATIN SMALL LETTER L WITH HIGH STROKE] +"\uA749" => "l" + +# êž [LATIN SMALL LETTER TURNED L] +"\uA781" => "l" + +# ｌ [FULLWIDTH LATIN SMALL LETTER L] +"\uFF4C" => "l" + +# LJ [LATIN CAPITAL LETTER LJ] +"\u01C7" => "LJ" + +# Ỻ [LATIN CAPITAL LETTER MIDDLE-WELSH LL] +"\u1EFA" => "LL" + +# Lj [LATIN CAPITAL LETTER L WITH SMALL LETTER J] +"\u01C8" => "Lj" + +# â’§ [PARENTHESIZED LATIN SMALL LETTER L] +"\u24A7" => "(l)" + +# lj [LATIN SMALL LETTER LJ] +"\u01C9" => "lj" + +# á»» [LATIN SMALL LETTER MIDDLE-WELSH LL] +"\u1EFB" => "ll" + +# ʪ [LATIN SMALL LETTER LS DIGRAPH] +"\u02AA" => "ls" + +# Ê« [LATIN SMALL LETTER LZ DIGRAPH] +"\u02AB" => "lz" + +# Æœ [LATIN CAPITAL LETTER TURNED M] +"\u019C" => "M" + +# á´ [LATIN LETTER SMALL CAPITAL M] +"\u1D0D" => "M" + +# Ḿ [LATIN CAPITAL LETTER M WITH ACUTE] +"\u1E3E" => "M" + +# á¹€ [LATIN CAPITAL LETTER M WITH DOT ABOVE] +"\u1E40" => "M" + +# Ṃ [LATIN CAPITAL LETTER M WITH DOT BELOW] +"\u1E42" => "M" + +# â“‚ [CIRCLED LATIN CAPITAL LETTER M] +"\u24C2" => "M" + +# â±® [LATIN CAPITAL LETTER M WITH HOOK] +"\u2C6E" => "M" + +# ꟽ [LATIN EPIGRAPHIC LETTER INVERTED M] +"\uA7FD" => "M" + +# ꟿ [LATIN EPIGRAPHIC LETTER ARCHAIC M] +"\uA7FF" => "M" + +# ï¼­ [FULLWIDTH LATIN CAPITAL LETTER M] +"\uFF2D" => "M" + +# ɯ [LATIN SMALL LETTER TURNED M] +"\u026F" => "m" + +# É° [LATIN SMALL LETTER TURNED M WITH LONG LEG] +"\u0270" => "m" + +# ɱ [LATIN SMALL LETTER M WITH HOOK] +"\u0271" => "m" + +# ᵯ [LATIN SMALL LETTER M WITH MIDDLE TILDE] +"\u1D6F" => "m" + +# ᶆ [LATIN SMALL LETTER M WITH PALATAL HOOK] +"\u1D86" => "m" + +# ḿ [LATIN SMALL LETTER M WITH ACUTE] +"\u1E3F" => "m" + +# á¹ [LATIN SMALL LETTER M WITH DOT ABOVE] +"\u1E41" => "m" + +# ṃ [LATIN SMALL LETTER M WITH DOT BELOW] +"\u1E43" => "m" + +# â“œ [CIRCLED LATIN SMALL LETTER M] +"\u24DC" => "m" + +# ï½ [FULLWIDTH LATIN SMALL LETTER M] +"\uFF4D" => "m" + +# â’¨ [PARENTHESIZED LATIN SMALL LETTER M] +"\u24A8" => "(m)" + +# Ñ [LATIN CAPITAL LETTER N WITH TILDE] +"\u00D1" => "N" + +# Ń [LATIN CAPITAL LETTER N WITH ACUTE] +"\u0143" => "N" + +# Å… [LATIN CAPITAL LETTER N WITH CEDILLA] +"\u0145" => "N" + +# Ň [LATIN CAPITAL LETTER N WITH CARON] +"\u0147" => "N" + +# ÅŠ http://en.wikipedia.org/wiki/Eng_(letter) [LATIN CAPITAL LETTER ENG] +"\u014A" => "N" + +# Æ [LATIN CAPITAL LETTER N WITH LEFT HOOK] +"\u019D" => "N" + +# Ǹ [LATIN CAPITAL LETTER N WITH GRAVE] +"\u01F8" => "N" + +# È  [LATIN CAPITAL LETTER N WITH LONG RIGHT LEG] +"\u0220" => "N" + +# É´ [LATIN LETTER SMALL CAPITAL N] +"\u0274" => "N" + +# á´Ž [LATIN LETTER SMALL CAPITAL REVERSED N] +"\u1D0E" => "N" + +# Ṅ [LATIN CAPITAL LETTER N WITH DOT ABOVE] +"\u1E44" => "N" + +# Ṇ [LATIN CAPITAL LETTER N WITH DOT BELOW] +"\u1E46" => "N" + +# Ṉ [LATIN CAPITAL LETTER N WITH LINE BELOW] +"\u1E48" => "N" + +# Ṋ [LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW] +"\u1E4A" => "N" + +# Ⓝ [CIRCLED LATIN CAPITAL LETTER N] +"\u24C3" => "N" + +# ï¼® [FULLWIDTH LATIN CAPITAL LETTER N] +"\uFF2E" => "N" + +# ñ [LATIN SMALL LETTER N WITH TILDE] +"\u00F1" => "n" + +# Å„ [LATIN SMALL LETTER N WITH ACUTE] +"\u0144" => "n" + +# ņ [LATIN SMALL LETTER N WITH CEDILLA] +"\u0146" => "n" + +# ň [LATIN SMALL LETTER N WITH CARON] +"\u0148" => "n" + +# ʼn [LATIN SMALL LETTER N PRECEDED BY APOSTROPHE] +"\u0149" => "n" + +# Å‹ http://en.wikipedia.org/wiki/Eng_(letter) [LATIN SMALL LETTER ENG] +"\u014B" => "n" + +# Æž [LATIN SMALL LETTER N WITH LONG RIGHT LEG] +"\u019E" => "n" + +# ǹ [LATIN SMALL LETTER N WITH GRAVE] +"\u01F9" => "n" + +# ȵ [LATIN SMALL LETTER N WITH CURL] +"\u0235" => "n" + +# ɲ [LATIN SMALL LETTER N WITH LEFT HOOK] +"\u0272" => "n" + +# ɳ [LATIN SMALL LETTER N WITH RETROFLEX HOOK] +"\u0273" => "n" + +# áµ° [LATIN SMALL LETTER N WITH MIDDLE TILDE] +"\u1D70" => "n" + +# ᶇ [LATIN SMALL LETTER N WITH PALATAL HOOK] +"\u1D87" => "n" + +# á¹… [LATIN SMALL LETTER N WITH DOT ABOVE] +"\u1E45" => "n" + +# ṇ [LATIN SMALL LETTER N WITH DOT BELOW] +"\u1E47" => "n" + +# ṉ [LATIN SMALL LETTER N WITH LINE BELOW] +"\u1E49" => "n" + +# ṋ [LATIN SMALL LETTER N WITH CIRCUMFLEX BELOW] +"\u1E4B" => "n" + +# â¿ [SUPERSCRIPT LATIN SMALL LETTER N] +"\u207F" => "n" + +# â“ [CIRCLED LATIN SMALL LETTER N] +"\u24DD" => "n" + +# ｎ [FULLWIDTH LATIN SMALL LETTER N] +"\uFF4E" => "n" + +# ÇŠ [LATIN CAPITAL LETTER NJ] +"\u01CA" => "NJ" + +# Ç‹ [LATIN CAPITAL LETTER N WITH SMALL LETTER J] +"\u01CB" => "Nj" + +# â’© [PARENTHESIZED LATIN SMALL LETTER N] +"\u24A9" => "(n)" + +# ÇŒ [LATIN SMALL LETTER NJ] +"\u01CC" => "nj" + +# Ã’ [LATIN CAPITAL LETTER O WITH GRAVE] +"\u00D2" => "O" + +# Ó [LATIN CAPITAL LETTER O WITH ACUTE] +"\u00D3" => "O" + +# Ô [LATIN CAPITAL LETTER O WITH CIRCUMFLEX] +"\u00D4" => "O" + +# Õ [LATIN CAPITAL LETTER O WITH TILDE] +"\u00D5" => "O" + +# Ö [LATIN CAPITAL LETTER O WITH DIAERESIS] +"\u00D6" => "O" + +# Ø [LATIN CAPITAL LETTER O WITH STROKE] +"\u00D8" => "O" + +# ÅŒ [LATIN CAPITAL LETTER O WITH MACRON] +"\u014C" => "O" + +# ÅŽ [LATIN CAPITAL LETTER O WITH BREVE] +"\u014E" => "O" + +# Å [LATIN CAPITAL LETTER O WITH DOUBLE ACUTE] +"\u0150" => "O" + +# Ɔ [LATIN CAPITAL LETTER OPEN O] +"\u0186" => "O" + +# ÆŸ [LATIN CAPITAL LETTER O WITH MIDDLE TILDE] +"\u019F" => "O" + +# Æ  [LATIN CAPITAL LETTER O WITH HORN] +"\u01A0" => "O" + +# Ç‘ [LATIN CAPITAL LETTER O WITH CARON] +"\u01D1" => "O" + +# Ǫ [LATIN CAPITAL LETTER O WITH OGONEK] +"\u01EA" => "O" + +# Ǭ [LATIN CAPITAL LETTER O WITH OGONEK AND MACRON] +"\u01EC" => "O" + +# Ǿ [LATIN CAPITAL LETTER O WITH STROKE AND ACUTE] +"\u01FE" => "O" + +# ÈŒ [LATIN CAPITAL LETTER O WITH DOUBLE GRAVE] +"\u020C" => "O" + +# ÈŽ [LATIN CAPITAL LETTER O WITH INVERTED BREVE] +"\u020E" => "O" + +# Ȫ [LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON] +"\u022A" => "O" + +# Ȭ [LATIN CAPITAL LETTER O WITH TILDE AND MACRON] +"\u022C" => "O" + +# È® [LATIN CAPITAL LETTER O WITH DOT ABOVE] +"\u022E" => "O" + +# È° [LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON] +"\u0230" => "O" + +# á´ [LATIN LETTER SMALL CAPITAL O] +"\u1D0F" => "O" + +# á´ [LATIN LETTER SMALL CAPITAL OPEN O] +"\u1D10" => "O" + +# Ṍ [LATIN CAPITAL LETTER O WITH TILDE AND ACUTE] +"\u1E4C" => "O" + +# Ṏ [LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS] +"\u1E4E" => "O" + +# á¹ [LATIN CAPITAL LETTER O WITH MACRON AND GRAVE] +"\u1E50" => "O" + +# á¹’ [LATIN CAPITAL LETTER O WITH MACRON AND ACUTE] +"\u1E52" => "O" + +# Ọ [LATIN CAPITAL LETTER O WITH DOT BELOW] +"\u1ECC" => "O" + +# Ỏ [LATIN CAPITAL LETTER O WITH HOOK ABOVE] +"\u1ECE" => "O" + +# á» [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE] +"\u1ED0" => "O" + +# á»’ [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE] +"\u1ED2" => "O" + +# á»” [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE] +"\u1ED4" => "O" + +# á»– [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE] +"\u1ED6" => "O" + +# Ộ [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW] +"\u1ED8" => "O" + +# Ớ [LATIN CAPITAL LETTER O WITH HORN AND ACUTE] +"\u1EDA" => "O" + +# Ờ [LATIN CAPITAL LETTER O WITH HORN AND GRAVE] +"\u1EDC" => "O" + +# Ở [LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE] +"\u1EDE" => "O" + +# á»  [LATIN CAPITAL LETTER O WITH HORN AND TILDE] +"\u1EE0" => "O" + +# Ợ [LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW] +"\u1EE2" => "O" + +# â“„ [CIRCLED LATIN CAPITAL LETTER O] +"\u24C4" => "O" + +# êŠ [LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY] +"\uA74A" => "O" + +# êŒ [LATIN CAPITAL LETTER O WITH LOOP] +"\uA74C" => "O" + +# Ｏ [FULLWIDTH LATIN CAPITAL LETTER O] +"\uFF2F" => "O" + +# ò [LATIN SMALL LETTER O WITH GRAVE] +"\u00F2" => "o" + +# ó [LATIN SMALL LETTER O WITH ACUTE] +"\u00F3" => "o" + +# ô [LATIN SMALL LETTER O WITH CIRCUMFLEX] +"\u00F4" => "o" + +# õ [LATIN SMALL LETTER O WITH TILDE] +"\u00F5" => "o" + +# ö [LATIN SMALL LETTER O WITH DIAERESIS] +"\u00F6" => "o" + +# ø [LATIN SMALL LETTER O WITH STROKE] +"\u00F8" => "o" + +# Å [LATIN SMALL LETTER O WITH MACRON] +"\u014D" => "o" + +# Å [LATIN SMALL LETTER O WITH BREVE] +"\u014F" => "o" + +# Å‘ [LATIN SMALL LETTER O WITH DOUBLE ACUTE] +"\u0151" => "o" + +# Æ¡ [LATIN SMALL LETTER O WITH HORN] +"\u01A1" => "o" + +# Ç’ [LATIN SMALL LETTER O WITH CARON] +"\u01D2" => "o" + +# Ç« [LATIN SMALL LETTER O WITH OGONEK] +"\u01EB" => "o" + +# Ç­ [LATIN SMALL LETTER O WITH OGONEK AND MACRON] +"\u01ED" => "o" + +# Ç¿ [LATIN SMALL LETTER O WITH STROKE AND ACUTE] +"\u01FF" => "o" + +# È [LATIN SMALL LETTER O WITH DOUBLE GRAVE] +"\u020D" => "o" + +# È [LATIN SMALL LETTER O WITH INVERTED BREVE] +"\u020F" => "o" + +# È« [LATIN SMALL LETTER O WITH DIAERESIS AND MACRON] +"\u022B" => "o" + +# È­ [LATIN SMALL LETTER O WITH TILDE AND MACRON] +"\u022D" => "o" + +# ȯ [LATIN SMALL LETTER O WITH DOT ABOVE] +"\u022F" => "o" + +# ȱ [LATIN SMALL LETTER O WITH DOT ABOVE AND MACRON] +"\u0231" => "o" + +# É” [LATIN SMALL LETTER OPEN O] +"\u0254" => "o" + +# ɵ [LATIN SMALL LETTER BARRED O] +"\u0275" => "o" + +# á´– [LATIN SMALL LETTER TOP HALF O] +"\u1D16" => "o" + +# á´— [LATIN SMALL LETTER BOTTOM HALF O] +"\u1D17" => "o" + +# ᶗ [LATIN SMALL LETTER OPEN O WITH RETROFLEX HOOK] +"\u1D97" => "o" + +# á¹ [LATIN SMALL LETTER O WITH TILDE AND ACUTE] +"\u1E4D" => "o" + +# á¹ [LATIN SMALL LETTER O WITH TILDE AND DIAERESIS] +"\u1E4F" => "o" + +# ṑ [LATIN SMALL LETTER O WITH MACRON AND GRAVE] +"\u1E51" => "o" + +# ṓ [LATIN SMALL LETTER O WITH MACRON AND ACUTE] +"\u1E53" => "o" + +# á» [LATIN SMALL LETTER O WITH DOT BELOW] +"\u1ECD" => "o" + +# á» [LATIN SMALL LETTER O WITH HOOK ABOVE] +"\u1ECF" => "o" + +# ố [LATIN SMALL LETTER O WITH CIRCUMFLEX AND ACUTE] +"\u1ED1" => "o" + +# ồ [LATIN SMALL LETTER O WITH CIRCUMFLEX AND GRAVE] +"\u1ED3" => "o" + +# ổ [LATIN SMALL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE] +"\u1ED5" => "o" + +# á»— [LATIN SMALL LETTER O WITH CIRCUMFLEX AND TILDE] +"\u1ED7" => "o" + +# á»™ [LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW] +"\u1ED9" => "o" + +# á»› [LATIN SMALL LETTER O WITH HORN AND ACUTE] +"\u1EDB" => "o" + +# á» [LATIN SMALL LETTER O WITH HORN AND GRAVE] +"\u1EDD" => "o" + +# ở [LATIN SMALL LETTER O WITH HORN AND HOOK ABOVE] +"\u1EDF" => "o" + +# ỡ [LATIN SMALL LETTER O WITH HORN AND TILDE] +"\u1EE1" => "o" + +# ợ [LATIN SMALL LETTER O WITH HORN AND DOT BELOW] +"\u1EE3" => "o" + +# â‚’ [LATIN SUBSCRIPT SMALL LETTER O] +"\u2092" => "o" + +# â“ž [CIRCLED LATIN SMALL LETTER O] +"\u24DE" => "o" + +# ⱺ [LATIN SMALL LETTER O WITH LOW RING INSIDE] +"\u2C7A" => "o" + +# ê‹ [LATIN SMALL LETTER O WITH LONG STROKE OVERLAY] +"\uA74B" => "o" + +# ê [LATIN SMALL LETTER O WITH LOOP] +"\uA74D" => "o" + +# ï½ [FULLWIDTH LATIN SMALL LETTER O] +"\uFF4F" => "o" + +# Å’ [LATIN CAPITAL LIGATURE OE] +"\u0152" => "OE" + +# ɶ [LATIN LETTER SMALL CAPITAL OE] +"\u0276" => "OE" + +# êŽ [LATIN CAPITAL LETTER OO] +"\uA74E" => "OO" + +# È¢ http://en.wikipedia.org/wiki/OU [LATIN CAPITAL LETTER OU] +"\u0222" => "OU" + +# á´• [LATIN LETTER SMALL CAPITAL OU] +"\u1D15" => "OU" + +# â’ª [PARENTHESIZED LATIN SMALL LETTER O] +"\u24AA" => "(o)" + +# Å“ [LATIN SMALL LIGATURE OE] +"\u0153" => "oe" + +# á´” [LATIN SMALL LETTER TURNED OE] +"\u1D14" => "oe" + +# ê [LATIN SMALL LETTER OO] +"\uA74F" => "oo" + +# È£ http://en.wikipedia.org/wiki/OU [LATIN SMALL LETTER OU] +"\u0223" => "ou" + +# Ƥ [LATIN CAPITAL LETTER P WITH HOOK] +"\u01A4" => "P" + +# á´˜ [LATIN LETTER SMALL CAPITAL P] +"\u1D18" => "P" + +# á¹” [LATIN CAPITAL LETTER P WITH ACUTE] +"\u1E54" => "P" + +# á¹– [LATIN CAPITAL LETTER P WITH DOT ABOVE] +"\u1E56" => "P" + +# â“… [CIRCLED LATIN CAPITAL LETTER P] +"\u24C5" => "P" + +# â±£ [LATIN CAPITAL LETTER P WITH STROKE] +"\u2C63" => "P" + +# ê [LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER] +"\uA750" => "P" + +# ê’ [LATIN CAPITAL LETTER P WITH FLOURISH] +"\uA752" => "P" + +# ê” [LATIN CAPITAL LETTER P WITH SQUIRREL TAIL] +"\uA754" => "P" + +# ï¼° [FULLWIDTH LATIN CAPITAL LETTER P] +"\uFF30" => "P" + +# Æ¥ [LATIN SMALL LETTER P WITH HOOK] +"\u01A5" => "p" + +# áµ± [LATIN SMALL LETTER P WITH MIDDLE TILDE] +"\u1D71" => "p" + +# áµ½ [LATIN SMALL LETTER P WITH STROKE] +"\u1D7D" => "p" + +# ᶈ [LATIN SMALL LETTER P WITH PALATAL HOOK] +"\u1D88" => "p" + +# ṕ [LATIN SMALL LETTER P WITH ACUTE] +"\u1E55" => "p" + +# á¹— [LATIN SMALL LETTER P WITH DOT ABOVE] +"\u1E57" => "p" + +# â“Ÿ [CIRCLED LATIN SMALL LETTER P] +"\u24DF" => "p" + +# ê‘ [LATIN SMALL LETTER P WITH STROKE THROUGH DESCENDER] +"\uA751" => "p" + +# ê“ [LATIN SMALL LETTER P WITH FLOURISH] +"\uA753" => "p" + +# ê• [LATIN SMALL LETTER P WITH SQUIRREL TAIL] +"\uA755" => "p" + +# ꟼ [LATIN EPIGRAPHIC LETTER REVERSED P] +"\uA7FC" => "p" + +# ï½ [FULLWIDTH LATIN SMALL LETTER P] +"\uFF50" => "p" + +# â’« [PARENTHESIZED LATIN SMALL LETTER P] +"\u24AB" => "(p)" + +# ÉŠ [LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL] +"\u024A" => "Q" + +# Ⓠ [CIRCLED LATIN CAPITAL LETTER Q] +"\u24C6" => "Q" + +# ê– [LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER] +"\uA756" => "Q" + +# ê˜ [LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE] +"\uA758" => "Q" + +# ï¼± [FULLWIDTH LATIN CAPITAL LETTER Q] +"\uFF31" => "Q" + +# ĸ http://en.wikipedia.org/wiki/Kra_(letter) [LATIN SMALL LETTER KRA] +"\u0138" => "q" + +# É‹ [LATIN SMALL LETTER Q WITH HOOK TAIL] +"\u024B" => "q" + +# Ê  [LATIN SMALL LETTER Q WITH HOOK] +"\u02A0" => "q" + +# â“  [CIRCLED LATIN SMALL LETTER Q] +"\u24E0" => "q" + +# ê— [LATIN SMALL LETTER Q WITH STROKE THROUGH DESCENDER] +"\uA757" => "q" + +# ê™ [LATIN SMALL LETTER Q WITH DIAGONAL STROKE] +"\uA759" => "q" + +# ｑ [FULLWIDTH LATIN SMALL LETTER Q] +"\uFF51" => "q" + +# â’¬ [PARENTHESIZED LATIN SMALL LETTER Q] +"\u24AC" => "(q)" + +# ȹ [LATIN SMALL LETTER QP DIGRAPH] +"\u0239" => "qp" + +# Å” [LATIN CAPITAL LETTER R WITH ACUTE] +"\u0154" => "R" + +# Å– [LATIN CAPITAL LETTER R WITH CEDILLA] +"\u0156" => "R" + +# Ř [LATIN CAPITAL LETTER R WITH CARON] +"\u0158" => "R" + +# È’ [LATIN CAPITAL LETTER R WITH DOUBLE GRAVE] +"\u0210" => "R" + +# È’ [LATIN CAPITAL LETTER R WITH INVERTED BREVE] +"\u0212" => "R" + +# ÉŒ [LATIN CAPITAL LETTER R WITH STROKE] +"\u024C" => "R" + +# Ê€ [LATIN LETTER SMALL CAPITAL R] +"\u0280" => "R" + +# Ê [LATIN LETTER SMALL CAPITAL INVERTED R] +"\u0281" => "R" + +# á´™ [LATIN LETTER SMALL CAPITAL REVERSED R] +"\u1D19" => "R" + +# á´š [LATIN LETTER SMALL CAPITAL TURNED R] +"\u1D1A" => "R" + +# Ṙ [LATIN CAPITAL LETTER R WITH DOT ABOVE] +"\u1E58" => "R" + +# Ṛ [LATIN CAPITAL LETTER R WITH DOT BELOW] +"\u1E5A" => "R" + +# Ṝ [LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON] +"\u1E5C" => "R" + +# Ṟ [LATIN CAPITAL LETTER R WITH LINE BELOW] +"\u1E5E" => "R" + +# Ⓡ [CIRCLED LATIN CAPITAL LETTER R] +"\u24C7" => "R" + +# Ɽ [LATIN CAPITAL LETTER R WITH TAIL] +"\u2C64" => "R" + +# êš [LATIN CAPITAL LETTER R ROTUNDA] +"\uA75A" => "R" + +# êž‚ [LATIN CAPITAL LETTER INSULAR R] +"\uA782" => "R" + +# ï¼² [FULLWIDTH LATIN CAPITAL LETTER R] +"\uFF32" => "R" + +# Å• [LATIN SMALL LETTER R WITH ACUTE] +"\u0155" => "r" + +# Å— [LATIN SMALL LETTER R WITH CEDILLA] +"\u0157" => "r" + +# Å™ [LATIN SMALL LETTER R WITH CARON] +"\u0159" => "r" + +# È‘ [LATIN SMALL LETTER R WITH DOUBLE GRAVE] +"\u0211" => "r" + +# È“ [LATIN SMALL LETTER R WITH INVERTED BREVE] +"\u0213" => "r" + +# É [LATIN SMALL LETTER R WITH STROKE] +"\u024D" => "r" + +# ɼ [LATIN SMALL LETTER R WITH LONG LEG] +"\u027C" => "r" + +# ɽ [LATIN SMALL LETTER R WITH TAIL] +"\u027D" => "r" + +# ɾ [LATIN SMALL LETTER R WITH FISHHOOK] +"\u027E" => "r" + +# É¿ [LATIN SMALL LETTER REVERSED R WITH FISHHOOK] +"\u027F" => "r" + +# áµ£ [LATIN SUBSCRIPT SMALL LETTER R] +"\u1D63" => "r" + +# áµ² [LATIN SMALL LETTER R WITH MIDDLE TILDE] +"\u1D72" => "r" + +# áµ³ [LATIN SMALL LETTER R WITH FISHHOOK AND MIDDLE TILDE] +"\u1D73" => "r" + +# ᶉ [LATIN SMALL LETTER R WITH PALATAL HOOK] +"\u1D89" => "r" + +# á¹™ [LATIN SMALL LETTER R WITH DOT ABOVE] +"\u1E59" => "r" + +# á¹› [LATIN SMALL LETTER R WITH DOT BELOW] +"\u1E5B" => "r" + +# á¹ [LATIN SMALL LETTER R WITH DOT BELOW AND MACRON] +"\u1E5D" => "r" + +# ṟ [LATIN SMALL LETTER R WITH LINE BELOW] +"\u1E5F" => "r" + +# â“¡ [CIRCLED LATIN SMALL LETTER R] +"\u24E1" => "r" + +# ê› [LATIN SMALL LETTER R ROTUNDA] +"\uA75B" => "r" + +# ꞃ [LATIN SMALL LETTER INSULAR R] +"\uA783" => "r" + +# ï½’ [FULLWIDTH LATIN SMALL LETTER R] +"\uFF52" => "r" + +# â’­ [PARENTHESIZED LATIN SMALL LETTER R] +"\u24AD" => "(r)" + +# Åš [LATIN CAPITAL LETTER S WITH ACUTE] +"\u015A" => "S" + +# Åœ [LATIN CAPITAL LETTER S WITH CIRCUMFLEX] +"\u015C" => "S" + +# Åž [LATIN CAPITAL LETTER S WITH CEDILLA] +"\u015E" => "S" + +# Å  [LATIN CAPITAL LETTER S WITH CARON] +"\u0160" => "S" + +# Ș [LATIN CAPITAL LETTER S WITH COMMA BELOW] +"\u0218" => "S" + +# á¹  [LATIN CAPITAL LETTER S WITH DOT ABOVE] +"\u1E60" => "S" + +# á¹¢ [LATIN CAPITAL LETTER S WITH DOT BELOW] +"\u1E62" => "S" + +# Ṥ [LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE] +"\u1E64" => "S" + +# Ṧ [LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE] +"\u1E66" => "S" + +# Ṩ [LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE] +"\u1E68" => "S" + +# Ⓢ [CIRCLED LATIN CAPITAL LETTER S] +"\u24C8" => "S" + +# ꜱ [LATIN LETTER SMALL CAPITAL S] +"\uA731" => "S" + +# êž… [LATIN SMALL LETTER INSULAR S] +"\uA785" => "S" + +# ï¼³ [FULLWIDTH LATIN CAPITAL LETTER S] +"\uFF33" => "S" + +# Å› [LATIN SMALL LETTER S WITH ACUTE] +"\u015B" => "s" + +# Å [LATIN SMALL LETTER S WITH CIRCUMFLEX] +"\u015D" => "s" + +# ÅŸ [LATIN SMALL LETTER S WITH CEDILLA] +"\u015F" => "s" + +# Å¡ [LATIN SMALL LETTER S WITH CARON] +"\u0161" => "s" + +# Å¿ http://en.wikipedia.org/wiki/Long_S [LATIN SMALL LETTER LONG S] +"\u017F" => "s" + +# È™ [LATIN SMALL LETTER S WITH COMMA BELOW] +"\u0219" => "s" + +# È¿ [LATIN SMALL LETTER S WITH SWASH TAIL] +"\u023F" => "s" + +# Ê‚ [LATIN SMALL LETTER S WITH HOOK] +"\u0282" => "s" + +# áµ´ [LATIN SMALL LETTER S WITH MIDDLE TILDE] +"\u1D74" => "s" + +# ᶊ [LATIN SMALL LETTER S WITH PALATAL HOOK] +"\u1D8A" => "s" + +# ṡ [LATIN SMALL LETTER S WITH DOT ABOVE] +"\u1E61" => "s" + +# á¹£ [LATIN SMALL LETTER S WITH DOT BELOW] +"\u1E63" => "s" + +# á¹¥ [LATIN SMALL LETTER S WITH ACUTE AND DOT ABOVE] +"\u1E65" => "s" + +# ṧ [LATIN SMALL LETTER S WITH CARON AND DOT ABOVE] +"\u1E67" => "s" + +# ṩ [LATIN SMALL LETTER S WITH DOT BELOW AND DOT ABOVE] +"\u1E69" => "s" + +# ẜ [LATIN SMALL LETTER LONG S WITH DIAGONAL STROKE] +"\u1E9C" => "s" + +# Ạ[LATIN SMALL LETTER LONG S WITH HIGH STROKE] +"\u1E9D" => "s" + +# â“¢ [CIRCLED LATIN SMALL LETTER S] +"\u24E2" => "s" + +# êž„ [LATIN CAPITAL LETTER INSULAR S] +"\uA784" => "s" + +# ｓ [FULLWIDTH LATIN SMALL LETTER S] +"\uFF53" => "s" + +# ẞ [LATIN CAPITAL LETTER SHARP S] +"\u1E9E" => "SS" + +# â’® [PARENTHESIZED LATIN SMALL LETTER S] +"\u24AE" => "(s)" + +# ß [LATIN SMALL LETTER SHARP S] +"\u00DF" => "ss" + +# st [LATIN SMALL LIGATURE ST] +"\uFB06" => "st" + +# Å¢ [LATIN CAPITAL LETTER T WITH CEDILLA] +"\u0162" => "T" + +# Ť [LATIN CAPITAL LETTER T WITH CARON] +"\u0164" => "T" + +# Ŧ [LATIN CAPITAL LETTER T WITH STROKE] +"\u0166" => "T" + +# Ƭ [LATIN CAPITAL LETTER T WITH HOOK] +"\u01AC" => "T" + +# Æ® [LATIN CAPITAL LETTER T WITH RETROFLEX HOOK] +"\u01AE" => "T" + +# Èš [LATIN CAPITAL LETTER T WITH COMMA BELOW] +"\u021A" => "T" + +# Ⱦ [LATIN CAPITAL LETTER T WITH DIAGONAL STROKE] +"\u023E" => "T" + +# á´› [LATIN LETTER SMALL CAPITAL T] +"\u1D1B" => "T" + +# Ṫ [LATIN CAPITAL LETTER T WITH DOT ABOVE] +"\u1E6A" => "T" + +# Ṭ [LATIN CAPITAL LETTER T WITH DOT BELOW] +"\u1E6C" => "T" + +# á¹® [LATIN CAPITAL LETTER T WITH LINE BELOW] +"\u1E6E" => "T" + +# á¹° [LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW] +"\u1E70" => "T" + +# Ⓣ [CIRCLED LATIN CAPITAL LETTER T] +"\u24C9" => "T" + +# Ꞇ [LATIN CAPITAL LETTER INSULAR T] +"\uA786" => "T" + +# ï¼´ [FULLWIDTH LATIN CAPITAL LETTER T] +"\uFF34" => "T" + +# Å£ [LATIN SMALL LETTER T WITH CEDILLA] +"\u0163" => "t" + +# Å¥ [LATIN SMALL LETTER T WITH CARON] +"\u0165" => "t" + +# ŧ [LATIN SMALL LETTER T WITH STROKE] +"\u0167" => "t" + +# Æ« [LATIN SMALL LETTER T WITH PALATAL HOOK] +"\u01AB" => "t" + +# Æ­ [LATIN SMALL LETTER T WITH HOOK] +"\u01AD" => "t" + +# È› [LATIN SMALL LETTER T WITH COMMA BELOW] +"\u021B" => "t" + +# ȶ [LATIN SMALL LETTER T WITH CURL] +"\u0236" => "t" + +# ʇ [LATIN SMALL LETTER TURNED T] +"\u0287" => "t" + +# ʈ [LATIN SMALL LETTER T WITH RETROFLEX HOOK] +"\u0288" => "t" + +# áµµ [LATIN SMALL LETTER T WITH MIDDLE TILDE] +"\u1D75" => "t" + +# ṫ [LATIN SMALL LETTER T WITH DOT ABOVE] +"\u1E6B" => "t" + +# á¹­ [LATIN SMALL LETTER T WITH DOT BELOW] +"\u1E6D" => "t" + +# ṯ [LATIN SMALL LETTER T WITH LINE BELOW] +"\u1E6F" => "t" + +# á¹± [LATIN SMALL LETTER T WITH CIRCUMFLEX BELOW] +"\u1E71" => "t" + +# ẗ [LATIN SMALL LETTER T WITH DIAERESIS] +"\u1E97" => "t" + +# â“£ [CIRCLED LATIN SMALL LETTER T] +"\u24E3" => "t" + +# ⱦ [LATIN SMALL LETTER T WITH DIAGONAL STROKE] +"\u2C66" => "t" + +# ï½” [FULLWIDTH LATIN SMALL LETTER T] +"\uFF54" => "t" + +# Þ [LATIN CAPITAL LETTER THORN] +"\u00DE" => "TH" + +# ê¦ [LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER] +"\uA766" => "TH" + +# Ꜩ [LATIN CAPITAL LETTER TZ] +"\uA728" => "TZ" + +# â’¯ [PARENTHESIZED LATIN SMALL LETTER T] +"\u24AF" => "(t)" + +# ʨ [LATIN SMALL LETTER TC DIGRAPH WITH CURL] +"\u02A8" => "tc" + +# þ [LATIN SMALL LETTER THORN] +"\u00FE" => "th" + +# ᵺ [LATIN SMALL LETTER TH WITH STRIKETHROUGH] +"\u1D7A" => "th" + +# ê§ [LATIN SMALL LETTER THORN WITH STROKE THROUGH DESCENDER] +"\uA767" => "th" + +# ʦ [LATIN SMALL LETTER TS DIGRAPH] +"\u02A6" => "ts" + +# ꜩ [LATIN SMALL LETTER TZ] +"\uA729" => "tz" + +# Ù [LATIN CAPITAL LETTER U WITH GRAVE] +"\u00D9" => "U" + +# Ú [LATIN CAPITAL LETTER U WITH ACUTE] +"\u00DA" => "U" + +# Û [LATIN CAPITAL LETTER U WITH CIRCUMFLEX] +"\u00DB" => "U" + +# Ãœ [LATIN CAPITAL LETTER U WITH DIAERESIS] +"\u00DC" => "U" + +# Ũ [LATIN CAPITAL LETTER U WITH TILDE] +"\u0168" => "U" + +# Ū [LATIN CAPITAL LETTER U WITH MACRON] +"\u016A" => "U" + +# Ŭ [LATIN CAPITAL LETTER U WITH BREVE] +"\u016C" => "U" + +# Å® [LATIN CAPITAL LETTER U WITH RING ABOVE] +"\u016E" => "U" + +# Å° [LATIN CAPITAL LETTER U WITH DOUBLE ACUTE] +"\u0170" => "U" + +# Ų [LATIN CAPITAL LETTER U WITH OGONEK] +"\u0172" => "U" + +# Ư [LATIN CAPITAL LETTER U WITH HORN] +"\u01AF" => "U" + +# Ç“ [LATIN CAPITAL LETTER U WITH CARON] +"\u01D3" => "U" + +# Ç• [LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON] +"\u01D5" => "U" + +# Ç— [LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE] +"\u01D7" => "U" + +# Ç™ [LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON] +"\u01D9" => "U" + +# Ç› [LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE] +"\u01DB" => "U" + +# È” [LATIN CAPITAL LETTER U WITH DOUBLE GRAVE] +"\u0214" => "U" + +# È– [LATIN CAPITAL LETTER U WITH INVERTED BREVE] +"\u0216" => "U" + +# É„ [LATIN CAPITAL LETTER U BAR] +"\u0244" => "U" + +# á´œ [LATIN LETTER SMALL CAPITAL U] +"\u1D1C" => "U" + +# áµ¾ [LATIN SMALL CAPITAL LETTER U WITH STROKE] +"\u1D7E" => "U" + +# á¹² [LATIN CAPITAL LETTER U WITH DIAERESIS BELOW] +"\u1E72" => "U" + +# á¹´ [LATIN CAPITAL LETTER U WITH TILDE BELOW] +"\u1E74" => "U" + +# Ṷ [LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW] +"\u1E76" => "U" + +# Ṹ [LATIN CAPITAL LETTER U WITH TILDE AND ACUTE] +"\u1E78" => "U" + +# Ṻ [LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS] +"\u1E7A" => "U" + +# Ụ [LATIN CAPITAL LETTER U WITH DOT BELOW] +"\u1EE4" => "U" + +# Ủ [LATIN CAPITAL LETTER U WITH HOOK ABOVE] +"\u1EE6" => "U" + +# Ứ [LATIN CAPITAL LETTER U WITH HORN AND ACUTE] +"\u1EE8" => "U" + +# Ừ [LATIN CAPITAL LETTER U WITH HORN AND GRAVE] +"\u1EEA" => "U" + +# Ử [LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE] +"\u1EEC" => "U" + +# á»® [LATIN CAPITAL LETTER U WITH HORN AND TILDE] +"\u1EEE" => "U" + +# á»° [LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW] +"\u1EF0" => "U" + +# â“Š [CIRCLED LATIN CAPITAL LETTER U] +"\u24CA" => "U" + +# ï¼µ [FULLWIDTH LATIN CAPITAL LETTER U] +"\uFF35" => "U" + +# ù [LATIN SMALL LETTER U WITH GRAVE] +"\u00F9" => "u" + +# ú [LATIN SMALL LETTER U WITH ACUTE] +"\u00FA" => "u" + +# û [LATIN SMALL LETTER U WITH CIRCUMFLEX] +"\u00FB" => "u" + +# ü [LATIN SMALL LETTER U WITH DIAERESIS] +"\u00FC" => "u" + +# Å© [LATIN SMALL LETTER U WITH TILDE] +"\u0169" => "u" + +# Å« [LATIN SMALL LETTER U WITH MACRON] +"\u016B" => "u" + +# Å­ [LATIN SMALL LETTER U WITH BREVE] +"\u016D" => "u" + +# ů [LATIN SMALL LETTER U WITH RING ABOVE] +"\u016F" => "u" + +# ű [LATIN SMALL LETTER U WITH DOUBLE ACUTE] +"\u0171" => "u" + +# ų [LATIN SMALL LETTER U WITH OGONEK] +"\u0173" => "u" + +# Æ° [LATIN SMALL LETTER U WITH HORN] +"\u01B0" => "u" + +# Ç” [LATIN SMALL LETTER U WITH CARON] +"\u01D4" => "u" + +# Ç– [LATIN SMALL LETTER U WITH DIAERESIS AND MACRON] +"\u01D6" => "u" + +# ǘ [LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE] +"\u01D8" => "u" + +# Çš [LATIN SMALL LETTER U WITH DIAERESIS AND CARON] +"\u01DA" => "u" + +# Çœ [LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE] +"\u01DC" => "u" + +# È• [LATIN SMALL LETTER U WITH DOUBLE GRAVE] +"\u0215" => "u" + +# È— [LATIN SMALL LETTER U WITH INVERTED BREVE] +"\u0217" => "u" + +# ʉ [LATIN SMALL LETTER U BAR] +"\u0289" => "u" + +# ᵤ [LATIN SUBSCRIPT SMALL LETTER U] +"\u1D64" => "u" + +# ᶙ [LATIN SMALL LETTER U WITH RETROFLEX HOOK] +"\u1D99" => "u" + +# á¹³ [LATIN SMALL LETTER U WITH DIAERESIS BELOW] +"\u1E73" => "u" + +# á¹µ [LATIN SMALL LETTER U WITH TILDE BELOW] +"\u1E75" => "u" + +# á¹· [LATIN SMALL LETTER U WITH CIRCUMFLEX BELOW] +"\u1E77" => "u" + +# á¹¹ [LATIN SMALL LETTER U WITH TILDE AND ACUTE] +"\u1E79" => "u" + +# á¹» [LATIN SMALL LETTER U WITH MACRON AND DIAERESIS] +"\u1E7B" => "u" + +# ụ [LATIN SMALL LETTER U WITH DOT BELOW] +"\u1EE5" => "u" + +# ủ [LATIN SMALL LETTER U WITH HOOK ABOVE] +"\u1EE7" => "u" + +# ứ [LATIN SMALL LETTER U WITH HORN AND ACUTE] +"\u1EE9" => "u" + +# ừ [LATIN SMALL LETTER U WITH HORN AND GRAVE] +"\u1EEB" => "u" + +# á»­ [LATIN SMALL LETTER U WITH HORN AND HOOK ABOVE] +"\u1EED" => "u" + +# ữ [LATIN SMALL LETTER U WITH HORN AND TILDE] +"\u1EEF" => "u" + +# á»± [LATIN SMALL LETTER U WITH HORN AND DOT BELOW] +"\u1EF1" => "u" + +# ⓤ [CIRCLED LATIN SMALL LETTER U] +"\u24E4" => "u" + +# ｕ [FULLWIDTH LATIN SMALL LETTER U] +"\uFF55" => "u" + +# â’° [PARENTHESIZED LATIN SMALL LETTER U] +"\u24B0" => "(u)" + +# ᵫ [LATIN SMALL LETTER UE] +"\u1D6B" => "ue" + +# Ʋ [LATIN CAPITAL LETTER V WITH HOOK] +"\u01B2" => "V" + +# É… [LATIN CAPITAL LETTER TURNED V] +"\u0245" => "V" + +# á´  [LATIN LETTER SMALL CAPITAL V] +"\u1D20" => "V" + +# á¹¼ [LATIN CAPITAL LETTER V WITH TILDE] +"\u1E7C" => "V" + +# á¹¾ [LATIN CAPITAL LETTER V WITH DOT BELOW] +"\u1E7E" => "V" + +# Ỽ [LATIN CAPITAL LETTER MIDDLE-WELSH V] +"\u1EFC" => "V" + +# â“‹ [CIRCLED LATIN CAPITAL LETTER V] +"\u24CB" => "V" + +# êž [LATIN CAPITAL LETTER V WITH DIAGONAL STROKE] +"\uA75E" => "V" + +# ê¨ [LATIN CAPITAL LETTER VEND] +"\uA768" => "V" + +# Ｖ [FULLWIDTH LATIN CAPITAL LETTER V] +"\uFF36" => "V" + +# Ê‹ [LATIN SMALL LETTER V WITH HOOK] +"\u028B" => "v" + +# ÊŒ [LATIN SMALL LETTER TURNED V] +"\u028C" => "v" + +# áµ¥ [LATIN SUBSCRIPT SMALL LETTER V] +"\u1D65" => "v" + +# ᶌ [LATIN SMALL LETTER V WITH PALATAL HOOK] +"\u1D8C" => "v" + +# á¹½ [LATIN SMALL LETTER V WITH TILDE] +"\u1E7D" => "v" + +# ṿ [LATIN SMALL LETTER V WITH DOT BELOW] +"\u1E7F" => "v" + +# â“¥ [CIRCLED LATIN SMALL LETTER V] +"\u24E5" => "v" + +# â±± [LATIN SMALL LETTER V WITH RIGHT HOOK] +"\u2C71" => "v" + +# â±´ [LATIN SMALL LETTER V WITH CURL] +"\u2C74" => "v" + +# êŸ [LATIN SMALL LETTER V WITH DIAGONAL STROKE] +"\uA75F" => "v" + +# ï½– [FULLWIDTH LATIN SMALL LETTER V] +"\uFF56" => "v" + +# ê  [LATIN CAPITAL LETTER VY] +"\uA760" => "VY" + +# â’± [PARENTHESIZED LATIN SMALL LETTER V] +"\u24B1" => "(v)" + +# ê¡ [LATIN SMALL LETTER VY] +"\uA761" => "vy" + +# Å´ [LATIN CAPITAL LETTER W WITH CIRCUMFLEX] +"\u0174" => "W" + +# Ç· http://en.wikipedia.org/wiki/Wynn [LATIN CAPITAL LETTER WYNN] +"\u01F7" => "W" + +# á´¡ [LATIN LETTER SMALL CAPITAL W] +"\u1D21" => "W" + +# Ẁ [LATIN CAPITAL LETTER W WITH GRAVE] +"\u1E80" => "W" + +# Ẃ [LATIN CAPITAL LETTER W WITH ACUTE] +"\u1E82" => "W" + +# Ẅ [LATIN CAPITAL LETTER W WITH DIAERESIS] +"\u1E84" => "W" + +# Ẇ [LATIN CAPITAL LETTER W WITH DOT ABOVE] +"\u1E86" => "W" + +# Ẉ [LATIN CAPITAL LETTER W WITH DOT BELOW] +"\u1E88" => "W" + +# â“Œ [CIRCLED LATIN CAPITAL LETTER W] +"\u24CC" => "W" + +# â±² [LATIN CAPITAL LETTER W WITH HOOK] +"\u2C72" => "W" + +# ï¼· [FULLWIDTH LATIN CAPITAL LETTER W] +"\uFF37" => "W" + +# ŵ [LATIN SMALL LETTER W WITH CIRCUMFLEX] +"\u0175" => "w" + +# Æ¿ http://en.wikipedia.org/wiki/Wynn [LATIN LETTER WYNN] +"\u01BF" => "w" + +# Ê [LATIN SMALL LETTER TURNED W] +"\u028D" => "w" + +# Ạ[LATIN SMALL LETTER W WITH GRAVE] +"\u1E81" => "w" + +# ẃ [LATIN SMALL LETTER W WITH ACUTE] +"\u1E83" => "w" + +# ẅ [LATIN SMALL LETTER W WITH DIAERESIS] +"\u1E85" => "w" + +# ẇ [LATIN SMALL LETTER W WITH DOT ABOVE] +"\u1E87" => "w" + +# ẉ [LATIN SMALL LETTER W WITH DOT BELOW] +"\u1E89" => "w" + +# ẘ [LATIN SMALL LETTER W WITH RING ABOVE] +"\u1E98" => "w" + +# ⓦ [CIRCLED LATIN SMALL LETTER W] +"\u24E6" => "w" + +# â±³ [LATIN SMALL LETTER W WITH HOOK] +"\u2C73" => "w" + +# ï½— [FULLWIDTH LATIN SMALL LETTER W] +"\uFF57" => "w" + +# â’² [PARENTHESIZED LATIN SMALL LETTER W] +"\u24B2" => "(w)" + +# Ẋ [LATIN CAPITAL LETTER X WITH DOT ABOVE] +"\u1E8A" => "X" + +# Ẍ [LATIN CAPITAL LETTER X WITH DIAERESIS] +"\u1E8C" => "X" + +# â“ [CIRCLED LATIN CAPITAL LETTER X] +"\u24CD" => "X" + +# Ｘ [FULLWIDTH LATIN CAPITAL LETTER X] +"\uFF38" => "X" + +# ᶠ[LATIN SMALL LETTER X WITH PALATAL HOOK] +"\u1D8D" => "x" + +# ẋ [LATIN SMALL LETTER X WITH DOT ABOVE] +"\u1E8B" => "x" + +# Ạ[LATIN SMALL LETTER X WITH DIAERESIS] +"\u1E8D" => "x" + +# â‚“ [LATIN SUBSCRIPT SMALL LETTER X] +"\u2093" => "x" + +# ⓧ [CIRCLED LATIN SMALL LETTER X] +"\u24E7" => "x" + +# ｘ [FULLWIDTH LATIN SMALL LETTER X] +"\uFF58" => "x" + +# â’³ [PARENTHESIZED LATIN SMALL LETTER X] +"\u24B3" => "(x)" + +# à [LATIN CAPITAL LETTER Y WITH ACUTE] +"\u00DD" => "Y" + +# Ŷ [LATIN CAPITAL LETTER Y WITH CIRCUMFLEX] +"\u0176" => "Y" + +# Ÿ [LATIN CAPITAL LETTER Y WITH DIAERESIS] +"\u0178" => "Y" + +# Ƴ [LATIN CAPITAL LETTER Y WITH HOOK] +"\u01B3" => "Y" + +# Ȳ [LATIN CAPITAL LETTER Y WITH MACRON] +"\u0232" => "Y" + +# ÉŽ [LATIN CAPITAL LETTER Y WITH STROKE] +"\u024E" => "Y" + +# Ê [LATIN LETTER SMALL CAPITAL Y] +"\u028F" => "Y" + +# Ẏ [LATIN CAPITAL LETTER Y WITH DOT ABOVE] +"\u1E8E" => "Y" + +# Ỳ [LATIN CAPITAL LETTER Y WITH GRAVE] +"\u1EF2" => "Y" + +# á»´ [LATIN CAPITAL LETTER Y WITH DOT BELOW] +"\u1EF4" => "Y" + +# Ỷ [LATIN CAPITAL LETTER Y WITH HOOK ABOVE] +"\u1EF6" => "Y" + +# Ỹ [LATIN CAPITAL LETTER Y WITH TILDE] +"\u1EF8" => "Y" + +# Ỿ [LATIN CAPITAL LETTER Y WITH LOOP] +"\u1EFE" => "Y" + +# â“Ž [CIRCLED LATIN CAPITAL LETTER Y] +"\u24CE" => "Y" + +# ï¼¹ [FULLWIDTH LATIN CAPITAL LETTER Y] +"\uFF39" => "Y" + +# ý [LATIN SMALL LETTER Y WITH ACUTE] +"\u00FD" => "y" + +# ÿ [LATIN SMALL LETTER Y WITH DIAERESIS] +"\u00FF" => "y" + +# Å· [LATIN SMALL LETTER Y WITH CIRCUMFLEX] +"\u0177" => "y" + +# Æ´ [LATIN SMALL LETTER Y WITH HOOK] +"\u01B4" => "y" + +# ȳ [LATIN SMALL LETTER Y WITH MACRON] +"\u0233" => "y" + +# É [LATIN SMALL LETTER Y WITH STROKE] +"\u024F" => "y" + +# ÊŽ [LATIN SMALL LETTER TURNED Y] +"\u028E" => "y" + +# Ạ[LATIN SMALL LETTER Y WITH DOT ABOVE] +"\u1E8F" => "y" + +# ẙ [LATIN SMALL LETTER Y WITH RING ABOVE] +"\u1E99" => "y" + +# ỳ [LATIN SMALL LETTER Y WITH GRAVE] +"\u1EF3" => "y" + +# ỵ [LATIN SMALL LETTER Y WITH DOT BELOW] +"\u1EF5" => "y" + +# á»· [LATIN SMALL LETTER Y WITH HOOK ABOVE] +"\u1EF7" => "y" + +# ỹ [LATIN SMALL LETTER Y WITH TILDE] +"\u1EF9" => "y" + +# ỿ [LATIN SMALL LETTER Y WITH LOOP] +"\u1EFF" => "y" + +# ⓨ [CIRCLED LATIN SMALL LETTER Y] +"\u24E8" => "y" + +# ï½™ [FULLWIDTH LATIN SMALL LETTER Y] +"\uFF59" => "y" + +# â’´ [PARENTHESIZED LATIN SMALL LETTER Y] +"\u24B4" => "(y)" + +# Ź [LATIN CAPITAL LETTER Z WITH ACUTE] +"\u0179" => "Z" + +# Å» [LATIN CAPITAL LETTER Z WITH DOT ABOVE] +"\u017B" => "Z" + +# Ž [LATIN CAPITAL LETTER Z WITH CARON] +"\u017D" => "Z" + +# Ƶ [LATIN CAPITAL LETTER Z WITH STROKE] +"\u01B5" => "Z" + +# Èœ http://en.wikipedia.org/wiki/Yogh [LATIN CAPITAL LETTER YOGH] +"\u021C" => "Z" + +# Ȥ [LATIN CAPITAL LETTER Z WITH HOOK] +"\u0224" => "Z" + +# á´¢ [LATIN LETTER SMALL CAPITAL Z] +"\u1D22" => "Z" + +# Ạ[LATIN CAPITAL LETTER Z WITH CIRCUMFLEX] +"\u1E90" => "Z" + +# Ẓ [LATIN CAPITAL LETTER Z WITH DOT BELOW] +"\u1E92" => "Z" + +# Ẕ [LATIN CAPITAL LETTER Z WITH LINE BELOW] +"\u1E94" => "Z" + +# â“ [CIRCLED LATIN CAPITAL LETTER Z] +"\u24CF" => "Z" + +# Ⱬ [LATIN CAPITAL LETTER Z WITH DESCENDER] +"\u2C6B" => "Z" + +# ê¢ [LATIN CAPITAL LETTER VISIGOTHIC Z] +"\uA762" => "Z" + +# Ｚ [FULLWIDTH LATIN CAPITAL LETTER Z] +"\uFF3A" => "Z" + +# ź [LATIN SMALL LETTER Z WITH ACUTE] +"\u017A" => "z" + +# ż [LATIN SMALL LETTER Z WITH DOT ABOVE] +"\u017C" => "z" + +# ž [LATIN SMALL LETTER Z WITH CARON] +"\u017E" => "z" + +# ƶ [LATIN SMALL LETTER Z WITH STROKE] +"\u01B6" => "z" + +# È http://en.wikipedia.org/wiki/Yogh [LATIN SMALL LETTER YOGH] +"\u021D" => "z" + +# È¥ [LATIN SMALL LETTER Z WITH HOOK] +"\u0225" => "z" + +# É€ [LATIN SMALL LETTER Z WITH SWASH TAIL] +"\u0240" => "z" + +# Ê [LATIN SMALL LETTER Z WITH RETROFLEX HOOK] +"\u0290" => "z" + +# Ê‘ [LATIN SMALL LETTER Z WITH CURL] +"\u0291" => "z" + +# ᵶ [LATIN SMALL LETTER Z WITH MIDDLE TILDE] +"\u1D76" => "z" + +# ᶎ [LATIN SMALL LETTER Z WITH PALATAL HOOK] +"\u1D8E" => "z" + +# ẑ [LATIN SMALL LETTER Z WITH CIRCUMFLEX] +"\u1E91" => "z" + +# ẓ [LATIN SMALL LETTER Z WITH DOT BELOW] +"\u1E93" => "z" + +# ẕ [LATIN SMALL LETTER Z WITH LINE BELOW] +"\u1E95" => "z" + +# â“© [CIRCLED LATIN SMALL LETTER Z] +"\u24E9" => "z" + +# ⱬ [LATIN SMALL LETTER Z WITH DESCENDER] +"\u2C6C" => "z" + +# ê£ [LATIN SMALL LETTER VISIGOTHIC Z] +"\uA763" => "z" + +# ｚ [FULLWIDTH LATIN SMALL LETTER Z] +"\uFF5A" => "z" + +# â’µ [PARENTHESIZED LATIN SMALL LETTER Z] +"\u24B5" => "(z)" + +# â° [SUPERSCRIPT ZERO] +"\u2070" => "0" + +# â‚€ [SUBSCRIPT ZERO] +"\u2080" => "0" + +# ⓪ [CIRCLED DIGIT ZERO] +"\u24EA" => "0" + +# â“¿ [NEGATIVE CIRCLED DIGIT ZERO] +"\u24FF" => "0" + +# ï¼ [FULLWIDTH DIGIT ZERO] +"\uFF10" => "0" + +# ¹ [SUPERSCRIPT ONE] +"\u00B9" => "1" + +# â‚ [SUBSCRIPT ONE] +"\u2081" => "1" + +# â‘  [CIRCLED DIGIT ONE] +"\u2460" => "1" + +# ⓵ [DOUBLE CIRCLED DIGIT ONE] +"\u24F5" => "1" + +# ⶠ[DINGBAT NEGATIVE CIRCLED DIGIT ONE] +"\u2776" => "1" + +# ➀ [DINGBAT CIRCLED SANS-SERIF DIGIT ONE] +"\u2780" => "1" + +# ➊ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT ONE] +"\u278A" => "1" + +# １ [FULLWIDTH DIGIT ONE] +"\uFF11" => "1" + +# â’ˆ [DIGIT ONE FULL STOP] +"\u2488" => "1." + +# â‘´ [PARENTHESIZED DIGIT ONE] +"\u2474" => "(1)" + +# ² [SUPERSCRIPT TWO] +"\u00B2" => "2" + +# â‚‚ [SUBSCRIPT TWO] +"\u2082" => "2" + +# â‘¡ [CIRCLED DIGIT TWO] +"\u2461" => "2" + +# ⓶ [DOUBLE CIRCLED DIGIT TWO] +"\u24F6" => "2" + +# â· [DINGBAT NEGATIVE CIRCLED DIGIT TWO] +"\u2777" => "2" + +# âž [DINGBAT CIRCLED SANS-SERIF DIGIT TWO] +"\u2781" => "2" + +# âž‹ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT TWO] +"\u278B" => "2" + +# ï¼’ [FULLWIDTH DIGIT TWO] +"\uFF12" => "2" + +# â’‰ [DIGIT TWO FULL STOP] +"\u2489" => "2." + +# ⑵ [PARENTHESIZED DIGIT TWO] +"\u2475" => "(2)" + +# ³ [SUPERSCRIPT THREE] +"\u00B3" => "3" + +# ₃ [SUBSCRIPT THREE] +"\u2083" => "3" + +# â‘¢ [CIRCLED DIGIT THREE] +"\u2462" => "3" + +# â“· [DOUBLE CIRCLED DIGIT THREE] +"\u24F7" => "3" + +# ⸠[DINGBAT NEGATIVE CIRCLED DIGIT THREE] +"\u2778" => "3" + +# âž‚ [DINGBAT CIRCLED SANS-SERIF DIGIT THREE] +"\u2782" => "3" + +# ➌ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT THREE] +"\u278C" => "3" + +# ３ [FULLWIDTH DIGIT THREE] +"\uFF13" => "3" + +# â’Š [DIGIT THREE FULL STOP] +"\u248A" => "3." + +# ⑶ [PARENTHESIZED DIGIT THREE] +"\u2476" => "(3)" + +# â´ [SUPERSCRIPT FOUR] +"\u2074" => "4" + +# â‚„ [SUBSCRIPT FOUR] +"\u2084" => "4" + +# â‘£ [CIRCLED DIGIT FOUR] +"\u2463" => "4" + +# ⓸ [DOUBLE CIRCLED DIGIT FOUR] +"\u24F8" => "4" + +# â¹ [DINGBAT NEGATIVE CIRCLED DIGIT FOUR] +"\u2779" => "4" + +# ➃ [DINGBAT CIRCLED SANS-SERIF DIGIT FOUR] +"\u2783" => "4" + +# âž [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT FOUR] +"\u278D" => "4" + +# ï¼” [FULLWIDTH DIGIT FOUR] +"\uFF14" => "4" + +# â’‹ [DIGIT FOUR FULL STOP] +"\u248B" => "4." + +# â‘· [PARENTHESIZED DIGIT FOUR] +"\u2477" => "(4)" + +# âµ [SUPERSCRIPT FIVE] +"\u2075" => "5" + +# â‚… [SUBSCRIPT FIVE] +"\u2085" => "5" + +# ⑤ [CIRCLED DIGIT FIVE] +"\u2464" => "5" + +# ⓹ [DOUBLE CIRCLED DIGIT FIVE] +"\u24F9" => "5" + +# ⺠[DINGBAT NEGATIVE CIRCLED DIGIT FIVE] +"\u277A" => "5" + +# âž„ [DINGBAT CIRCLED SANS-SERIF DIGIT FIVE] +"\u2784" => "5" + +# ➎ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT FIVE] +"\u278E" => "5" + +# ５ [FULLWIDTH DIGIT FIVE] +"\uFF15" => "5" + +# â’Œ [DIGIT FIVE FULL STOP] +"\u248C" => "5." + +# ⑸ [PARENTHESIZED DIGIT FIVE] +"\u2478" => "(5)" + +# ⶠ[SUPERSCRIPT SIX] +"\u2076" => "6" + +# ₆ [SUBSCRIPT SIX] +"\u2086" => "6" + +# â‘¥ [CIRCLED DIGIT SIX] +"\u2465" => "6" + +# ⓺ [DOUBLE CIRCLED DIGIT SIX] +"\u24FA" => "6" + +# â» [DINGBAT NEGATIVE CIRCLED DIGIT SIX] +"\u277B" => "6" + +# âž… [DINGBAT CIRCLED SANS-SERIF DIGIT SIX] +"\u2785" => "6" + +# âž [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT SIX] +"\u278F" => "6" + +# ï¼– [FULLWIDTH DIGIT SIX] +"\uFF16" => "6" + +# â’ [DIGIT SIX FULL STOP] +"\u248D" => "6." + +# ⑹ [PARENTHESIZED DIGIT SIX] +"\u2479" => "(6)" + +# â· [SUPERSCRIPT SEVEN] +"\u2077" => "7" + +# ₇ [SUBSCRIPT SEVEN] +"\u2087" => "7" + +# ⑦ [CIRCLED DIGIT SEVEN] +"\u2466" => "7" + +# â“» [DOUBLE CIRCLED DIGIT SEVEN] +"\u24FB" => "7" + +# â¼ [DINGBAT NEGATIVE CIRCLED DIGIT SEVEN] +"\u277C" => "7" + +# ➆ [DINGBAT CIRCLED SANS-SERIF DIGIT SEVEN] +"\u2786" => "7" + +# âž [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT SEVEN] +"\u2790" => "7" + +# ï¼— [FULLWIDTH DIGIT SEVEN] +"\uFF17" => "7" + +# â’Ž [DIGIT SEVEN FULL STOP] +"\u248E" => "7." + +# ⑺ [PARENTHESIZED DIGIT SEVEN] +"\u247A" => "(7)" + +# ⸠[SUPERSCRIPT EIGHT] +"\u2078" => "8" + +# ₈ [SUBSCRIPT EIGHT] +"\u2088" => "8" + +# ⑧ [CIRCLED DIGIT EIGHT] +"\u2467" => "8" + +# ⓼ [DOUBLE CIRCLED DIGIT EIGHT] +"\u24FC" => "8" + +# â½ [DINGBAT NEGATIVE CIRCLED DIGIT EIGHT] +"\u277D" => "8" + +# ➇ [DINGBAT CIRCLED SANS-SERIF DIGIT EIGHT] +"\u2787" => "8" + +# âž‘ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT EIGHT] +"\u2791" => "8" + +# ８ [FULLWIDTH DIGIT EIGHT] +"\uFF18" => "8" + +# â’ [DIGIT EIGHT FULL STOP] +"\u248F" => "8." + +# â‘» [PARENTHESIZED DIGIT EIGHT] +"\u247B" => "(8)" + +# â¹ [SUPERSCRIPT NINE] +"\u2079" => "9" + +# ₉ [SUBSCRIPT NINE] +"\u2089" => "9" + +# ⑨ [CIRCLED DIGIT NINE] +"\u2468" => "9" + +# ⓽ [DOUBLE CIRCLED DIGIT NINE] +"\u24FD" => "9" + +# â¾ [DINGBAT NEGATIVE CIRCLED DIGIT NINE] +"\u277E" => "9" + +# ➈ [DINGBAT CIRCLED SANS-SERIF DIGIT NINE] +"\u2788" => "9" + +# âž’ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT NINE] +"\u2792" => "9" + +# ï¼™ [FULLWIDTH DIGIT NINE] +"\uFF19" => "9" + +# â’ [DIGIT NINE FULL STOP] +"\u2490" => "9." + +# ⑼ [PARENTHESIZED DIGIT NINE] +"\u247C" => "(9)" + +# â‘© [CIRCLED NUMBER TEN] +"\u2469" => "10" + +# ⓾ [DOUBLE CIRCLED NUMBER TEN] +"\u24FE" => "10" + +# â¿ [DINGBAT NEGATIVE CIRCLED NUMBER TEN] +"\u277F" => "10" + +# ➉ [DINGBAT CIRCLED SANS-SERIF NUMBER TEN] +"\u2789" => "10" + +# âž“ [DINGBAT NEGATIVE CIRCLED SANS-SERIF NUMBER TEN] +"\u2793" => "10" + +# â’‘ [NUMBER TEN FULL STOP] +"\u2491" => "10." + +# ⑽ [PARENTHESIZED NUMBER TEN] +"\u247D" => "(10)" + +# ⑪ [CIRCLED NUMBER ELEVEN] +"\u246A" => "11" + +# â“« [NEGATIVE CIRCLED NUMBER ELEVEN] +"\u24EB" => "11" + +# â’’ [NUMBER ELEVEN FULL STOP] +"\u2492" => "11." + +# ⑾ [PARENTHESIZED NUMBER ELEVEN] +"\u247E" => "(11)" + +# â‘« [CIRCLED NUMBER TWELVE] +"\u246B" => "12" + +# ⓬ [NEGATIVE CIRCLED NUMBER TWELVE] +"\u24EC" => "12" + +# â’“ [NUMBER TWELVE FULL STOP] +"\u2493" => "12." + +# â‘¿ [PARENTHESIZED NUMBER TWELVE] +"\u247F" => "(12)" + +# ⑬ [CIRCLED NUMBER THIRTEEN] +"\u246C" => "13" + +# â“­ [NEGATIVE CIRCLED NUMBER THIRTEEN] +"\u24ED" => "13" + +# â’” [NUMBER THIRTEEN FULL STOP] +"\u2494" => "13." + +# â’€ [PARENTHESIZED NUMBER THIRTEEN] +"\u2480" => "(13)" + +# â‘­ [CIRCLED NUMBER FOURTEEN] +"\u246D" => "14" + +# â“® [NEGATIVE CIRCLED NUMBER FOURTEEN] +"\u24EE" => "14" + +# â’• [NUMBER FOURTEEN FULL STOP] +"\u2495" => "14." + +# â’ [PARENTHESIZED NUMBER FOURTEEN] +"\u2481" => "(14)" + +# â‘® [CIRCLED NUMBER FIFTEEN] +"\u246E" => "15" + +# ⓯ [NEGATIVE CIRCLED NUMBER FIFTEEN] +"\u24EF" => "15" + +# â’– [NUMBER FIFTEEN FULL STOP] +"\u2496" => "15." + +# â’‚ [PARENTHESIZED NUMBER FIFTEEN] +"\u2482" => "(15)" + +# ⑯ [CIRCLED NUMBER SIXTEEN] +"\u246F" => "16" + +# â“° [NEGATIVE CIRCLED NUMBER SIXTEEN] +"\u24F0" => "16" + +# â’— [NUMBER SIXTEEN FULL STOP] +"\u2497" => "16." + +# â’ƒ [PARENTHESIZED NUMBER SIXTEEN] +"\u2483" => "(16)" + +# â‘° [CIRCLED NUMBER SEVENTEEN] +"\u2470" => "17" + +# ⓱ [NEGATIVE CIRCLED NUMBER SEVENTEEN] +"\u24F1" => "17" + +# â’˜ [NUMBER SEVENTEEN FULL STOP] +"\u2498" => "17." + +# â’„ [PARENTHESIZED NUMBER SEVENTEEN] +"\u2484" => "(17)" + +# ⑱ [CIRCLED NUMBER EIGHTEEN] +"\u2471" => "18" + +# ⓲ [NEGATIVE CIRCLED NUMBER EIGHTEEN] +"\u24F2" => "18" + +# â’™ [NUMBER EIGHTEEN FULL STOP] +"\u2499" => "18." + +# â’… [PARENTHESIZED NUMBER EIGHTEEN] +"\u2485" => "(18)" + +# ⑲ [CIRCLED NUMBER NINETEEN] +"\u2472" => "19" + +# ⓳ [NEGATIVE CIRCLED NUMBER NINETEEN] +"\u24F3" => "19" + +# â’š [NUMBER NINETEEN FULL STOP] +"\u249A" => "19." + +# â’† [PARENTHESIZED NUMBER NINETEEN] +"\u2486" => "(19)" + +# ⑳ [CIRCLED NUMBER TWENTY] +"\u2473" => "20" + +# â“´ [NEGATIVE CIRCLED NUMBER TWENTY] +"\u24F4" => "20" + +# â’› [NUMBER TWENTY FULL STOP] +"\u249B" => "20." + +# â’‡ [PARENTHESIZED NUMBER TWENTY] +"\u2487" => "(20)" + +# « [LEFT-POINTING DOUBLE ANGLE QUOTATION MARK] +"\u00AB" => "\"" + +# » [RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK] +"\u00BB" => "\"" + +# “ [LEFT DOUBLE QUOTATION MARK] +"\u201C" => "\"" + +# †[RIGHT DOUBLE QUOTATION MARK] +"\u201D" => "\"" + +# „ [DOUBLE LOW-9 QUOTATION MARK] +"\u201E" => "\"" + +# ″ [DOUBLE PRIME] +"\u2033" => "\"" + +# ‶ [REVERSED DOUBLE PRIME] +"\u2036" => "\"" + +# â [HEAVY DOUBLE TURNED COMMA QUOTATION MARK ORNAMENT] +"\u275D" => "\"" + +# âž [HEAVY DOUBLE COMMA QUOTATION MARK ORNAMENT] +"\u275E" => "\"" + +# â® [HEAVY LEFT-POINTING ANGLE QUOTATION MARK ORNAMENT] +"\u276E" => "\"" + +# ⯠[HEAVY RIGHT-POINTING ANGLE QUOTATION MARK ORNAMENT] +"\u276F" => "\"" + +# ＂ [FULLWIDTH QUOTATION MARK] +"\uFF02" => "\"" + +# ‘ [LEFT SINGLE QUOTATION MARK] +"\u2018" => "\'" + +# ’ [RIGHT SINGLE QUOTATION MARK] +"\u2019" => "\'" + +# ‚ [SINGLE LOW-9 QUOTATION MARK] +"\u201A" => "\'" + +# ‛ [SINGLE HIGH-REVERSED-9 QUOTATION MARK] +"\u201B" => "\'" + +# ′ [PRIME] +"\u2032" => "\'" + +# ‵ [REVERSED PRIME] +"\u2035" => "\'" + +# ‹ [SINGLE LEFT-POINTING ANGLE QUOTATION MARK] +"\u2039" => "\'" + +# › [SINGLE RIGHT-POINTING ANGLE QUOTATION MARK] +"\u203A" => "\'" + +# â› [HEAVY SINGLE TURNED COMMA QUOTATION MARK ORNAMENT] +"\u275B" => "\'" + +# ✠[HEAVY SINGLE COMMA QUOTATION MARK ORNAMENT] +"\u275C" => "\'" + +# ＇ [FULLWIDTH APOSTROPHE] +"\uFF07" => "\'" + +# †[HYPHEN] +"\u2010" => "-" + +# ‑ [NON-BREAKING HYPHEN] +"\u2011" => "-" + +# ‒ [FIGURE DASH] +"\u2012" => "-" + +# – [EN DASH] +"\u2013" => "-" + +# — [EM DASH] +"\u2014" => "-" + +# â» [SUPERSCRIPT MINUS] +"\u207B" => "-" + +# â‚‹ [SUBSCRIPT MINUS] +"\u208B" => "-" + +# ï¼ [FULLWIDTH HYPHEN-MINUS] +"\uFF0D" => "-" + +# â… [LEFT SQUARE BRACKET WITH QUILL] +"\u2045" => "[" + +# â² [LIGHT LEFT TORTOISE SHELL BRACKET ORNAMENT] +"\u2772" => "[" + +# ï¼» [FULLWIDTH LEFT SQUARE BRACKET] +"\uFF3B" => "[" + +# ↠[RIGHT SQUARE BRACKET WITH QUILL] +"\u2046" => "]" + +# â³ [LIGHT RIGHT TORTOISE SHELL BRACKET ORNAMENT] +"\u2773" => "]" + +# ï¼½ [FULLWIDTH RIGHT SQUARE BRACKET] +"\uFF3D" => "]" + +# â½ [SUPERSCRIPT LEFT PARENTHESIS] +"\u207D" => "(" + +# â‚ [SUBSCRIPT LEFT PARENTHESIS] +"\u208D" => "(" + +# ⨠[MEDIUM LEFT PARENTHESIS ORNAMENT] +"\u2768" => "(" + +# ⪠[MEDIUM FLATTENED LEFT PARENTHESIS ORNAMENT] +"\u276A" => "(" + +# （ [FULLWIDTH LEFT PARENTHESIS] +"\uFF08" => "(" + +# ⸨ [LEFT DOUBLE PARENTHESIS] +"\u2E28" => "((" + +# â¾ [SUPERSCRIPT RIGHT PARENTHESIS] +"\u207E" => ")" + +# â‚Ž [SUBSCRIPT RIGHT PARENTHESIS] +"\u208E" => ")" + +# â© [MEDIUM RIGHT PARENTHESIS ORNAMENT] +"\u2769" => ")" + +# â« [MEDIUM FLATTENED RIGHT PARENTHESIS ORNAMENT] +"\u276B" => ")" + +# ） [FULLWIDTH RIGHT PARENTHESIS] +"\uFF09" => ")" + +# ⸩ [RIGHT DOUBLE PARENTHESIS] +"\u2E29" => "))" + +# ⬠[MEDIUM LEFT-POINTING ANGLE BRACKET ORNAMENT] +"\u276C" => "<" + +# â° [HEAVY LEFT-POINTING ANGLE BRACKET ORNAMENT] +"\u2770" => "<" + +# ＜ [FULLWIDTH LESS-THAN SIGN] +"\uFF1C" => "<" + +# â­ [MEDIUM RIGHT-POINTING ANGLE BRACKET ORNAMENT] +"\u276D" => ">" + +# â± [HEAVY RIGHT-POINTING ANGLE BRACKET ORNAMENT] +"\u2771" => ">" + +# ＞ [FULLWIDTH GREATER-THAN SIGN] +"\uFF1E" => ">" + +# â´ [MEDIUM LEFT CURLY BRACKET ORNAMENT] +"\u2774" => "{" + +# ï½› [FULLWIDTH LEFT CURLY BRACKET] +"\uFF5B" => "{" + +# âµ [MEDIUM RIGHT CURLY BRACKET ORNAMENT] +"\u2775" => "}" + +# ï½ [FULLWIDTH RIGHT CURLY BRACKET] +"\uFF5D" => "}" + +# ⺠[SUPERSCRIPT PLUS SIGN] +"\u207A" => "+" + +# â‚Š [SUBSCRIPT PLUS SIGN] +"\u208A" => "+" + +# ＋ [FULLWIDTH PLUS SIGN] +"\uFF0B" => "+" + +# â¼ [SUPERSCRIPT EQUALS SIGN] +"\u207C" => "=" + +# â‚Œ [SUBSCRIPT EQUALS SIGN] +"\u208C" => "=" + +# ï¼ [FULLWIDTH EQUALS SIGN] +"\uFF1D" => "=" + +# ï¼ [FULLWIDTH EXCLAMATION MARK] +"\uFF01" => "!" + +# ‼ [DOUBLE EXCLAMATION MARK] +"\u203C" => "!!" + +# ≠[EXCLAMATION QUESTION MARK] +"\u2049" => "!?" + +# ＃ [FULLWIDTH NUMBER SIGN] +"\uFF03" => "#" + +# ＄ [FULLWIDTH DOLLAR SIGN] +"\uFF04" => "$" + +# â’ [COMMERCIAL MINUS SIGN] +"\u2052" => "%" + +# ï¼… [FULLWIDTH PERCENT SIGN] +"\uFF05" => "%" + +# ＆ [FULLWIDTH AMPERSAND] +"\uFF06" => "&" + +# ⎠[LOW ASTERISK] +"\u204E" => "*" + +# ＊ [FULLWIDTH ASTERISK] +"\uFF0A" => "*" + +# ， [FULLWIDTH COMMA] +"\uFF0C" => "," + +# ． [FULLWIDTH FULL STOP] +"\uFF0E" => "." + +# â„ [FRACTION SLASH] +"\u2044" => "/" + +# ï¼ [FULLWIDTH SOLIDUS] +"\uFF0F" => "/" + +# ： [FULLWIDTH COLON] +"\uFF1A" => ":" + +# â [REVERSED SEMICOLON] +"\u204F" => ";" + +# ï¼› [FULLWIDTH SEMICOLON] +"\uFF1B" => ";" + +# ？ [FULLWIDTH QUESTION MARK] +"\uFF1F" => "?" + +# ⇠[DOUBLE QUESTION MARK] +"\u2047" => "??" + +# ∠[QUESTION EXCLAMATION MARK] +"\u2048" => "?!" + +# ï¼  [FULLWIDTH COMMERCIAL AT] +"\uFF20" => "@" + +# ï¼¼ [FULLWIDTH REVERSE SOLIDUS] +"\uFF3C" => "\\" + +# ‸ [CARET] +"\u2038" => "^" + +# ï¼¾ [FULLWIDTH CIRCUMFLEX ACCENT] +"\uFF3E" => "^" + +# ＿ [FULLWIDTH LOW LINE] +"\uFF3F" => "_" + +# â“ [SWUNG DASH] +"\u2053" => "~" + +# ～ [FULLWIDTH TILDE] +"\uFF5E" => "~" + +################################################################ +# Below is the Perl script used to generate the above mappings # +# from ASCIIFoldingFilter.java: # +################################################################ +# +# #!/usr/bin/perl +# +# use warnings; +# use strict; +# +# my @source_chars = (); +# my @source_char_descriptions = (); +# my $target = ''; +# +# while (<>) { +# if (/case\s+'(\\u[A-F0-9]+)':\s*\/\/\s*(.*)/i) { +# push @source_chars, $1; +# push @source_char_descriptions, $2; +# next; +# } +# if (/output\[[^\]]+\]\s*=\s*'(\\'|\\\\|.)'/) { +# $target .= $1; +# next; +# } +# if (/break;/) { +# $target = "\\\"" if ($target eq '"'); +# for my $source_char_num (0..$#source_chars) { +# print "# $source_char_descriptions[$source_char_num]\n"; +# print "\"$source_chars[$source_char_num]\" => \"$target\"\n\n"; +# } +# @source_chars = (); +# @source_char_descriptions = (); +# $target = ''; +# } +# } diff --git a/solr/example/solr/collection1/conf/mapping-ISOLatin1Accent.txt b/solr/example/solr/collection1/conf/mapping-ISOLatin1Accent.txt new file mode 100644 index 0000000..ede7742 --- /dev/null +++ b/solr/example/solr/collection1/conf/mapping-ISOLatin1Accent.txt @@ -0,0 +1,246 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Syntax: +# "source" => "target" +# "source".length() > 0 (source cannot be empty.) +# "target".length() >= 0 (target can be empty.) + +# example: +# "À" => "A" +# "\u00C0" => "A" +# "\u00C0" => "\u0041" +# "ß" => "ss" +# "\t" => " " +# "\n" => "" + +# À => A +"\u00C0" => "A" + +# à => A +"\u00C1" => "A" + +#  => A +"\u00C2" => "A" + +# à => A +"\u00C3" => "A" + +# Ä => A +"\u00C4" => "A" + +# Ã… => A +"\u00C5" => "A" + +# Æ => AE +"\u00C6" => "AE" + +# Ç => C +"\u00C7" => "C" + +# È => E +"\u00C8" => "E" + +# É => E +"\u00C9" => "E" + +# Ê => E +"\u00CA" => "E" + +# Ë => E +"\u00CB" => "E" + +# ÃŒ => I +"\u00CC" => "I" + +# à => I +"\u00CD" => "I" + +# ÃŽ => I +"\u00CE" => "I" + +# à => I +"\u00CF" => "I" + +# IJ => IJ +"\u0132" => "IJ" + +# à => D +"\u00D0" => "D" + +# Ñ => N +"\u00D1" => "N" + +# Ã’ => O +"\u00D2" => "O" + +# Ó => O +"\u00D3" => "O" + +# Ô => O +"\u00D4" => "O" + +# Õ => O +"\u00D5" => "O" + +# Ö => O +"\u00D6" => "O" + +# Ø => O +"\u00D8" => "O" + +# Å’ => OE +"\u0152" => "OE" + +# Þ +"\u00DE" => "TH" + +# Ù => U +"\u00D9" => "U" + +# Ú => U +"\u00DA" => "U" + +# Û => U +"\u00DB" => "U" + +# Ãœ => U +"\u00DC" => "U" + +# à => Y +"\u00DD" => "Y" + +# Ÿ => Y +"\u0178" => "Y" + +# à => a +"\u00E0" => "a" + +# á => a +"\u00E1" => "a" + +# â => a +"\u00E2" => "a" + +# ã => a +"\u00E3" => "a" + +# ä => a +"\u00E4" => "a" + +# Ã¥ => a +"\u00E5" => "a" + +# æ => ae +"\u00E6" => "ae" + +# ç => c +"\u00E7" => "c" + +# è => e +"\u00E8" => "e" + +# é => e +"\u00E9" => "e" + +# ê => e +"\u00EA" => "e" + +# ë => e +"\u00EB" => "e" + +# ì => i +"\u00EC" => "i" + +# í => i +"\u00ED" => "i" + +# î => i +"\u00EE" => "i" + +# ï => i +"\u00EF" => "i" + +# ij => ij +"\u0133" => "ij" + +# ð => d +"\u00F0" => "d" + +# ñ => n +"\u00F1" => "n" + +# ò => o +"\u00F2" => "o" + +# ó => o +"\u00F3" => "o" + +# ô => o +"\u00F4" => "o" + +# õ => o +"\u00F5" => "o" + +# ö => o +"\u00F6" => "o" + +# ø => o +"\u00F8" => "o" + +# Å“ => oe +"\u0153" => "oe" + +# ß => ss +"\u00DF" => "ss" + +# þ => th +"\u00FE" => "th" + +# ù => u +"\u00F9" => "u" + +# ú => u +"\u00FA" => "u" + +# û => u +"\u00FB" => "u" + +# ü => u +"\u00FC" => "u" + +# ý => y +"\u00FD" => "y" + +# ÿ => y +"\u00FF" => "y" + +# ff => ff +"\uFB00" => "ff" + +# ï¬ => fi +"\uFB01" => "fi" + +# fl => fl +"\uFB02" => "fl" + +# ffi => ffi +"\uFB03" => "ffi" + +# ffl => ffl +"\uFB04" => "ffl" + +# ſt => ft +"\uFB05" => "ft" + +# st => st +"\uFB06" => "st" diff --git a/solr/example/solr/collection1/conf/protwords.txt b/solr/example/solr/collection1/conf/protwords.txt new file mode 100644 index 0000000..1dfc0ab --- /dev/null +++ b/solr/example/solr/collection1/conf/protwords.txt @@ -0,0 +1,21 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# Use a protected word file to protect against the stemmer reducing two +# unrelated words to the same base word. + +# Some non-words that normally won't be encountered, +# just to test that they won't be stemmed. +dontstems +zwhacky + diff --git a/solr/example/solr/collection1/conf/schema.xml b/solr/example/solr/collection1/conf/schema.xml new file mode 100755 index 0000000..644d769 --- /dev/null +++ b/solr/example/solr/collection1/conf/schema.xml @@ -0,0 +1,1055 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/solr/example/solr/collection1/conf/scripts.conf b/solr/example/solr/collection1/conf/scripts.conf new file mode 100644 index 0000000..f58b262 --- /dev/null +++ b/solr/example/solr/collection1/conf/scripts.conf @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +user= +solr_hostname=localhost +solr_port=8983 +rsyncd_port=18983 +data_dir= +webapp_name=solr +master_host= +master_data_dir= +master_status_dir= diff --git a/solr/example/solr/collection1/conf/solrconfig.xml b/solr/example/solr/collection1/conf/solrconfig.xml new file mode 100755 index 0000000..b78c295 --- /dev/null +++ b/solr/example/solr/collection1/conf/solrconfig.xml @@ -0,0 +1,1714 @@ + + + + + + + + + LUCENE_50 + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 15000 + false + + + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + + + + + + 1024 + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + static firstSearcher warming in solrconfig.xml + + + + + + false + + + 2 + + + + + + + + + + + + + + + + + + + + + + + explicit + 10 + text + + + + + + + + + + + + + + explicit + json + true + text + + + + + + + + true + + + + + + + + explicit + + + velocity + browse + layout + Solritas + + + edismax + + text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 + + 100% + *:* + 10 + *,score + + + text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 + + text,features,name,sku,id,manu,cat + 3 + + + on + cat + manu_exact + ipod + GB + 1 + cat,inStock + after + price + 0 + 600 + 50 + popularity + 0 + 10 + 3 + manufacturedate_dt + NOW/YEAR-10YEARS + NOW + +1YEAR + before + after + + + on + text features name + 0 + name + + + on + false + 5 + 2 + 5 + true + true + 5 + 3 + + + + + spellcheck + + + + + + + + + + + + + + + + text + true + ignored_ + + + true + links + ignored_ + + + + + + + + + + + + + + + + + + + + + + solrpingquery + + + all + + + + + + + + + explicit + true + + + + + + + + + + + + + + + + textSpell + + + + + + default + name + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + wordbreak + solr.WordBreakSolrSpellChecker + name + true + true + 10 + + + + + + + + + + + + + + + + text + + default + wordbreak + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + + + + + text + true + + + tvComponent + + + + + + + + + default + + + org.carrot2.clustering.lingo.LingoClusteringAlgorithm + + + 20 + + + clustering/carrot2 + + + ENGLISH + + + stc + org.carrot2.clustering.stc.STCClusteringAlgorithm + + + + + + + true + default + true + + name + id + + features + + true + + + + false + + edismax + + text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 + + *:* + 10 + *,score + + + clustering + + + + + + + + + + true + + + terms + + + + + + + + string + elevate.xml + + + + + + explicit + text + + + elevator + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + + + + + 5 + + + + + + + + + + + + + + + + + + *:* + + + diff --git a/solr/example/solr/collection1/conf/spellings.txt b/solr/example/solr/collection1/conf/spellings.txt new file mode 100644 index 0000000..d7ede6f --- /dev/null +++ b/solr/example/solr/collection1/conf/spellings.txt @@ -0,0 +1,2 @@ +pizza +history \ No newline at end of file diff --git a/solr/example/solr/collection1/conf/stopwords.txt b/solr/example/solr/collection1/conf/stopwords.txt new file mode 100644 index 0000000..ae1e83e --- /dev/null +++ b/solr/example/solr/collection1/conf/stopwords.txt @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/solr/example/solr/collection1/conf/synonyms.txt b/solr/example/solr/collection1/conf/synonyms.txt new file mode 100644 index 0000000..7f72128 --- /dev/null +++ b/solr/example/solr/collection1/conf/synonyms.txt @@ -0,0 +1,29 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +aaafoo => aaabar +bbbfoo => bbbfoo bbbbar +cccfoo => cccbar cccbaz +fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +pixima => pixma + diff --git a/solr/example/solr/collection1/conf/velocity/VM_global_library.vm b/solr/example/solr/collection1/conf/velocity/VM_global_library.vm new file mode 100644 index 0000000..eae1bfe --- /dev/null +++ b/solr/example/solr/collection1/conf/velocity/VM_global_library.vm @@ -0,0 +1,163 @@ + +#macro(param $key)$request.params.get($key)#end + +#macro(url_root)/solr#end + +## TODO: s/url_for_solr/url_for_core/ and s/url_root/url_for_solr/ +#macro(url_for_solr)#{url_root}#if($request.core.name != "")/$request.core.name#end#end +#macro(url_for_home)#url_for_solr/browse#end + +#macro(q)&q=$!{esc.url($params.get('q'))}#end + +#macro(fqs $p)#foreach($fq in $p)#if($velocityCount>1)&#{end}fq=$esc.url($fq)#end#end + +#macro(debug)#if($request.params.get('debugQuery'))&debugQuery=true#end#end + +#macro(boostPrice)#if($request.params.get('bf') == 'price')&bf=price#end#end + +#macro(annotate)#if($request.params.get('annotateBrowse'))&annotateBrowse=true#end#end + +#macro(annTitle $msg)#if($annotate == true)title="$msg"#end#end + +#macro(spatial)#if($request.params.get('sfield'))&sfield=store#end#if($request.params.get('pt'))&pt=$request.params.get('pt')#end#if($request.params.get('d'))&d=$request.params.get('d')#end#end + +#macro(qOpts)#set($queryOpts = $request.params.get("queryOpts"))#if($queryOpts && $queryOpts != "")&queryOpts=$queryOpts#end#end + +#macro(group)#if($request.params.getBool("group") == true)&group=true#end#if($request.params.get("group.field"))#foreach($grp in $request.params.getParams('group.field'))&group.field=$grp#end#end#end + +#macro(lensNoQ)?#if($request.params.getParams('fq') and $list.size($request.params.getParams('fq')) > 0)&#fqs($request.params.getParams('fq'))#end#debug#boostPrice#annotate#spatial#qOpts#group#end +#macro(lens)#lensNoQ#q#end + + +#macro(url_for_lens)#{url_for_home}#lens#end + +#macro(url_for_start $start)#url_for_home#lens&start=$start#end + +#macro(url_for_filters $p)#url_for_home?#q#boostPrice#spatial#qOpts#if($list.size($p) > 0)&#fqs($p)#end#debug#end + +#macro(url_for_nested_facet_query $field)#url_for_home#lens&fq=$esc.url($field)#end + +## TODO: convert to use {!raw f=$field}$value (with escaping of course) +#macro(url_for_facet_filter $field $value)#url_for_home#lens&fq=$esc.url($field):%22$esc.url($value)%22#end + +#macro(url_for_facet_date_filter $field $value)#url_for_home#lens&fq=$esc.url($field):$esc.url($value)#end + +#macro(url_for_facet_range_filter $field $value)#url_for_home#lens&fq=$esc.url($field):$esc.url($value)#end + + +#macro(link_to_previous_page $text) + #if($page.current_page_number > 1) + #set($prev_start = $page.start - $page.results_per_page) + $text + #end +#end + +#macro(link_to_next_page $text) + #if($page.current_page_number < $page.page_count) + #set($next_start = $page.start + $page.results_per_page) + $text + #end +#end + +#macro(link_to_page $page_number $text) + #if($page_number == $page.current_page_number) + $text + #else + #if($page_number <= $page.page_count) + #set($page_start = $page_number * $page.results_per_page - $page.results_per_page) + $text + #end + #end +#end + +#macro(display_facet_query $field, $display, $fieldName) + #if($field.size() > 0) + $display +
        + #foreach ($facet in $field) + #if ($facet.value > 0) + #set($facetURL = "#url_for_nested_facet_query($facet.key)") + #if ($facetURL != '') +
      • $facet.key ($facet.value)
        • + #end + #end + #end +
      + #end +#end + + +#macro(display_facet_range $field, $display, $fieldName, $start, $end, $gap, $before, $after) + $display +
        + #if($before && $before != "") + #set($value = "[* TO " + "#format_value($start)" + "}") + #set($facetURL = "#url_for_facet_range_filter($fieldName, $value)") +
      • Less than #format_value($start) ($before)
        • + #end + #foreach ($facet in $field) + #set($rangeEnd = "#range_get_to_value($facet.key, $gap)") + #set($value = "[" + $facet.key + " TO " + $rangeEnd + "}") + #set($facetURL = "#url_for_facet_range_filter($fieldName, $value)") + #if ($facetURL != '') +
      • $facet.key - #format_value($rangeEnd) ($facet.value)
        • + #end + #end + #if($end && $end != "" && $after > 0) + #set($value = "[" + "#format_value($end)" + " TO *}") + #set($facetURL = "#url_for_facet_range_filter($fieldName, $value)") +
      • More than #format_value($end) ($after)
        • + #end +
      +#end + +## $pivots is a list of facet_pivot +#macro(display_facet_pivot $pivots, $display) + #if($pivots.size() > 0) + $display + + #end +#end + +#macro(field $f) + #if($response.response.highlighting.get($docId).get($f).get(0)) + $!response.response.highlighting.get($docId).get($f).get(0) + #else + #foreach($v in $doc.getFieldValues($f)) + $v + #end + #end +#end + +#macro(utc_date $theDate) +$date.format("yyyy-MM-dd'T'HH:mm:ss'Z'",$theDate,$date.getLocale(),$date.getTimeZone().getTimeZone("UTC"))## +#end + +#macro(format_value $val) +#if(${val.class.name} == "java.util.Date") +#utc_date($val)## +#else +$val## +#end +#end + +#macro(range_get_to_value $inval, $gapval) +#if(${gapval.class.name} == "java.lang.String") +$inval$gapval## +#elseif(${gapval.class.name} == "java.lang.Float" || ${inval.class.name} == "java.lang.Float") +$math.toDouble($math.add($inval,$gapval))## +#else +$math.add($inval,$gapval)## +#end +#end \ No newline at end of file diff --git a/solr/example/solr/collection1/conf/velocity/browse.vm b/solr/example/solr/collection1/conf/velocity/browse.vm new file mode 100644 index 0000000..d7c3c43 --- /dev/null +++ b/solr/example/solr/collection1/conf/velocity/browse.vm @@ -0,0 +1,45 @@ +#set($searcher=$request.searcher) +#set($params=$request.params) +#set($clusters = $response.response.clusters) +#set($mltResults = $response.response.get("moreLikeThis")) +#set($annotate = $params.get("annotateBrowse")) +#parse('query.vm') +#if($response.response.spellcheck.suggestions and $response.response.spellcheck.suggestions.size() > 0) + Did you mean $response.response.spellcheck.suggestions.collation? +#end + +
      + #parse("facets.vm") +
      + + + +
      + #if($response.response.get('grouped')) + #foreach($grouping in $response.response.get('grouped')) + #parse("hitGrouped.vm") + #end + #else + #foreach($doc in $response.results) + #parse("hit.vm") + #end + #end +
      + + diff --git a/solr/example/solr/collection1/conf/velocity/cluster.vm b/solr/example/solr/collection1/conf/velocity/cluster.vm new file mode 100644 index 0000000..ab2f0e4 --- /dev/null +++ b/solr/example/solr/collection1/conf/velocity/cluster.vm @@ -0,0 +1,9 @@ +

      Clusters

      +
      + Run Solr with java -Dsolr.clustering.enabled=true -jar start.jar to see results +
      + diff --git a/solr/example/solr/collection1/conf/velocity/clusterResults.vm b/solr/example/solr/collection1/conf/velocity/clusterResults.vm new file mode 100644 index 0000000..db1774d --- /dev/null +++ b/solr/example/solr/collection1/conf/velocity/clusterResults.vm @@ -0,0 +1,12 @@ +#foreach ($clusters in $response.response.clusters) + #set($labels = $clusters.get('labels')) + #set($docs = $clusters.get('docs')) +

      #foreach ($label in $labels)$label#if( $foreach.hasNext ),#end#end

      +
        + #foreach ($cluDoc in $docs) +
      1. $cluDoc
        2. + #end +
      + + +#end \ No newline at end of file diff --git a/solr/example/solr/collection1/conf/velocity/doc.vm b/solr/example/solr/collection1/conf/velocity/doc.vm new file mode 100644 index 0000000..7da97cb --- /dev/null +++ b/solr/example/solr/collection1/conf/velocity/doc.vm @@ -0,0 +1,42 @@ +
      #field('name')#if($params.getBool('mlt', false) == false)More Like This#end
      +##do we have a physical store for this product +#set($store = $doc.getFieldValue('store')) +#if($store)
      #end +
      Price: #field('price_c')
      +
      Features: #field('features')
      +
      In Stock: #field('inStock')
      +
      + #set($mlt = $mltResults.get($docId)) + #set($mltOn = $params.getBool('mlt')) + #if($mltOn == true)
      Similar Items
      #end + #if ($mltOn && $mlt && $mlt.size() > 0) +
        + #foreach($mltHit in $mlt) + #set($mltId = $mltHit.getFieldValue('id')) +
      • Name: $mltHit.getFieldValue('name')
        +
        Price: $!number.currency($mltHit.getFieldValue('price')) In Stock: $mltHit.getFieldValue('inStock')
        + +
        • + #end +
      + #elseif($mltOn && $mlt.size() == 0) +
      No Similar Items Found
      + #end +
      +#if($params.getBool("debugQuery",false)) + toggle explain + 
      $response.getExplainMap().get($doc.getFirstValue('id'))
      + toggle all fields + + #foreach($fieldname in $doc.fieldNames) +
      + $fieldname : + + #foreach($value in $doc.getFieldValues($fieldname)) + $value + #end + + #end +
      +       +#end \ No newline at end of file diff --git a/solr/example/solr/collection1/conf/velocity/facet_fields.vm b/solr/example/solr/collection1/conf/velocity/facet_fields.vm new file mode 100644 index 0000000..91c50be --- /dev/null +++ b/solr/example/solr/collection1/conf/velocity/facet_fields.vm @@ -0,0 +1,12 @@ +#if($response.facetFields) +

      Field Facets

      + #foreach($field in $response.facetFields) + $field.name + +
        + #foreach($facet in $field.values) +
      • $facet.name ($facet.count)
        • + #end +
      + #end + #end \ No newline at end of file diff --git a/solr/example/solr/collection1/conf/velocity/facet_pivot.vm b/solr/example/solr/collection1/conf/velocity/facet_pivot.vm new file mode 100644 index 0000000..5e1e151 --- /dev/null +++ b/solr/example/solr/collection1/conf/velocity/facet_pivot.vm @@ -0,0 +1,3 @@ +

      Pivot Facets

      +#set($pivot = $response.response.facet_counts.facet_pivot) +#display_facet_pivot($pivot, "") diff --git a/solr/example/solr/collection1/conf/velocity/facet_queries.vm b/solr/example/solr/collection1/conf/velocity/facet_queries.vm new file mode 100644 index 0000000..2ac65d6 --- /dev/null +++ b/solr/example/solr/collection1/conf/velocity/facet_queries.vm @@ -0,0 +1,3 @@ +#set($field = $response.response.facet_counts.facet_queries) +

      Query Facets

      +#display_facet_query($field, "", "") \ No newline at end of file diff --git a/solr/example/solr/collection1/conf/velocity/facet_ranges.vm b/solr/example/solr/collection1/conf/velocity/facet_ranges.vm new file mode 100644 index 0000000..0118fe0 --- /dev/null +++ b/solr/example/solr/collection1/conf/velocity/facet_ranges.vm @@ -0,0 +1,12 @@ +

      Range Facets

      +#foreach ($field in $response.response.facet_counts.facet_ranges) + #set($name = $field.key) + #set($display = $name) + #set($f = $field.value.counts) + #set($start = $field.value.start) + #set($end = $field.value.end) + #set($gap = $field.value.gap) + #set($before = $field.value.before) + #set($after = $field.value.after) + #display_facet_range($f, $display, $name, $start, $end, $gap, $before, $after) +#end \ No newline at end of file diff --git a/solr/example/solr/collection1/conf/velocity/facets.vm b/solr/example/solr/collection1/conf/velocity/facets.vm new file mode 100644 index 0000000..fd5f5d9 --- /dev/null +++ b/solr/example/solr/collection1/conf/velocity/facets.vm @@ -0,0 +1,5 @@ +#parse('facet_fields.vm') +#parse('facet_queries.vm') +#parse('facet_ranges.vm') +#parse('facet_pivot.vm') +#parse('cluster.vm') \ No newline at end of file diff --git a/solr/example/solr/collection1/conf/velocity/footer.vm b/solr/example/solr/collection1/conf/velocity/footer.vm new file mode 100644 index 0000000..b55e8a5 --- /dev/null +++ b/solr/example/solr/collection1/conf/velocity/footer.vm @@ -0,0 +1,17 @@ +
      +
      + Options: + #if($request.params.get('debugQuery')) + disable debug + #else + enable debug + #end + #if($annotate) + disable annotation + #else + enable annotation + #end + XML
      + +
      Documentation: Solr Home Page, Solr Wiki
      +
      Disclaimer: The locations displayed in this demonstration are purely fictional. It is more than likely that no store with the items listed actually exists at that location!
      \ No newline at end of file diff --git a/solr/example/solr/collection1/conf/velocity/head.vm b/solr/example/solr/collection1/conf/velocity/head.vm new file mode 100644 index 0000000..265b7d3 --- /dev/null +++ b/solr/example/solr/collection1/conf/velocity/head.vm @@ -0,0 +1,28 @@ + + ## An example of using an arbitrary request parameter + #param('title') + + + + + + + + + \ No newline at end of file diff --git a/solr/example/solr/collection1/conf/velocity/header.vm b/solr/example/solr/collection1/conf/velocity/header.vm new file mode 100644 index 0000000..46f0bab --- /dev/null +++ b/solr/example/solr/collection1/conf/velocity/header.vm @@ -0,0 +1,3 @@ + \ No newline at end of file diff --git a/solr/example/solr/collection1/conf/velocity/hit.vm b/solr/example/solr/collection1/conf/velocity/hit.vm new file mode 100644 index 0000000..5cd4fe8 --- /dev/null +++ b/solr/example/solr/collection1/conf/velocity/hit.vm @@ -0,0 +1,5 @@ +#set($docId = $doc.getFieldValue('id')) + +
      + #parse("doc.vm") +
      diff --git a/solr/example/solr/collection1/conf/velocity/hitGrouped.vm b/solr/example/solr/collection1/conf/velocity/hitGrouped.vm new file mode 100644 index 0000000..c3cccf4 --- /dev/null +++ b/solr/example/solr/collection1/conf/velocity/hitGrouped.vm @@ -0,0 +1,18 @@ +
      +
      $grouping.key
      +
      Total Matches in Group: $grouping.value.matches
      +
      #foreach ($group in $grouping.value.groups) +
      $group.groupValue ($group.doclist.numFound)
      +
      + #foreach ($doc in $group.doclist) + #set($docId = $doc.getFieldValue('id')) + #parse("doc.vm") + #end +
      + #end
      +
      + #if($params.getBool("debugQuery",false)) + toggle explain + 
      $response.getExplainMap().get($doc.getFirstValue('id'))
      + #end + diff --git a/solr/example/solr/collection1/conf/velocity/jquery.autocomplete.css b/solr/example/solr/collection1/conf/velocity/jquery.autocomplete.css new file mode 100644 index 0000000..91b6228 --- /dev/null +++ b/solr/example/solr/collection1/conf/velocity/jquery.autocomplete.css @@ -0,0 +1,48 @@ +.ac_results { + padding: 0px; + border: 1px solid black; + background-color: white; + overflow: hidden; + z-index: 99999; +} + +.ac_results ul { + width: 100%; + list-style-position: outside; + list-style: none; + padding: 0; + margin: 0; +} + +.ac_results li { + margin: 0px; + padding: 2px 5px; + cursor: default; + display: block; + /* + if width will be 100% horizontal scrollbar will apear + when scroll mode will be used + */ + /*width: 100%;*/ + font: menu; + font-size: 12px; + /* + it is very important, if line-height not setted or setted + in relative units scroll will be broken in firefox + */ + line-height: 16px; + overflow: hidden; +} + +.ac_loading { + background: white url('indicator.gif') right center no-repeat; +} + +.ac_odd { + background-color: #eee; +} + +.ac_over { + background-color: #0A246A; + color: white; +} diff --git a/solr/example/solr/collection1/conf/velocity/jquery.autocomplete.js b/solr/example/solr/collection1/conf/velocity/jquery.autocomplete.js new file mode 100644 index 0000000..f1b2642 --- /dev/null +++ b/solr/example/solr/collection1/conf/velocity/jquery.autocomplete.js @@ -0,0 +1,762 @@ +/* + * Autocomplete - jQuery plugin 1.1pre + * + * Copyright (c) 2007 Dylan Verheul, Dan G. Switzer, Anjesh Tuladhar, Jörn Zaefferer + * + * Dual licensed under the MIT and GPL licenses: + * http://www.opensource.org/licenses/mit-license.php + * http://www.gnu.org/licenses/gpl.html + * + * Revision: $Id: jquery.autocomplete.js 5785 2008-07-12 10:37:33Z joern.zaefferer $ + * + */ + +;(function($) { + +$.fn.extend({ + autocomplete: function(urlOrData, options) { + var isUrl = typeof urlOrData == "string"; + options = $.extend({}, $.Autocompleter.defaults, { + url: isUrl ? urlOrData : null, + data: isUrl ? null : urlOrData, + delay: isUrl ? $.Autocompleter.defaults.delay : 10, + max: options && !options.scroll ? 10 : 150 + }, options); + + // if highlight is set to false, replace it with a do-nothing function + options.highlight = options.highlight || function(value) { return value; }; + + // if the formatMatch option is not specified, then use formatItem for backwards compatibility + options.formatMatch = options.formatMatch || options.formatItem; + + return this.each(function() { + new $.Autocompleter(this, options); + }); + }, + result: function(handler) { + return this.bind("result", handler); + }, + search: function(handler) { + return this.trigger("search", [handler]); + }, + flushCache: function() { + return this.trigger("flushCache"); + }, + setOptions: function(options){ + return this.trigger("setOptions", [options]); + }, + unautocomplete: function() { + return this.trigger("unautocomplete"); + } +}); + +$.Autocompleter = function(input, options) { + + var KEY = { + UP: 38, + DOWN: 40, + DEL: 46, + TAB: 9, + RETURN: 13, + ESC: 27, + COMMA: 188, + PAGEUP: 33, + PAGEDOWN: 34, + BACKSPACE: 8 + }; + + // Create $ object for input element + var $input = $(input).attr("autocomplete", "off").addClass(options.inputClass); + + var timeout; + var previousValue = ""; + var cache = $.Autocompleter.Cache(options); + var hasFocus = 0; + var lastKeyPressCode; + var config = { + mouseDownOnSelect: false + }; + var select = $.Autocompleter.Select(options, input, selectCurrent, config); + + var blockSubmit; + + // prevent form submit in opera when selecting with return key + $.browser.opera && $(input.form).bind("submit.autocomplete", function() { + if (blockSubmit) { + blockSubmit = false; + return false; + } + }); + + // only opera doesn't trigger keydown multiple times while pressed, others don't work with keypress at all + $input.bind(($.browser.opera ? "keypress" : "keydown") + ".autocomplete", function(event) { + // track last key pressed + lastKeyPressCode = event.keyCode; + switch(event.keyCode) { + + case KEY.UP: + event.preventDefault(); + if ( select.visible() ) { + select.prev(); + } else { + onChange(0, true); + } + break; + + case KEY.DOWN: + event.preventDefault(); + if ( select.visible() ) { + select.next(); + } else { + onChange(0, true); + } + break; + + case KEY.PAGEUP: + event.preventDefault(); + if ( select.visible() ) { + select.pageUp(); + } else { + onChange(0, true); + } + break; + + case KEY.PAGEDOWN: + event.preventDefault(); + if ( select.visible() ) { + select.pageDown(); + } else { + onChange(0, true); + } + break; + + // matches also semicolon + case options.multiple && $.trim(options.multipleSeparator) == "," && KEY.COMMA: + case KEY.TAB: + case KEY.RETURN: + if( selectCurrent() ) { + // stop default to prevent a form submit, Opera needs special handling + event.preventDefault(); + blockSubmit = true; + return false; + } + break; + + case KEY.ESC: + select.hide(); + break; + + default: + clearTimeout(timeout); + timeout = setTimeout(onChange, options.delay); + break; + } + }).focus(function(){ + // track whether the field has focus, we shouldn't process any + // results if the field no longer has focus + hasFocus++; + }).blur(function() { + hasFocus = 0; + if (!config.mouseDownOnSelect) { + hideResults(); + } + }).click(function() { + // show select when clicking in a focused field + if ( hasFocus++ > 1 && !select.visible() ) { + onChange(0, true); + } + }).bind("search", function() { + // TODO why not just specifying both arguments? + var fn = (arguments.length > 1) ? arguments[1] : null; + function findValueCallback(q, data) { + var result; + if( data && data.length ) { + for (var i=0; i < data.length; i++) { + if( data[i].result.toLowerCase() == q.toLowerCase() ) { + result = data[i]; + break; + } + } + } + if( typeof fn == "function" ) fn(result); + else $input.trigger("result", result && [result.data, result.value]); + } + $.each(trimWords($input.val()), function(i, value) { + request(value, findValueCallback, findValueCallback); + }); + }).bind("flushCache", function() { + cache.flush(); + }).bind("setOptions", function() { + $.extend(options, arguments[1]); + // if we've updated the data, repopulate + if ( "data" in arguments[1] ) + cache.populate(); + }).bind("unautocomplete", function() { + select.unbind(); + $input.unbind(); + $(input.form).unbind(".autocomplete"); + }); + + + function selectCurrent() { + var selected = select.selected(); + if( !selected ) + return false; + + var v = selected.result; + previousValue = v; + + if ( options.multiple ) { + var words = trimWords($input.val()); + if ( words.length > 1 ) { + v = words.slice(0, words.length - 1).join( options.multipleSeparator ) + options.multipleSeparator + v; + } + v += options.multipleSeparator; + } + + $input.val(v); + hideResultsNow(); + $input.trigger("result", [selected.data, selected.value]); + return true; + } + + function onChange(crap, skipPrevCheck) { + if( lastKeyPressCode == KEY.DEL ) { + select.hide(); + return; + } + + var currentValue = $input.val(); + + if ( !skipPrevCheck && currentValue == previousValue ) + return; + + previousValue = currentValue; + + currentValue = lastWord(currentValue); + if ( currentValue.length >= options.minChars) { + $input.addClass(options.loadingClass); + if (!options.matchCase) + currentValue = currentValue.toLowerCase(); + request(currentValue, receiveData, hideResultsNow); + } else { + stopLoading(); + select.hide(); + } + }; + + function trimWords(value) { + if ( !value ) { + return [""]; + } + var words = value.split( options.multipleSeparator ); + var result = []; + $.each(words, function(i, value) { + if ( $.trim(value) ) + result[i] = $.trim(value); + }); + return result; + } + + function lastWord(value) { + if ( !options.multiple ) + return value; + var words = trimWords(value); + return words[words.length - 1]; + } + + // fills in the input box w/the first match (assumed to be the best match) + // q: the term entered + // sValue: the first matching result + function autoFill(q, sValue){ + // autofill in the complete box w/the first match as long as the user hasn't entered in more data + // if the last user key pressed was backspace, don't autofill + if( options.autoFill && (lastWord($input.val()).toLowerCase() == q.toLowerCase()) && lastKeyPressCode != KEY.BACKSPACE ) { + // fill in the value (keep the case the user has typed) + $input.val($input.val() + sValue.substring(lastWord(previousValue).length)); + // select the portion of the value not typed by the user (so the next character will erase) + $.Autocompleter.Selection(input, previousValue.length, previousValue.length + sValue.length); + } + }; + + function hideResults() { + clearTimeout(timeout); + timeout = setTimeout(hideResultsNow, 200); + }; + + function hideResultsNow() { + var wasVisible = select.visible(); + select.hide(); + clearTimeout(timeout); + stopLoading(); + if (options.mustMatch) { + // call search and run callback + $input.search( + function (result){ + // if no value found, clear the input box + if( !result ) { + if (options.multiple) { + var words = trimWords($input.val()).slice(0, -1); + $input.val( words.join(options.multipleSeparator) + (words.length ? options.multipleSeparator : "") ); + } + else + $input.val( "" ); + } + } + ); + } + if (wasVisible) + // position cursor at end of input field + $.Autocompleter.Selection(input, input.value.length, input.value.length); + }; + + function receiveData(q, data) { + if ( data && data.length && hasFocus ) { + stopLoading(); + select.display(data, q); + autoFill(q, data[0].value); + select.show(); + } else { + hideResultsNow(); + } + }; + + function request(term, success, failure) { + if (!options.matchCase) + term = term.toLowerCase(); + var data = cache.load(term); + // recieve the cached data + if (data && data.length) { + success(term, data); + // if an AJAX url has been supplied, try loading the data now + } else if( (typeof options.url == "string") && (options.url.length > 0) ){ + + var extraParams = { + timestamp: +new Date() + }; + $.each(options.extraParams, function(key, param) { + extraParams[key] = typeof param == "function" ? param() : param; + }); + + $.ajax({ + // try to leverage ajaxQueue plugin to abort previous requests + mode: "abort", + // limit abortion to this input + port: "autocomplete" + input.name, + dataType: options.dataType, + url: options.url, + data: $.extend({ + q: lastWord(term), + limit: options.max + }, extraParams), + success: function(data) { + var parsed = options.parse && options.parse(data) || parse(data); + cache.add(term, parsed); + success(term, parsed); + } + }); + } else { + // if we have a failure, we need to empty the list -- this prevents the the [TAB] key from selecting the last successful match + select.emptyList(); + failure(term); + } + }; + + function parse(data) { + var parsed = []; + var rows = data.split("\n"); + for (var i=0; i < rows.length; i++) { + var row = $.trim(rows[i]); + if (row) { + row = row.split("|"); + parsed[parsed.length] = { + data: row, + value: row[0], + result: options.formatResult && options.formatResult(row, row[0]) || row[0] + }; + } + } + return parsed; + }; + + function stopLoading() { + $input.removeClass(options.loadingClass); + }; + +}; + +$.Autocompleter.defaults = { + inputClass: "ac_input", + resultsClass: "ac_results", + loadingClass: "ac_loading", + minChars: 1, + delay: 400, + matchCase: false, + matchSubset: true, + matchContains: false, + cacheLength: 10, + max: 100, + mustMatch: false, + extraParams: {}, + selectFirst: true, + formatItem: function(row) { return row[0]; }, + formatMatch: null, + autoFill: false, + width: 0, + multiple: false, + multipleSeparator: ", ", + highlight: function(value, term) { + return value.replace(new RegExp("(?![^&;]+;)(?!<[^<>]*)(" + term.replace(/([\^\$\(\)\[\]\{\}\*\.\+\?\|\\])/gi, "\\$1") + ")(?![^<>]*>)(?![^&;]+;)", "gi"), "$1"); + }, + scroll: true, + scrollHeight: 180 +}; + +$.Autocompleter.Cache = function(options) { + + var data = {}; + var length = 0; + + function matchSubset(s, sub) { + if (!options.matchCase) + s = s.toLowerCase(); + var i = s.indexOf(sub); + if (options.matchContains == "word"){ + i = s.toLowerCase().search("\\b" + sub.toLowerCase()); + } + if (i == -1) return false; + return i == 0 || options.matchContains; + }; + + function add(q, value) { + if (length > options.cacheLength){ + flush(); + } + if (!data[q]){ + length++; + } + data[q] = value; + } + + function populate(){ + if( !options.data ) return false; + // track the matches + var stMatchSets = {}, + nullData = 0; + + // no url was specified, we need to adjust the cache length to make sure it fits the local data store + if( !options.url ) options.cacheLength = 1; + + // track all options for minChars = 0 + stMatchSets[""] = []; + + // loop through the array and create a lookup structure + for ( var i = 0, ol = options.data.length; i < ol; i++ ) { + var rawValue = options.data[i]; + // if rawValue is a string, make an array otherwise just reference the array + rawValue = (typeof rawValue == "string") ? [rawValue] : rawValue; + + var value = options.formatMatch(rawValue, i+1, options.data.length); + if ( value === false ) + continue; + + var firstChar = value.charAt(0).toLowerCase(); + // if no lookup array for this character exists, look it up now + if( !stMatchSets[firstChar] ) + stMatchSets[firstChar] = []; + + // if the match is a string + var row = { + value: value, + data: rawValue, + result: options.formatResult && options.formatResult(rawValue) || value + }; + + // push the current match into the set list + stMatchSets[firstChar].push(row); + + // keep track of minChars zero items + if ( nullData++ < options.max ) { + stMatchSets[""].push(row); + } + }; + + // add the data items to the cache + $.each(stMatchSets, function(i, value) { + // increase the cache size + options.cacheLength++; + // add to the cache + add(i, value); + }); + } + + // populate any existing data + setTimeout(populate, 25); + + function flush(){ + data = {}; + length = 0; + } + + return { + flush: flush, + add: add, + populate: populate, + load: function(q) { + if (!options.cacheLength || !length) + return null; + /* + * if dealing w/local data and matchContains than we must make sure + * to loop through all the data collections looking for matches + */ + if( !options.url && options.matchContains ){ + // track all matches + var csub = []; + // loop through all the data grids for matches + for( var k in data ){ + // don't search through the stMatchSets[""] (minChars: 0) cache + // this prevents duplicates + if( k.length > 0 ){ + var c = data[k]; + $.each(c, function(i, x) { + // if we've got a match, add it to the array + if (matchSubset(x.value, q)) { + csub.push(x); + } + }); + } + } + return csub; + } else + // if the exact item exists, use it + if (data[q]){ + return data[q]; + } else + if (options.matchSubset) { + for (var i = q.length - 1; i >= options.minChars; i--) { + var c = data[q.substr(0, i)]; + if (c) { + var csub = []; + $.each(c, function(i, x) { + if (matchSubset(x.value, q)) { + csub[csub.length] = x; + } + }); + return csub; + } + } + } + return null; + } + }; +}; + +$.Autocompleter.Select = function (options, input, select, config) { + var CLASSES = { + ACTIVE: "ac_over" + }; + + var listItems, + active = -1, + data, + term = "", + needsInit = true, + element, + list; + + // Create results + function init() { + if (!needsInit) + return; + element = $("
      ") + .hide() + .addClass(options.resultsClass) + .css("position", "absolute") + .appendTo(document.body); + + list = $("
      diff --git a/solr/example/solr/conf/velocity/jquery.autocomplete.css b/solr/example/solr/conf/velocity/jquery.autocomplete.css deleted file mode 100644 index 91b6228..0000000 --- a/solr/example/solr/conf/velocity/jquery.autocomplete.css +++ /dev/null @@ -1,48 +0,0 @@ -.ac_results { - padding: 0px; - border: 1px solid black; - background-color: white; - overflow: hidden; - z-index: 99999; -} - -.ac_results ul { - width: 100%; - list-style-position: outside; - list-style: none; - padding: 0; - margin: 0; -} - -.ac_results li { - margin: 0px; - padding: 2px 5px; - cursor: default; - display: block; - /* - if width will be 100% horizontal scrollbar will apear - when scroll mode will be used - */ - /*width: 100%;*/ - font: menu; - font-size: 12px; - /* - it is very important, if line-height not setted or setted - in relative units scroll will be broken in firefox - */ - line-height: 16px; - overflow: hidden; -} - -.ac_loading { - background: white url('indicator.gif') right center no-repeat; -} - -.ac_odd { - background-color: #eee; -} - -.ac_over { - background-color: #0A246A; - color: white; -} diff --git a/solr/example/solr/conf/velocity/jquery.autocomplete.js b/solr/example/solr/conf/velocity/jquery.autocomplete.js deleted file mode 100644 index f1b2642..0000000 --- a/solr/example/solr/conf/velocity/jquery.autocomplete.js +++ /dev/null @@ -1,762 +0,0 @@ -/* - * Autocomplete - jQuery plugin 1.1pre - * - * Copyright (c) 2007 Dylan Verheul, Dan G. Switzer, Anjesh Tuladhar, Jörn Zaefferer - * - * Dual licensed under the MIT and GPL licenses: - * http://www.opensource.org/licenses/mit-license.php - * http://www.gnu.org/licenses/gpl.html - * - * Revision: $Id: jquery.autocomplete.js 5785 2008-07-12 10:37:33Z joern.zaefferer $ - * - */ - -;(function($) { - -$.fn.extend({ - autocomplete: function(urlOrData, options) { - var isUrl = typeof urlOrData == "string"; - options = $.extend({}, $.Autocompleter.defaults, { - url: isUrl ? urlOrData : null, - data: isUrl ? null : urlOrData, - delay: isUrl ? $.Autocompleter.defaults.delay : 10, - max: options && !options.scroll ? 10 : 150 - }, options); - - // if highlight is set to false, replace it with a do-nothing function - options.highlight = options.highlight || function(value) { return value; }; - - // if the formatMatch option is not specified, then use formatItem for backwards compatibility - options.formatMatch = options.formatMatch || options.formatItem; - - return this.each(function() { - new $.Autocompleter(this, options); - }); - }, - result: function(handler) { - return this.bind("result", handler); - }, - search: function(handler) { - return this.trigger("search", [handler]); - }, - flushCache: function() { - return this.trigger("flushCache"); - }, - setOptions: function(options){ - return this.trigger("setOptions", [options]); - }, - unautocomplete: function() { - return this.trigger("unautocomplete"); - } -}); - -$.Autocompleter = function(input, options) { - - var KEY = { - UP: 38, - DOWN: 40, - DEL: 46, - TAB: 9, - RETURN: 13, - ESC: 27, - COMMA: 188, - PAGEUP: 33, - PAGEDOWN: 34, - BACKSPACE: 8 - }; - - // Create $ object for input element - var $input = $(input).attr("autocomplete", "off").addClass(options.inputClass); - - var timeout; - var previousValue = ""; - var cache = $.Autocompleter.Cache(options); - var hasFocus = 0; - var lastKeyPressCode; - var config = { - mouseDownOnSelect: false - }; - var select = $.Autocompleter.Select(options, input, selectCurrent, config); - - var blockSubmit; - - // prevent form submit in opera when selecting with return key - $.browser.opera && $(input.form).bind("submit.autocomplete", function() { - if (blockSubmit) { - blockSubmit = false; - return false; - } - }); - - // only opera doesn't trigger keydown multiple times while pressed, others don't work with keypress at all - $input.bind(($.browser.opera ? "keypress" : "keydown") + ".autocomplete", function(event) { - // track last key pressed - lastKeyPressCode = event.keyCode; - switch(event.keyCode) { - - case KEY.UP: - event.preventDefault(); - if ( select.visible() ) { - select.prev(); - } else { - onChange(0, true); - } - break; - - case KEY.DOWN: - event.preventDefault(); - if ( select.visible() ) { - select.next(); - } else { - onChange(0, true); - } - break; - - case KEY.PAGEUP: - event.preventDefault(); - if ( select.visible() ) { - select.pageUp(); - } else { - onChange(0, true); - } - break; - - case KEY.PAGEDOWN: - event.preventDefault(); - if ( select.visible() ) { - select.pageDown(); - } else { - onChange(0, true); - } - break; - - // matches also semicolon - case options.multiple && $.trim(options.multipleSeparator) == "," && KEY.COMMA: - case KEY.TAB: - case KEY.RETURN: - if( selectCurrent() ) { - // stop default to prevent a form submit, Opera needs special handling - event.preventDefault(); - blockSubmit = true; - return false; - } - break; - - case KEY.ESC: - select.hide(); - break; - - default: - clearTimeout(timeout); - timeout = setTimeout(onChange, options.delay); - break; - } - }).focus(function(){ - // track whether the field has focus, we shouldn't process any - // results if the field no longer has focus - hasFocus++; - }).blur(function() { - hasFocus = 0; - if (!config.mouseDownOnSelect) { - hideResults(); - } - }).click(function() { - // show select when clicking in a focused field - if ( hasFocus++ > 1 && !select.visible() ) { - onChange(0, true); - } - }).bind("search", function() { - // TODO why not just specifying both arguments? - var fn = (arguments.length > 1) ? arguments[1] : null; - function findValueCallback(q, data) { - var result; - if( data && data.length ) { - for (var i=0; i < data.length; i++) { - if( data[i].result.toLowerCase() == q.toLowerCase() ) { - result = data[i]; - break; - } - } - } - if( typeof fn == "function" ) fn(result); - else $input.trigger("result", result && [result.data, result.value]); - } - $.each(trimWords($input.val()), function(i, value) { - request(value, findValueCallback, findValueCallback); - }); - }).bind("flushCache", function() { - cache.flush(); - }).bind("setOptions", function() { - $.extend(options, arguments[1]); - // if we've updated the data, repopulate - if ( "data" in arguments[1] ) - cache.populate(); - }).bind("unautocomplete", function() { - select.unbind(); - $input.unbind(); - $(input.form).unbind(".autocomplete"); - }); - - - function selectCurrent() { - var selected = select.selected(); - if( !selected ) - return false; - - var v = selected.result; - previousValue = v; - - if ( options.multiple ) { - var words = trimWords($input.val()); - if ( words.length > 1 ) { - v = words.slice(0, words.length - 1).join( options.multipleSeparator ) + options.multipleSeparator + v; - } - v += options.multipleSeparator; - } - - $input.val(v); - hideResultsNow(); - $input.trigger("result", [selected.data, selected.value]); - return true; - } - - function onChange(crap, skipPrevCheck) { - if( lastKeyPressCode == KEY.DEL ) { - select.hide(); - return; - } - - var currentValue = $input.val(); - - if ( !skipPrevCheck && currentValue == previousValue ) - return; - - previousValue = currentValue; - - currentValue = lastWord(currentValue); - if ( currentValue.length >= options.minChars) { - $input.addClass(options.loadingClass); - if (!options.matchCase) - currentValue = currentValue.toLowerCase(); - request(currentValue, receiveData, hideResultsNow); - } else { - stopLoading(); - select.hide(); - } - }; - - function trimWords(value) { - if ( !value ) { - return [""]; - } - var words = value.split( options.multipleSeparator ); - var result = []; - $.each(words, function(i, value) { - if ( $.trim(value) ) - result[i] = $.trim(value); - }); - return result; - } - - function lastWord(value) { - if ( !options.multiple ) - return value; - var words = trimWords(value); - return words[words.length - 1]; - } - - // fills in the input box w/the first match (assumed to be the best match) - // q: the term entered - // sValue: the first matching result - function autoFill(q, sValue){ - // autofill in the complete box w/the first match as long as the user hasn't entered in more data - // if the last user key pressed was backspace, don't autofill - if( options.autoFill && (lastWord($input.val()).toLowerCase() == q.toLowerCase()) && lastKeyPressCode != KEY.BACKSPACE ) { - // fill in the value (keep the case the user has typed) - $input.val($input.val() + sValue.substring(lastWord(previousValue).length)); - // select the portion of the value not typed by the user (so the next character will erase) - $.Autocompleter.Selection(input, previousValue.length, previousValue.length + sValue.length); - } - }; - - function hideResults() { - clearTimeout(timeout); - timeout = setTimeout(hideResultsNow, 200); - }; - - function hideResultsNow() { - var wasVisible = select.visible(); - select.hide(); - clearTimeout(timeout); - stopLoading(); - if (options.mustMatch) { - // call search and run callback - $input.search( - function (result){ - // if no value found, clear the input box - if( !result ) { - if (options.multiple) { - var words = trimWords($input.val()).slice(0, -1); - $input.val( words.join(options.multipleSeparator) + (words.length ? options.multipleSeparator : "") ); - } - else - $input.val( "" ); - } - } - ); - } - if (wasVisible) - // position cursor at end of input field - $.Autocompleter.Selection(input, input.value.length, input.value.length); - }; - - function receiveData(q, data) { - if ( data && data.length && hasFocus ) { - stopLoading(); - select.display(data, q); - autoFill(q, data[0].value); - select.show(); - } else { - hideResultsNow(); - } - }; - - function request(term, success, failure) { - if (!options.matchCase) - term = term.toLowerCase(); - var data = cache.load(term); - // recieve the cached data - if (data && data.length) { - success(term, data); - // if an AJAX url has been supplied, try loading the data now - } else if( (typeof options.url == "string") && (options.url.length > 0) ){ - - var extraParams = { - timestamp: +new Date() - }; - $.each(options.extraParams, function(key, param) { - extraParams[key] = typeof param == "function" ? param() : param; - }); - - $.ajax({ - // try to leverage ajaxQueue plugin to abort previous requests - mode: "abort", - // limit abortion to this input - port: "autocomplete" + input.name, - dataType: options.dataType, - url: options.url, - data: $.extend({ - q: lastWord(term), - limit: options.max - }, extraParams), - success: function(data) { - var parsed = options.parse && options.parse(data) || parse(data); - cache.add(term, parsed); - success(term, parsed); - } - }); - } else { - // if we have a failure, we need to empty the list -- this prevents the the [TAB] key from selecting the last successful match - select.emptyList(); - failure(term); - } - }; - - function parse(data) { - var parsed = []; - var rows = data.split("\n"); - for (var i=0; i < rows.length; i++) { - var row = $.trim(rows[i]); - if (row) { - row = row.split("|"); - parsed[parsed.length] = { - data: row, - value: row[0], - result: options.formatResult && options.formatResult(row, row[0]) || row[0] - }; - } - } - return parsed; - }; - - function stopLoading() { - $input.removeClass(options.loadingClass); - }; - -}; - -$.Autocompleter.defaults = { - inputClass: "ac_input", - resultsClass: "ac_results", - loadingClass: "ac_loading", - minChars: 1, - delay: 400, - matchCase: false, - matchSubset: true, - matchContains: false, - cacheLength: 10, - max: 100, - mustMatch: false, - extraParams: {}, - selectFirst: true, - formatItem: function(row) { return row[0]; }, - formatMatch: null, - autoFill: false, - width: 0, - multiple: false, - multipleSeparator: ", ", - highlight: function(value, term) { - return value.replace(new RegExp("(?![^&;]+;)(?!<[^<>]*)(" + term.replace(/([\^\$\(\)\[\]\{\}\*\.\+\?\|\\])/gi, "\\$1") + ")(?![^<>]*>)(?![^&;]+;)", "gi"), "$1"); - }, - scroll: true, - scrollHeight: 180 -}; - -$.Autocompleter.Cache = function(options) { - - var data = {}; - var length = 0; - - function matchSubset(s, sub) { - if (!options.matchCase) - s = s.toLowerCase(); - var i = s.indexOf(sub); - if (options.matchContains == "word"){ - i = s.toLowerCase().search("\\b" + sub.toLowerCase()); - } - if (i == -1) return false; - return i == 0 || options.matchContains; - }; - - function add(q, value) { - if (length > options.cacheLength){ - flush(); - } - if (!data[q]){ - length++; - } - data[q] = value; - } - - function populate(){ - if( !options.data ) return false; - // track the matches - var stMatchSets = {}, - nullData = 0; - - // no url was specified, we need to adjust the cache length to make sure it fits the local data store - if( !options.url ) options.cacheLength = 1; - - // track all options for minChars = 0 - stMatchSets[""] = []; - - // loop through the array and create a lookup structure - for ( var i = 0, ol = options.data.length; i < ol; i++ ) { - var rawValue = options.data[i]; - // if rawValue is a string, make an array otherwise just reference the array - rawValue = (typeof rawValue == "string") ? [rawValue] : rawValue; - - var value = options.formatMatch(rawValue, i+1, options.data.length); - if ( value === false ) - continue; - - var firstChar = value.charAt(0).toLowerCase(); - // if no lookup array for this character exists, look it up now - if( !stMatchSets[firstChar] ) - stMatchSets[firstChar] = []; - - // if the match is a string - var row = { - value: value, - data: rawValue, - result: options.formatResult && options.formatResult(rawValue) || value - }; - - // push the current match into the set list - stMatchSets[firstChar].push(row); - - // keep track of minChars zero items - if ( nullData++ < options.max ) { - stMatchSets[""].push(row); - } - }; - - // add the data items to the cache - $.each(stMatchSets, function(i, value) { - // increase the cache size - options.cacheLength++; - // add to the cache - add(i, value); - }); - } - - // populate any existing data - setTimeout(populate, 25); - - function flush(){ - data = {}; - length = 0; - } - - return { - flush: flush, - add: add, - populate: populate, - load: function(q) { - if (!options.cacheLength || !length) - return null; - /* - * if dealing w/local data and matchContains than we must make sure - * to loop through all the data collections looking for matches - */ - if( !options.url && options.matchContains ){ - // track all matches - var csub = []; - // loop through all the data grids for matches - for( var k in data ){ - // don't search through the stMatchSets[""] (minChars: 0) cache - // this prevents duplicates - if( k.length > 0 ){ - var c = data[k]; - $.each(c, function(i, x) { - // if we've got a match, add it to the array - if (matchSubset(x.value, q)) { - csub.push(x); - } - }); - } - } - return csub; - } else - // if the exact item exists, use it - if (data[q]){ - return data[q]; - } else - if (options.matchSubset) { - for (var i = q.length - 1; i >= options.minChars; i--) { - var c = data[q.substr(0, i)]; - if (c) { - var csub = []; - $.each(c, function(i, x) { - if (matchSubset(x.value, q)) { - csub[csub.length] = x; - } - }); - return csub; - } - } - } - return null; - } - }; -}; - -$.Autocompleter.Select = function (options, input, select, config) { - var CLASSES = { - ACTIVE: "ac_over" - }; - - var listItems, - active = -1, - data, - term = "", - needsInit = true, - element, - list; - - // Create results - function init() { - if (!needsInit) - return; - element = $("
      ") - .hide() - .addClass(options.resultsClass) - .css("position", "absolute") - .appendTo(document.body); - - list = $("