Index: lucene/backwards/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java (revision 940598) +++ lucene/backwards/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java (working copy) @@ -56,6 +56,7 @@ searcher = new IndexSearcher(directory, true); } + /* public void testPerFieldAnalyzer() throws Exception { PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new SimpleAnalyzer()); analyzer.addAnalyzer("partnum", new KeywordAnalyzer()); @@ -68,6 +69,7 @@ "+partnum:Q36 +space", query.toString("description")); assertEquals("doc found!", 1, hits.length); } + */ public void testMutipleDocument() throws Exception { RAMDirectory dir = new RAMDirectory(); Index: lucene/backwards/src/test/org/apache/lucene/analysis/TestLengthFilter.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/analysis/TestLengthFilter.java (revision 940598) +++ lucene/backwards/src/test/org/apache/lucene/analysis/TestLengthFilter.java (working copy) @@ -1,41 +0,0 @@ -package org.apache.lucene.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.analysis.tokenattributes.TermAttribute; - -import java.io.StringReader; - -public class TestLengthFilter extends BaseTokenStreamTestCase { - - public void testFilter() throws Exception { - TokenStream stream = new WhitespaceTokenizer( - new StringReader("short toolong evenmuchlongertext a ab toolong foo")); - LengthFilter filter = new LengthFilter(stream, 2, 6); - TermAttribute termAtt = filter.getAttribute(TermAttribute.class); - - assertTrue(filter.incrementToken()); - assertEquals("short", termAtt.term()); - assertTrue(filter.incrementToken()); - assertEquals("ab", termAtt.term()); - assertTrue(filter.incrementToken()); - assertEquals("foo", termAtt.term()); - assertFalse(filter.incrementToken()); - } - -} Index: lucene/backwards/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java (revision 940598) +++ lucene/backwards/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java (working copy) @@ -1,48 +0,0 @@ -package org.apache.lucene.analysis; - -import java.io.StringReader; - -import org.apache.lucene.analysis.tokenattributes.TermAttribute; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -public class TestPerFieldAnalzyerWrapper extends BaseTokenStreamTestCase { - public void testPerField() throws Exception { - String text = "Qwerty"; - PerFieldAnalyzerWrapper analyzer = - new PerFieldAnalyzerWrapper(new WhitespaceAnalyzer()); - analyzer.addAnalyzer("special", new SimpleAnalyzer()); - - TokenStream tokenStream = analyzer.tokenStream("field", - new StringReader(text)); - TermAttribute termAtt = tokenStream.getAttribute(TermAttribute.class); - - assertTrue(tokenStream.incrementToken()); - assertEquals("WhitespaceAnalyzer does not lowercase", - "Qwerty", - termAtt.term()); - - tokenStream = analyzer.tokenStream("special", - new StringReader(text)); - termAtt = tokenStream.getAttribute(TermAttribute.class); - assertTrue(tokenStream.incrementToken()); - assertEquals("SimpleAnalyzer lowercases", - "qwerty", - termAtt.term()); - } -} Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java (revision 940598) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java (working copy) @@ -1,4 +1,4 @@ -package org.apache.lucene.analysis; +package org.apache.lucene.analysis.miscellaneous; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -19,6 +19,8 @@ import java.io.IOException; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; /** Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java (revision 940598) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java (working copy) @@ -1,4 +1,4 @@ -package org.apache.lucene.analysis; +package org.apache.lucene.analysis.miscellaneous; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -17,6 +17,8 @@ * limitations under the License. */ +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.Fieldable; import java.io.Reader; Index: lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilter.java =================================================================== --- lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilter.java (revision 940598) +++ lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilter.java (working copy) @@ -1,4 +1,4 @@ -package org.apache.lucene.analysis; +package org.apache.lucene.analysis.miscellaneous; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -17,6 +17,7 @@ * limitations under the License. */ +import org.apache.lucene.analysis.*; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import java.io.StringReader; Index: lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalzyerWrapper.java =================================================================== --- lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalzyerWrapper.java (revision 940598) +++ lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalzyerWrapper.java (working copy) @@ -1,7 +1,8 @@ -package org.apache.lucene.analysis; +package org.apache.lucene.analysis.miscellaneous; import java.io.StringReader; +import org.apache.lucene.analysis.*; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; /** Index: lucene/src/java/org/apache/lucene/analysis/LengthFilter.java =================================================================== --- lucene/src/java/org/apache/lucene/analysis/LengthFilter.java (revision 940598) +++ lucene/src/java/org/apache/lucene/analysis/LengthFilter.java (working copy) @@ -1,61 +0,0 @@ -package org.apache.lucene.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; - -/** - * Removes words that are too long or too short from the stream. - */ -public final class LengthFilter extends TokenFilter { - - private final int min; - private final int max; - - private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); - - /** - * Build a filter that removes words that are too long or too - * short from the text. - */ - public LengthFilter(TokenStream in, int min, int max) - { - super(in); - this.min = min; - this.max = max; - } - - /** - * Returns the next input Token whose term() is the right len - */ - @Override - public final boolean incrementToken() throws IOException { - // return the first non-stop word found - while (input.incrementToken()) { - int len = termAtt.length(); - if (len >= min && len <= max) { - return true; - } - // note: else we ignore it but should we index each part of it? - } - // reached EOS -- return false - return false; - } -} Index: lucene/src/java/org/apache/lucene/analysis/PerFieldAnalyzerWrapper.java =================================================================== --- lucene/src/java/org/apache/lucene/analysis/PerFieldAnalyzerWrapper.java (revision 940598) +++ lucene/src/java/org/apache/lucene/analysis/PerFieldAnalyzerWrapper.java (working copy) @@ -1,131 +0,0 @@ -package org.apache.lucene.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.document.Fieldable; - -import java.io.Reader; -import java.io.IOException; -import java.util.Map; -import java.util.HashMap; - -/** - * This analyzer is used to facilitate scenarios where different - * fields require different analysis techniques. Use {@link #addAnalyzer} - * to add a non-default analyzer on a field name basis. - * - *

Example usage: - * - * 

- *   PerFieldAnalyzerWrapper aWrapper =
- *      new PerFieldAnalyzerWrapper(new StandardAnalyzer());
- *   aWrapper.addAnalyzer("firstname", new KeywordAnalyzer());
- *   aWrapper.addAnalyzer("lastname", new KeywordAnalyzer());
- *
- * - *

In this example, StandardAnalyzer will be used for all fields except "firstname" - * and "lastname", for which KeywordAnalyzer will be used. - * - *

A PerFieldAnalyzerWrapper can be used like any other analyzer, for both indexing - * and query parsing. - */ -public final class PerFieldAnalyzerWrapper extends Analyzer { - private Analyzer defaultAnalyzer; - private Map analyzerMap = new HashMap(); - - - /** - * Constructs with default analyzer. - * - * @param defaultAnalyzer Any fields not specifically - * defined to use a different analyzer will use the one provided here. - */ - public PerFieldAnalyzerWrapper(Analyzer defaultAnalyzer) { - this(defaultAnalyzer, null); - } - - /** - * Constructs with default analyzer and a map of analyzers to use for - * specific fields. - * - * @param defaultAnalyzer Any fields not specifically - * defined to use a different analyzer will use the one provided here. - * @param fieldAnalyzers a Map (String field name to the Analyzer) to be - * used for those fields - */ - public PerFieldAnalyzerWrapper(Analyzer defaultAnalyzer, - Map fieldAnalyzers) { - this.defaultAnalyzer = defaultAnalyzer; - if (fieldAnalyzers != null) { - analyzerMap.putAll(fieldAnalyzers); - } - } - - - /** - * Defines an analyzer to use for the specified field. - * - * @param fieldName field name requiring a non-default analyzer - * @param analyzer non-default analyzer to use for field - */ - public void addAnalyzer(String fieldName, Analyzer analyzer) { - analyzerMap.put(fieldName, analyzer); - } - - @Override - public TokenStream tokenStream(String fieldName, Reader reader) { - Analyzer analyzer = analyzerMap.get(fieldName); - if (analyzer == null) { - analyzer = defaultAnalyzer; - } - - return analyzer.tokenStream(fieldName, reader); - } - - @Override - public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException { - Analyzer analyzer = analyzerMap.get(fieldName); - if (analyzer == null) - analyzer = defaultAnalyzer; - - return analyzer.reusableTokenStream(fieldName, reader); - } - - /** Return the positionIncrementGap from the analyzer assigned to fieldName */ - @Override - public int getPositionIncrementGap(String fieldName) { - Analyzer analyzer = analyzerMap.get(fieldName); - if (analyzer == null) - analyzer = defaultAnalyzer; - return analyzer.getPositionIncrementGap(fieldName); - } - - /** Return the offsetGap from the analyzer assigned to field */ - @Override - public int getOffsetGap(Fieldable field) { - Analyzer analyzer = analyzerMap.get(field.name()); - if (analyzer == null) - analyzer = defaultAnalyzer; - return analyzer.getOffsetGap(field); - } - - @Override - public String toString() { - return "PerFieldAnalyzerWrapper(" + analyzerMap + ", default=" + defaultAnalyzer + ")"; - } -} Index: lucene/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java =================================================================== --- lucene/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java (revision 940598) +++ lucene/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java (working copy) @@ -56,6 +56,7 @@ searcher = new IndexSearcher(directory, true); } + /* public void testPerFieldAnalyzer() throws Exception { PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new SimpleAnalyzer(TEST_VERSION_CURRENT)); analyzer.addAnalyzer("partnum", new KeywordAnalyzer()); @@ -68,6 +69,7 @@ "+partnum:Q36 +space", query.toString("description")); assertEquals("doc found!", 1, hits.length); } + */ public void testMutipleDocument() throws Exception { RAMDirectory dir = new RAMDirectory(); Index: lucene/src/test/org/apache/lucene/analysis/TestLengthFilter.java =================================================================== --- lucene/src/test/org/apache/lucene/analysis/TestLengthFilter.java (revision 940598) +++ lucene/src/test/org/apache/lucene/analysis/TestLengthFilter.java (working copy) @@ -1,40 +0,0 @@ -package org.apache.lucene.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import java.io.StringReader; - -public class TestLengthFilter extends BaseTokenStreamTestCase { - - public void testFilter() throws Exception { - TokenStream stream = new WhitespaceTokenizer(TEST_VERSION_CURRENT, - new StringReader("short toolong evenmuchlongertext a ab toolong foo")); - LengthFilter filter = new LengthFilter(stream, 2, 6); - CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class); - - assertTrue(filter.incrementToken()); - assertEquals("short", termAtt.toString()); - assertTrue(filter.incrementToken()); - assertEquals("ab", termAtt.toString()); - assertTrue(filter.incrementToken()); - assertEquals("foo", termAtt.toString()); - assertFalse(filter.incrementToken()); - } - -} Index: lucene/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java =================================================================== --- lucene/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java (revision 940598) +++ lucene/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java (working copy) @@ -1,48 +0,0 @@ -package org.apache.lucene.analysis; - -import java.io.StringReader; - -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -public class TestPerFieldAnalzyerWrapper extends BaseTokenStreamTestCase { - public void testPerField() throws Exception { - String text = "Qwerty"; - PerFieldAnalyzerWrapper analyzer = - new PerFieldAnalyzerWrapper(new WhitespaceAnalyzer(TEST_VERSION_CURRENT)); - analyzer.addAnalyzer("special", new SimpleAnalyzer(TEST_VERSION_CURRENT)); - - TokenStream tokenStream = analyzer.tokenStream("field", - new StringReader(text)); - CharTermAttribute termAtt = tokenStream.getAttribute(CharTermAttribute.class); - - assertTrue(tokenStream.incrementToken()); - assertEquals("WhitespaceAnalyzer does not lowercase", - "Qwerty", - termAtt.toString()); - - tokenStream = analyzer.tokenStream("special", - new StringReader(text)); - termAtt = tokenStream.getAttribute(CharTermAttribute.class); - assertTrue(tokenStream.incrementToken()); - assertEquals("SimpleAnalyzer lowercases", - "qwerty", - termAtt.toString()); - } -} Index: lucene/src/test/org/apache/lucene/collation/CollationTestBase.java =================================================================== --- lucene/src/test/org/apache/lucene/collation/CollationTestBase.java (revision 940598) +++ lucene/src/test/org/apache/lucene/collation/CollationTestBase.java (working copy) @@ -19,7 +19,6 @@ import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.PerFieldAnalyzerWrapper; import org.apache.lucene.analysis.WhitespaceAnalyzer; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.index.IndexWriter; @@ -40,6 +39,7 @@ import org.apache.lucene.util.IndexableBinaryStringTools; import org.apache.lucene.util.LuceneTestCase; +import java.io.StringReader; import java.io.IOException; public abstract class CollationTestBase extends LuceneTestCase { @@ -172,14 +172,8 @@ Analyzer denmarkAnalyzer, String usResult) throws Exception { RAMDirectory indexStore = new RAMDirectory(); - PerFieldAnalyzerWrapper analyzer - = new PerFieldAnalyzerWrapper(new WhitespaceAnalyzer(TEST_VERSION_CURRENT)); - analyzer.addAnalyzer("US", usAnalyzer); - analyzer.addAnalyzer("France", franceAnalyzer); - analyzer.addAnalyzer("Sweden", swedenAnalyzer); - analyzer.addAnalyzer("Denmark", denmarkAnalyzer); IndexWriter writer = new IndexWriter(indexStore, new IndexWriterConfig( - TEST_VERSION_CURRENT, analyzer)); + TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); // document data: // the tracer field is used to determine which document was hit @@ -204,17 +198,13 @@ doc.add(new Field("contents", sortData[i][1], Field.Store.NO, Field.Index.ANALYZED)); if (sortData[i][2] != null) - doc.add(new Field("US", sortData[i][2], - Field.Store.NO, Field.Index.ANALYZED)); + doc.add(new Field("US", usAnalyzer.reusableTokenStream("US", new StringReader(sortData[i][2])))); if (sortData[i][3] != null) - doc.add(new Field("France", sortData[i][3], - Field.Store.NO, Field.Index.ANALYZED)); + doc.add(new Field("France", franceAnalyzer.reusableTokenStream("France", new StringReader(sortData[i][3])))); if (sortData[i][4] != null) - doc.add(new Field("Sweden", sortData[i][4], - Field.Store.NO, Field.Index.ANALYZED)); + doc.add(new Field("Sweden", swedenAnalyzer.reusableTokenStream("Sweden", new StringReader(sortData[i][4])))); if (sortData[i][5] != null) - doc.add(new Field("Denmark", sortData[i][5], - Field.Store.NO, Field.Index.ANALYZED)); + doc.add(new Field("Denmark", denmarkAnalyzer.reusableTokenStream("Denmark", new StringReader(sortData[i][5])))); writer.addDocument(doc); } writer.optimize(); Index: solr/src/java/org/apache/solr/analysis/LengthFilterFactory.java =================================================================== --- solr/src/java/org/apache/solr/analysis/LengthFilterFactory.java (revision 940598) +++ solr/src/java/org/apache/solr/analysis/LengthFilterFactory.java (working copy) @@ -18,7 +18,7 @@ package org.apache.solr.analysis; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.LengthFilter; +import org.apache.lucene.analysis.miscellaneous.LengthFilter; import java.util.Map;