Index: solr/test-framework/src/java/org/apache/solr/analysis/MockCharFilterFactory.java =================================================================== --- solr/test-framework/src/java/org/apache/solr/analysis/MockCharFilterFactory.java (revision 0) +++ solr/test-framework/src/java/org/apache/solr/analysis/MockCharFilterFactory.java (working copy) @@ -0,0 +1,46 @@ +package org.apache.solr.analysis; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Map; + +import org.apache.lucene.analysis.CharStream; +import org.apache.lucene.analysis.MockCharFilter; +import org.apache.lucene.analysis.util.CharFilterFactory; + +/** + * Factory for {@link MockCharFilter} for testing purposes. + */ +public class MockCharFilterFactory extends CharFilterFactory { + int remainder; + + @Override + public void init(Map args) { + super.init(args); + String sval = args.get("remainder"); + if (sval == null) { + throw new IllegalArgumentException("remainder is mandatory"); + } + remainder = Integer.parseInt(sval); + } + + @Override + public CharStream create(CharStream input) { + return new MockCharFilter(input, remainder); + } +} Property changes on: solr/test-framework/src/java/org/apache/solr/analysis/MockCharFilterFactory.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native Index: solr/core/src/test-files/solr/collection1/conf/schema-charfilters.xml =================================================================== --- solr/core/src/test-files/solr/collection1/conf/schema-charfilters.xml (revision 0) +++ solr/core/src/test-files/solr/collection1/conf/schema-charfilters.xml (working copy) @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + content + id + + Property changes on: solr/core/src/test-files/solr/collection1/conf/schema-charfilters.xml ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native Index: solr/core/src/test/org/apache/solr/analysis/TestCharFilters.java =================================================================== --- solr/core/src/test/org/apache/solr/analysis/TestCharFilters.java (revision 0) +++ solr/core/src/test/org/apache/solr/analysis/TestCharFilters.java (working copy) @@ -0,0 +1,77 @@ +package org.apache.solr.analysis; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.solr.SolrTestCaseJ4; +import org.junit.BeforeClass; + +/** + * Tests that charfilters are being applied properly + * (e.g. once and only once) with mockcharfilter. + */ +public class TestCharFilters extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig-basic.xml","schema-charfilters.xml"); + // add some docs + assertU(adoc("id", "1", "content", "aab")); + assertU(adoc("id", "2", "content", "aabaa")); + assertU(adoc("id", "3", "content2", "ab")); + assertU(adoc("id", "4", "content2", "aba")); + assertU(commit()); + } + + /** + * Test query analysis: at querytime MockCharFilter will + * double the 'a', so ab -> aab, and aba -> aabaa + * + * We run the test twice to make sure reuse is working + */ + public void testQueryAnalysis() { + assertQ("Query analysis: ", + req("fl", "id", "q", "content:ab", "sort", "id asc" ), + "//*[@numFound='1']", + "//result/doc[1]/int[@name='id'][.=1]" + ); + assertQ("Query analysis: ", + req("fl", "id", "q", "content:aba", "sort", "id asc" ), + "//*[@numFound='1']", + "//result/doc[1]/int[@name='id'][.=2]" + ); + } + + /** + * Test index analysis: at indextime MockCharFilter will + * double the 'a', so ab -> aab, and aba -> aabaa + * + * We run the test twice to make sure reuse is working + */ + public void testIndexAnalysis() { + assertQ("Index analysis: ", + req("fl", "id", "q", "content2:aab", "sort", "id asc" ), + "//*[@numFound='1']", + "//result/doc[1]/int[@name='id'][.=3]" + ); + assertQ("Index analysis: ", + req("fl", "id", "q", "content2:aabaa", "sort", "id asc" ), + "//*[@numFound='1']", + "//result/doc[1]/int[@name='id'][.=4]" + ); + } +} Property changes on: solr/core/src/test/org/apache/solr/analysis/TestCharFilters.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native Index: solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java =================================================================== --- solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java (revision 1358250) +++ solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java (working copy) @@ -22,7 +22,6 @@ import org.apache.lucene.analysis.util.TokenFilterFactory; import org.apache.lucene.analysis.util.TokenizerFactory; -import java.io.IOException; import java.io.Reader; /** @@ -52,22 +51,6 @@ public TokenizerFactory getTokenizerFactory() { return tokenizer; } public TokenFilterFactory[] getTokenFilterFactories() { return filters; } - class SolrTokenStreamComponents extends TokenStreamComponents { - public SolrTokenStreamComponents(final Tokenizer source, final TokenStream result) { - super(source, result); - } - - // TODO: what is going on here? - @Override - protected void reset(Reader reader) throws IOException { - // the tokenizers are currently reset by the indexing process, so only - // the tokenizer needs to be reset. - Reader r = initReader(null, reader); - super.reset(r); - } - } - - @Override public Reader initReader(String fieldName, Reader reader) { if (charFilters != null && charFilters.length > 0) { @@ -82,12 +65,12 @@ @Override protected TokenStreamComponents createComponents(String fieldName, Reader aReader) { - Tokenizer tk = tokenizer.create( initReader(fieldName, aReader) ); + Tokenizer tk = tokenizer.create( aReader ); TokenStream ts = tk; for (TokenFilterFactory filter : filters) { ts = filter.create(ts); } - return new SolrTokenStreamComponents(tk, ts); + return new TokenStreamComponents(tk, ts); } @Override