Index: lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java =================================================================== --- lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java (revision 1362105) +++ lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java (working copy) @@ -19,11 +19,9 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.CharReader; -import org.apache.lucene.analysis.CharStream; +import org.apache.lucene.analysis.CharFilter; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.charfilter.CharFilter; import java.io.IOException; import java.io.Reader; @@ -40,7 +38,7 @@ @Override protected Reader initReader(String fieldName, Reader reader) { - return new JapaneseIterationMarkCharFilter(CharReader.get(reader)); + return new JapaneseIterationMarkCharFilter(reader); } }; @@ -53,7 +51,7 @@ @Override protected Reader initReader(String fieldName, Reader reader) { - return new JapaneseIterationMarkCharFilter(CharReader.get(reader)); + return new JapaneseIterationMarkCharFilter(reader); } }; @@ -138,7 +136,7 @@ public void testKanjiOnly() throws IOException { // Test kanji only repetition marks CharFilter filter = new JapaneseIterationMarkCharFilter( - CharReader.get(new StringReader("時々、おゝのさんと一緒にお寿司が食べたいです。abcところゞゝゝ。")), + new StringReader("時々、おゝのさんと一緒にお寿司が食べたいです。abcところゞゝゝ。"), true, // kanji false // no kana ); @@ -148,7 +146,7 @@ public void testKanaOnly() throws IOException { // Test kana only repetition marks CharFilter filter = new JapaneseIterationMarkCharFilter( - CharReader.get(new StringReader("時々、おゝのさんと一緒にお寿司が食べたいです。abcところゞゝゝ。")), + new StringReader("時々、おゝのさんと一緒にお寿司が食べたいです。abcところゞゝゝ。"), false, // no kanji true // kana ); @@ -158,7 +156,7 @@ public void testNone() throws IOException { // Test no repetition marks CharFilter filter = new JapaneseIterationMarkCharFilter( - CharReader.get(new StringReader("時々、おゝのさんと一緒にお寿司が食べたいです。abcところゞゝゝ。")), + new StringReader("時々、おゝのさんと一緒にお寿司が食べたいです。abcところゞゝゝ。"), false, // no kanji false // no kana ); @@ -210,7 +208,7 @@ assertEquals(expected, actual); } - private String readFully(CharStream stream) throws IOException { + private String readFully(Reader stream) throws IOException { StringBuffer buffer = new StringBuffer(); int ch; while ((ch = stream.read()) != -1) { Index: lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilter.java =================================================================== --- lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilter.java (revision 1362220) +++ lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilter.java (working copy) @@ -17,11 +17,11 @@ * limitations under the License. */ -import org.apache.lucene.analysis.CharStream; -import org.apache.lucene.analysis.charfilter.CharFilter; +import org.apache.lucene.analysis.CharFilter; import org.apache.lucene.util.RollingCharBuffer; import java.io.IOException; +import java.io.Reader; /** * Normalizes Japanese horizontal iteration marks (odoriji) to their expanded form. @@ -147,7 +147,7 @@ * * @param input char stream */ - public JapaneseIterationMarkCharFilter(CharStream input) { + public JapaneseIterationMarkCharFilter(Reader input) { this(input, NORMALIZE_KANJI_DEFAULT, NORMALIZE_KANA_DEFAULT); } @@ -159,7 +159,7 @@ * @param normalizeKanji indicates whether kanji iteration marks should be normalized * @param normalizeKana indicates whether kana iteration marks should be normalized */ - public JapaneseIterationMarkCharFilter(CharStream input, boolean normalizeKanji, boolean normalizeKana) { + public JapaneseIterationMarkCharFilter(Reader input, boolean normalizeKanji, boolean normalizeKana) { super(input); this.normalizeKanji = normalizeKanji; this.normalizeKana = normalizeKana; @@ -453,4 +453,10 @@ private boolean inside(char c, char[] map, char offset) { return c >= offset && c < offset + map.length; } + + + @Override + protected int correct(int currentOff) { + return currentOff; // this filter doesn't change the length of strings + } } Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java =================================================================== --- lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java (revision 1362220) +++ lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java (working copy) @@ -29,8 +29,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.CharReader; -import org.apache.lucene.analysis.CharStream; +import org.apache.lucene.analysis.CharFilter; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; @@ -60,7 +59,7 @@ } public void testReaderReset() throws Exception { - CharStream cs = new MappingCharFilter( normMap, new StringReader( "x" ) ); + CharFilter cs = new MappingCharFilter( normMap, new StringReader( "x" ) ); char[] buf = new char[10]; int len = cs.read(buf, 0, 10); assertEquals( 1, len ); @@ -76,55 +75,55 @@ } public void testNothingChange() throws Exception { - CharStream cs = new MappingCharFilter( normMap, new StringReader( "x" ) ); + CharFilter cs = new MappingCharFilter( normMap, new StringReader( "x" ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[]{"x"}, new int[]{0}, new int[]{1}, 1); } public void test1to1() throws Exception { - CharStream cs = new MappingCharFilter( normMap, new StringReader( "h" ) ); + CharFilter cs = new MappingCharFilter( normMap, new StringReader( "h" ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[]{"i"}, new int[]{0}, new int[]{1}, 1); } public void test1to2() throws Exception { - CharStream cs = new MappingCharFilter( normMap, new StringReader( "j" ) ); + CharFilter cs = new MappingCharFilter( normMap, new StringReader( "j" ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[]{"jj"}, new int[]{0}, new int[]{1}, 1); } public void test1to3() throws Exception { - CharStream cs = new MappingCharFilter( normMap, new StringReader( "k" ) ); + CharFilter cs = new MappingCharFilter( normMap, new StringReader( "k" ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[]{"kkk"}, new int[]{0}, new int[]{1}, 1); } public void test2to4() throws Exception { - CharStream cs = new MappingCharFilter( normMap, new StringReader( "ll" ) ); + CharFilter cs = new MappingCharFilter( normMap, new StringReader( "ll" ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[]{"llll"}, new int[]{0}, new int[]{2}, 2); } public void test2to1() throws Exception { - CharStream cs = new MappingCharFilter( normMap, new StringReader( "aa" ) ); + CharFilter cs = new MappingCharFilter( normMap, new StringReader( "aa" ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[]{"a"}, new int[]{0}, new int[]{2}, 2); } public void test3to1() throws Exception { - CharStream cs = new MappingCharFilter( normMap, new StringReader( "bbb" ) ); + CharFilter cs = new MappingCharFilter( normMap, new StringReader( "bbb" ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[]{"b"}, new int[]{0}, new int[]{3}, 3); } public void test4to2() throws Exception { - CharStream cs = new MappingCharFilter( normMap, new StringReader( "cccc" ) ); + CharFilter cs = new MappingCharFilter( normMap, new StringReader( "cccc" ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[]{"cc"}, new int[]{0}, new int[]{4}, 4); } public void test5to0() throws Exception { - CharStream cs = new MappingCharFilter( normMap, new StringReader( "empty" ) ); + CharFilter cs = new MappingCharFilter( normMap, new StringReader( "empty" ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[0], new int[]{}, new int[]{}, 5); } @@ -149,7 +148,7 @@ // public void testTokenStream() throws Exception { String testString = "h i j k ll cccc bbb aa"; - CharStream cs = new MappingCharFilter( normMap, CharReader.get( new StringReader( testString ) ) ); + CharFilter cs = new MappingCharFilter( normMap, new StringReader( testString ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[]{"i","i","jj","kkk","llll","cc","b","a"}, @@ -171,8 +170,8 @@ // h,8,9 => i,8,9 public void testChained() throws Exception { String testString = "aaaa ll h"; - CharStream cs = new MappingCharFilter( normMap, - new MappingCharFilter( normMap, CharReader.get( new StringReader( testString ) ) ) ); + CharFilter cs = new MappingCharFilter( normMap, + new MappingCharFilter( normMap, new StringReader( testString ) ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[]{"a","llllllll","i"}, @@ -193,7 +192,7 @@ @Override protected Reader initReader(String fieldName, Reader reader) { - return new MappingCharFilter(normMap, CharReader.get(reader)); + return new MappingCharFilter(normMap, reader); } }; @@ -219,7 +218,7 @@ @Override protected Reader initReader(String fieldName, Reader reader) { - return new MappingCharFilter(map, CharReader.get(reader)); + return new MappingCharFilter(map, reader); } }; @@ -241,7 +240,7 @@ @Override protected Reader initReader(String fieldName, Reader reader) { - return new MappingCharFilter(map, CharReader.get(reader)); + return new MappingCharFilter(map, reader); } }; int numRounds = 100; Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestCharFilter.java =================================================================== --- lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestCharFilter.java (revision 1362105) +++ lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestCharFilter.java (working copy) @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.lucene.analysis.charfilter; - -import java.io.StringReader; - -import org.apache.lucene.analysis.CharReader; -import org.apache.lucene.analysis.CharStream; -import org.apache.lucene.analysis.charfilter.CharFilter; -import org.apache.lucene.util.LuceneTestCase; - -public class TestCharFilter extends LuceneTestCase { - - public void testCharFilter1() throws Exception { - CharStream cs = new CharFilter1( CharReader.get( new StringReader("") ) ); - assertEquals( "corrected offset is invalid", 1, cs.correctOffset( 0 ) ); - } - - public void testCharFilter2() throws Exception { - CharStream cs = new CharFilter2( CharReader.get( new StringReader("") ) ); - assertEquals( "corrected offset is invalid", 2, cs.correctOffset( 0 ) ); - } - - public void testCharFilter12() throws Exception { - CharStream cs = new CharFilter2( new CharFilter1( CharReader.get( new StringReader("") ) ) ); - assertEquals( "corrected offset is invalid", 3, cs.correctOffset( 0 ) ); - } - - public void testCharFilter11() throws Exception { - CharStream cs = new CharFilter1( new CharFilter1( CharReader.get( new StringReader("") ) ) ); - assertEquals( "corrected offset is invalid", 2, cs.correctOffset( 0 ) ); - } - - static class CharFilter1 extends CharFilter { - - protected CharFilter1(CharStream in) { - super(in); - } - - @Override - protected int correct(int currentOff) { - return currentOff + 1; - } - } - - static class CharFilter2 extends CharFilter { - - protected CharFilter2(CharStream in) { - super(in); - } - - @Override - protected int correct(int currentOff) { - return currentOff + 2; - } - } -} Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterTest.java =================================================================== --- lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterTest.java (revision 1362220) +++ lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterTest.java (working copy) @@ -29,7 +29,6 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.CharReader; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.util._TestUtil; @@ -46,7 +45,7 @@ @Override protected Reader initReader(String fieldName, Reader reader) { - return new HTMLStripCharFilter(CharReader.get(reader)); + return new HTMLStripCharFilter(reader); } }; } @@ -60,7 +59,7 @@ String gold = "\nthis is some text\n here is a link and " + "another link. " + "This is an entity: & plus a <. Here is an &. "; - HTMLStripCharFilter reader = new HTMLStripCharFilter(CharReader.get(new StringReader(html))); + HTMLStripCharFilter reader = new HTMLStripCharFilter(new StringReader(html)); StringBuilder builder = new StringBuilder(); int ch = -1; char [] goldArray = gold.toCharArray(); @@ -79,7 +78,7 @@ //Some sanity checks, but not a full-fledged check public void testHTML() throws Exception { InputStream stream = getClass().getResourceAsStream("htmlStripReaderTest.html"); - HTMLStripCharFilter reader = new HTMLStripCharFilter(CharReader.get(new InputStreamReader(stream, "UTF-8"))); + HTMLStripCharFilter reader = new HTMLStripCharFilter(new InputStreamReader(stream, "UTF-8")); StringBuilder builder = new StringBuilder(); int ch = -1; while ((ch = reader.read()) != -1){ @@ -96,7 +95,7 @@ public void testMSWord14GeneratedHTML() throws Exception { InputStream stream = getClass().getResourceAsStream("MS-Word 14 generated.htm"); - HTMLStripCharFilter reader = new HTMLStripCharFilter(CharReader.get(new InputStreamReader(stream, "UTF-8"))); + HTMLStripCharFilter reader = new HTMLStripCharFilter(new InputStreamReader(stream, "UTF-8")); String gold = "This is a test"; StringBuilder builder = new StringBuilder(); int ch = 0; @@ -117,7 +116,7 @@ String gold = "\u0393"; Set set = new HashSet(); set.add("reserved"); - Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)), set); + Reader reader = new HTMLStripCharFilter(new StringReader(test), set); StringBuilder builder = new StringBuilder(); int ch = 0; while ((ch = reader.read()) != -1){ @@ -132,7 +131,7 @@ String gold = " \u00DCbermensch = \u0393 bar \u0393"; Set set = new HashSet(); set.add("reserved"); - Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)), set); + Reader reader = new HTMLStripCharFilter(new StringReader(test), set); StringBuilder builder = new StringBuilder(); int ch = 0; while ((ch = reader.read()) != -1){ @@ -147,7 +146,7 @@ String gold = " ! @ and ’"; Set set = new HashSet(); set.add("reserved"); - Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)), set); + Reader reader = new HTMLStripCharFilter(new StringReader(test), set); StringBuilder builder = new StringBuilder(); int ch = 0; while ((ch = reader.read()) != -1){ @@ -161,7 +160,7 @@ String test = "aaa bbb eeee ffff "; Set set = new HashSet(); set.add("reserved"); - Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)), set); + Reader reader = new HTMLStripCharFilter(new StringReader(test), set); StringBuilder builder = new StringBuilder(); int ch = 0; while ((ch = reader.read()) != -1){ @@ -346,7 +345,7 @@ for (int i = 0 ; i < testGold.length ; i += 2) { String test = testGold[i]; String gold = testGold[i + 1]; - Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test))); + Reader reader = new HTMLStripCharFilter(new StringReader(test)); StringBuilder builder = new StringBuilder(); int ch = 0; while ((ch = reader.read()) != -1){ @@ -370,7 +369,7 @@ testBuilder.append("-->foo"); String gold = "foo"; - Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(testBuilder.toString()))); + Reader reader = new HTMLStripCharFilter(new StringReader(testBuilder.toString())); int ch = 0; StringBuilder builder = new StringBuilder(); try { @@ -388,7 +387,7 @@ appendChars(testBuilder, HTMLStripCharFilter.getInitialBufferSize() + 500); testBuilder.append("?>"); gold = ""; - reader = new HTMLStripCharFilter(CharReader.get(new StringReader(testBuilder.toString()))); + reader = new HTMLStripCharFilter(new StringReader(testBuilder.toString())); ch = 0; builder = new StringBuilder(); try { @@ -406,7 +405,7 @@ appendChars(testBuilder, HTMLStripCharFilter.getInitialBufferSize() + 500); testBuilder.append("/>"); gold = ""; - reader = new HTMLStripCharFilter(CharReader.get(new StringReader(testBuilder.toString()))); + reader = new HTMLStripCharFilter(new StringReader(testBuilder.toString())); ch = 0; builder = new StringBuilder(); try { @@ -430,7 +429,7 @@ private void processBuffer(String test, String assertMsg) throws IOException { // System.out.println("-------------------processBuffer----------"); - Reader reader = new HTMLStripCharFilter(CharReader.get(new BufferedReader(new StringReader(test))));//force the use of BufferedReader + Reader reader = new HTMLStripCharFilter(new BufferedReader(new StringReader(test)));//force the use of BufferedReader int ch = 0; StringBuilder builder = new StringBuilder(); try { @@ -448,7 +447,7 @@ String test = " "; String gold = " "; - Reader reader = new HTMLStripCharFilter(CharReader.get(new BufferedReader(new StringReader(test))));//force the use of BufferedReader + Reader reader = new HTMLStripCharFilter(new BufferedReader(new StringReader(test)));//force the use of BufferedReader int ch = 0; StringBuilder builder = new StringBuilder(); try { @@ -464,7 +463,7 @@ public void doTestOffsets(String in) throws Exception { - HTMLStripCharFilter reader = new HTMLStripCharFilter(CharReader.get(new BufferedReader(new StringReader(in)))); + HTMLStripCharFilter reader = new HTMLStripCharFilter(new BufferedReader(new StringReader(in))); int ch = 0; int off = 0; // offset in the reader int strOff = -1; // offset in the original string @@ -491,7 +490,7 @@ static void assertLegalOffsets(String in) throws Exception { int length = in.length(); - HTMLStripCharFilter reader = new HTMLStripCharFilter(CharReader.get(new BufferedReader(new StringReader(in)))); + HTMLStripCharFilter reader = new HTMLStripCharFilter(new BufferedReader(new StringReader(in))); int ch = 0; int off = 0; while ((ch = reader.read()) != -1) { @@ -526,7 +525,7 @@ + " alt = \"Alt: \\'Comment\\'}' -->\"\n\n" + " title=\"Title: \">two"; String gold = "onetwo"; - Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test))); + Reader reader = new HTMLStripCharFilter(new StringReader(test)); int ch = 0; StringBuilder builder = new StringBuilder(); try { @@ -540,7 +539,7 @@ test = "onetwo"; gold = "one\ntwo"; - reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test))); + reader = new HTMLStripCharFilter(new StringReader(test)); ch = 0; builder = new StringBuilder(); try { @@ -557,7 +556,7 @@ public void testScriptQuotes() throws Exception { String test = "onetwo"; String gold = "one\ntwo"; - Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test))); + Reader reader = new HTMLStripCharFilter(new StringReader(test)); int ch = 0; StringBuilder builder = new StringBuilder(); try { @@ -572,7 +571,7 @@ test = "hello'); -->"; gold = "hello\n"; - reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test))); + reader = new HTMLStripCharFilter(new StringReader(test)); ch = 0; builder = new StringBuilder(); try { @@ -591,7 +590,7 @@ String gold = "onetwo"; Set escapedTags = new HashSet(Arrays.asList("SCRIPT")); Reader reader = new HTMLStripCharFilter - (CharReader.get(new StringReader(test)), escapedTags); + (new StringReader(test), escapedTags); int ch = 0; StringBuilder builder = new StringBuilder(); try { @@ -612,7 +611,7 @@ + "-->\n" + "two"; String gold = "one\ntwo"; - Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test))); + Reader reader = new HTMLStripCharFilter(new StringReader(test)); int ch = 0; StringBuilder builder = new StringBuilder(); try { @@ -631,7 +630,7 @@ String gold = "onetwo"; Set escapedTags = new HashSet(Arrays.asList("STYLE")); Reader reader = new HTMLStripCharFilter - (CharReader.get(new StringReader(test)), escapedTags); + (new StringReader(test), escapedTags); int ch = 0; StringBuilder builder = new StringBuilder(); try { @@ -656,7 +655,7 @@ for (int i = 0 ; i < testGold.length ; i += 2) { String test = testGold[i]; String gold = testGold[i + 1]; - Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test))); + Reader reader = new HTMLStripCharFilter(new StringReader(test)); StringBuilder builder = new StringBuilder(); int ch = 0; while ((ch = reader.read()) != -1){ @@ -671,7 +670,7 @@ String gold = "one
two"; Set escapedTags = new HashSet(Arrays.asList("BR")); Reader reader = new HTMLStripCharFilter - (CharReader.get(new StringReader(test)), escapedTags); + (new StringReader(test), escapedTags); int ch = 0; StringBuilder builder = new StringBuilder(); try { @@ -688,7 +687,7 @@ public void testInlineTagsNoSpace() throws Exception { String test = "onetwo2e.three"; String gold = "onetwo2e.three"; - Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test))); + Reader reader = new HTMLStripCharFilter(new StringReader(test)); int ch = 0; StringBuilder builder = new StringBuilder(); try { @@ -705,7 +704,7 @@ public void testCDATA() throws Exception { String test = "onethree]]>two"; String gold = "onethreetwo"; - Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test))); + Reader reader = new HTMLStripCharFilter(new StringReader(test)); int ch = 0; StringBuilder builder = new StringBuilder(); try { @@ -720,7 +719,7 @@ test = "onefour]]>five"; gold = "onetwofourfive"; - reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test))); + reader = new HTMLStripCharFilter(new StringReader(test)); ch = 0; builder = new StringBuilder(); try { @@ -737,7 +736,7 @@ public void testUppercaseCharacterEntityVariants() throws Exception { String test = " "-©>><<®&"; String gold = " \"-\u00A9>><<\u00AE&"; - Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test))); + Reader reader = new HTMLStripCharFilter(new StringReader(test)); int ch = 0; StringBuilder builder = new StringBuilder(); try { @@ -754,7 +753,7 @@ public void testMSWordMalformedProcessingInstruction() throws Exception { String test = "onetwo"; String gold = "onetwo"; - Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test))); + Reader reader = new HTMLStripCharFilter(new StringReader(test)); int ch = 0; StringBuilder builder = new StringBuilder(); try { @@ -771,7 +770,7 @@ public void testSupplementaryCharsInTags() throws Exception { String test = "one<𩬅艱鍟䇹愯瀛>two<瀛愯𩬅>three 瀛愯𩬅fourfive<𠀀𠀀>six<𠀀𠀀/>seven"; String gold = "one\ntwo\nthree 瀛愯𩬅\nfour\nfive\nsix\nseven"; - Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test))); + Reader reader = new HTMLStripCharFilter(new StringReader(test)); int ch = 0; StringBuilder builder = new StringBuilder(); try { @@ -822,7 +821,7 @@ } } Reader reader = new HTMLStripCharFilter - (CharReader.get(new StringReader(text.toString()))); + (new StringReader(text.toString())); while (reader.read() != -1); } Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java =================================================================== --- lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java (revision 1362105) +++ lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java (working copy) @@ -24,7 +24,6 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.CharReader; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; @@ -327,7 +326,7 @@ @Override protected Reader initReader(String fieldName, Reader reader) { - return new MappingCharFilter(normMap, CharReader.get(reader)); + return new MappingCharFilter(normMap, reader); } }; Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java =================================================================== --- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java (revision 1362220) +++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java (working copy) @@ -68,7 +68,7 @@ @Override protected Reader initReader(String fieldName, Reader reader) { - return new MockCharFilter(CharReader.get(reader), 7); + return new MockCharFilter(reader, 7); } }; assertAnalyzesTo(a, "ab", Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java =================================================================== --- lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java (revision 1362220) +++ lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java (working copy) @@ -26,8 +26,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.CharReader; -import org.apache.lucene.analysis.CharStream; +import org.apache.lucene.analysis.CharFilter; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; @@ -102,8 +101,8 @@ private void checkOutput(String input, String pattern, String replacement, String expectedOutput, String expectedIndexMatchedOutput) throws IOException { - CharStream cs = new PatternReplaceCharFilter(pattern(pattern), replacement, - CharReader.get(new StringReader(input))); + CharFilter cs = new PatternReplaceCharFilter(pattern(pattern), replacement, + new StringReader(input)); StringBuilder output = new StringBuilder(); for (int chr = cs.read(); chr > 0; chr = cs.read()) { @@ -138,8 +137,8 @@ // this is test. public void testNothingChange() throws IOException { final String BLOCK = "this is test."; - CharStream cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1$2$3", - CharReader.get( new StringReader( BLOCK ) ) ); + CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1$2$3", + new StringReader( BLOCK ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[] { "this", "is", "test." }, @@ -152,8 +151,8 @@ // aa bb cc public void testReplaceByEmpty() throws IOException { final String BLOCK = "aa bb cc"; - CharStream cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "", - CharReader.get( new StringReader( BLOCK ) ) ); + CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "", + new StringReader( BLOCK ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[] {}); } @@ -163,8 +162,8 @@ // aa#bb#cc public void test1block1matchSameLength() throws IOException { final String BLOCK = "aa bb cc"; - CharStream cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1#$2#$3", - CharReader.get( new StringReader( BLOCK ) ) ); + CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1#$2#$3", + new StringReader( BLOCK ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[] { "aa#bb#cc" }, @@ -179,8 +178,8 @@ // aa##bb###cc dd public void test1block1matchLonger() throws IOException { final String BLOCK = "aa bb cc dd"; - CharStream cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1##$2###$3", - CharReader.get( new StringReader( BLOCK ) ) ); + CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1##$2###$3", + new StringReader( BLOCK ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[] { "aa##bb###cc", "dd" }, @@ -194,8 +193,8 @@ // aa aa public void test1block2matchLonger() throws IOException { final String BLOCK = " a a"; - CharStream cs = new PatternReplaceCharFilter( pattern("a"), "aa", - CharReader.get( new StringReader( BLOCK ) ) ); + CharFilter cs = new PatternReplaceCharFilter( pattern("a"), "aa", + new StringReader( BLOCK ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[] { "aa", "aa" }, @@ -210,8 +209,8 @@ // aa#bb dd public void test1block1matchShorter() throws IOException { final String BLOCK = "aa bb cc dd"; - CharStream cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1#$2", - CharReader.get( new StringReader( BLOCK ) ) ); + CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1#$2", + new StringReader( BLOCK ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[] { "aa#bb", "dd" }, @@ -226,8 +225,8 @@ // aa bb cc --- aa bb aa bb cc public void test1blockMultiMatches() throws IOException { final String BLOCK = " aa bb cc --- aa bb aa bb cc"; - CharStream cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1 $2 $3", - CharReader.get( new StringReader( BLOCK ) ) ); + CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1 $2 $3", + new StringReader( BLOCK ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[] { "aa", "bb", "cc", "---", "aa", "bb", "aa", "bb", "cc" }, @@ -246,8 +245,8 @@ public void test2blocksMultiMatches() throws IOException { final String BLOCK = " aa bb cc --- aa bb aa. bb aa bb cc"; - CharStream cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)"), "$1##$2", - CharReader.get( new StringReader( BLOCK ) ) ); + CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)"), "$1##$2", + new StringReader( BLOCK ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[] { "aa##bb", "cc", "---", "aa##bb", "aa.", "bb", "aa##bb", "cc" }, @@ -262,8 +261,8 @@ // aa b - c . --- b aa . c c b public void testChain() throws IOException { final String BLOCK = " a bb - ccc . --- bb a . ccc ccc bb"; - CharStream cs = new PatternReplaceCharFilter( pattern("a"), "aa", - CharReader.get( new StringReader( BLOCK ) ) ); + CharFilter cs = new PatternReplaceCharFilter( pattern("a"), "aa", + new StringReader( BLOCK ) ); cs = new PatternReplaceCharFilter( pattern("bb"), "b", cs ); cs = new PatternReplaceCharFilter( pattern("ccc"), "c", cs ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); @@ -315,7 +314,7 @@ @Override protected Reader initReader(String fieldName, Reader reader) { - return new PatternReplaceCharFilter(p, replacement, CharReader.get(reader)); + return new PatternReplaceCharFilter(p, replacement, reader); } }; Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java =================================================================== --- lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java (revision 1362220) +++ lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java (working copy) @@ -26,8 +26,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.CharReader; -import org.apache.lucene.analysis.CharStream; +import org.apache.lucene.analysis.CharFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.Analyzer.TokenStreamComponents; @@ -83,7 +82,7 @@ NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder(); builder.add("ü", "ü"); NormalizeCharMap normMap = builder.build(); - CharStream charStream = new MappingCharFilter( normMap, CharReader.get( new StringReader( INPUT ) ) ); + CharFilter charStream = new MappingCharFilter( normMap, new StringReader( INPUT ) ); // create PatternTokenizer TokenStream stream = new PatternTokenizer(charStream, Pattern.compile("[,;/\\s]+"), -1); @@ -93,7 +92,7 @@ new int[] { 12, 25, 28, 33 }, INPUT.length()); - charStream = new MappingCharFilter( normMap, CharReader.get( new StringReader( INPUT ) ) ); + charStream = new MappingCharFilter( normMap, new StringReader( INPUT ) ); stream = new PatternTokenizer(charStream, Pattern.compile("Günther"), 0); assertTokenStreamContents(stream, new String[] { "Günther", "Günther" }, Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java =================================================================== --- lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java (revision 1362105) +++ lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java (working copy) @@ -44,8 +44,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.CachingTokenFilter; -import org.apache.lucene.analysis.CharReader; -import org.apache.lucene.analysis.CharStream; +import org.apache.lucene.analysis.CharFilter; import org.apache.lucene.analysis.EmptyTokenizer; import org.apache.lucene.analysis.MockGraphTokenFilter; import org.apache.lucene.analysis.MockRandomLookaheadTokenFilter; @@ -101,7 +100,7 @@ static List> tokenizers; static List> tokenfilters; - static List> charfilters; + static List> charfilters; // TODO: fix those and remove private static final Set> brokenComponents = Collections.newSetFromMap(new IdentityHashMap,Boolean>()); @@ -170,7 +169,7 @@ getClassesForPackage("org.apache.lucene.analysis", analysisClasses); tokenizers = new ArrayList>(); tokenfilters = new ArrayList>(); - charfilters = new ArrayList>(); + charfilters = new ArrayList>(); for (final Class c : analysisClasses) { final int modifiers = c.getModifiers(); if ( @@ -179,7 +178,7 @@ || c.isSynthetic() || c.isAnonymousClass() || c.isMemberClass() || c.isInterface() || brokenComponents.contains(c) || c.isAnnotationPresent(Deprecated.class) - || !(Tokenizer.class.isAssignableFrom(c) || TokenFilter.class.isAssignableFrom(c) || CharStream.class.isAssignableFrom(c)) + || !(Tokenizer.class.isAssignableFrom(c) || TokenFilter.class.isAssignableFrom(c) || CharFilter.class.isAssignableFrom(c)) ) { continue; } @@ -197,10 +196,10 @@ assertTrue(ctor.toGenericString() + " has unsupported parameter types", allowedTokenFilterArgs.containsAll(Arrays.asList(ctor.getParameterTypes()))); tokenfilters.add(castConstructor(TokenFilter.class, ctor)); - } else if (CharStream.class.isAssignableFrom(c)) { + } else if (CharFilter.class.isAssignableFrom(c)) { assertTrue(ctor.toGenericString() + " has unsupported parameter types", allowedCharFilterArgs.containsAll(Arrays.asList(ctor.getParameterTypes()))); - charfilters.add(castConstructor(CharStream.class, ctor)); + charfilters.add(castConstructor(CharFilter.class, ctor)); } else { fail("Cannot get here"); } @@ -524,7 +523,6 @@ allowedCharFilterArgs = Collections.newSetFromMap(new IdentityHashMap,Boolean>()); allowedCharFilterArgs.addAll(argProducers.keySet()); allowedCharFilterArgs.add(Reader.class); - allowedCharFilterArgs.add(CharStream.class); } @SuppressWarnings("unchecked") @@ -560,8 +558,6 @@ Class paramType = paramTypes[i]; if (paramType == Reader.class) { args[i] = reader; - } else if (paramType == CharStream.class) { - args[i] = CharReader.get(reader); } else { args[i] = newRandomArg(random, paramType); } @@ -701,7 +697,7 @@ int numFilters = random.nextInt(3); for (int i = 0; i < numFilters; i++) { while (true) { - final Constructor ctor = charfilters.get(random.nextInt(charfilters.size())); + final Constructor ctor = charfilters.get(random.nextInt(charfilters.size())); final Object args[] = newCharFilterArgs(random, spec.reader, ctor.getParameterTypes()); reader = createComponent(ctor, args, descr); if (reader != null) { @@ -760,27 +756,19 @@ } } - // wants charfilter to be a filterreader... - // do *NOT*, do *NOT* refactor me to be a charfilter: LUCENE-3990 - static class CheckThatYouDidntReadAnythingReaderWrapper extends CharStream { + static class CheckThatYouDidntReadAnythingReaderWrapper extends CharFilter { boolean readSomething; - CharStream in; CheckThatYouDidntReadAnythingReaderWrapper(Reader in) { - this.in = CharReader.get(in); + super(in); } @Override - public int correctOffset(int currentOff) { - return in.correctOffset(currentOff); + public int correct(int currentOff) { + return currentOff; // we don't change any offsets } @Override - public void close() throws IOException { - in.close(); - } - - @Override public int read(char[] cbuf, int off, int len) throws IOException { readSomething = true; return in.read(cbuf, off, len); @@ -799,32 +787,12 @@ } @Override - public void mark(int readAheadLimit) throws IOException { - in.mark(readAheadLimit); - } - - @Override - public boolean markSupported() { - return in.markSupported(); - } - - @Override public int read(char[] cbuf) throws IOException { readSomething = true; return in.read(cbuf); } @Override - public boolean ready() throws IOException { - return in.ready(); - } - - @Override - public void reset() throws IOException { - in.reset(); - } - - @Override public long skip(long n) throws IOException { readSomething = true; return in.skip(n); Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestBugInSomething.java =================================================================== --- lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestBugInSomething.java (revision 1362105) +++ lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestBugInSomething.java (working copy) @@ -1,11 +1,12 @@ package org.apache.lucene.analysis.core; import java.io.Reader; +import java.io.StringReader; import java.nio.CharBuffer; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.CharStream; +import org.apache.lucene.analysis.CharFilter; import org.apache.lucene.analysis.MockCharFilter; import org.apache.lucene.analysis.MockTokenFilter; import org.apache.lucene.analysis.MockTokenizer; @@ -64,7 +65,7 @@ checkAnalysisConsistency(random(), a, false, "wmgddzunizdomqyj"); } - CharStream wrappedStream = new CharStream() { + CharFilter wrappedStream = new CharFilter(new StringReader("bogus")) { @Override public void mark(int readAheadLimit) { @@ -107,8 +108,8 @@ } @Override - public int correctOffset(int currentOff) { - throw new UnsupportedOperationException("correctOffset(int)"); + public int correct(int currentOff) { + throw new UnsupportedOperationException("correct(int)"); } @Override @@ -123,7 +124,7 @@ }; public void testWrapping() throws Exception { - CharStream cs = new TestRandomChains.CheckThatYouDidntReadAnythingReaderWrapper(wrappedStream); + CharFilter cs = new TestRandomChains.CheckThatYouDidntReadAnythingReaderWrapper(wrappedStream); try { cs.mark(1); fail(); @@ -177,7 +178,7 @@ cs.correctOffset(1); fail(); } catch (Exception e) { - assertEquals("correctOffset(int)", e.getMessage()); + assertEquals("correct(int)", e.getMessage()); } try { Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKAnalyzer.java =================================================================== --- lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKAnalyzer.java (revision 1362220) +++ lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKAnalyzer.java (working copy) @@ -23,7 +23,6 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.CharReader; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; @@ -216,7 +215,7 @@ @Override protected Reader initReader(String fieldName, Reader reader) { - return new MappingCharFilter(norm, CharReader.get(reader)); + return new MappingCharFilter(norm, reader); } }; Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java =================================================================== --- lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java (revision 1362220) +++ lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java (working copy) @@ -23,7 +23,6 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.CharStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.charfilter.MappingCharFilter; import org.apache.lucene.analysis.charfilter.NormalizeCharMap; @@ -123,7 +122,7 @@ builder.add("\\", "/"); NormalizeCharMap normMap = builder.build(); String path = "c:\\a\\b\\c"; - CharStream cs = new MappingCharFilter(normMap, new StringReader(path)); + Reader cs = new MappingCharFilter(normMap, new StringReader(path)); PathHierarchyTokenizer t = new PathHierarchyTokenizer( cs ); assertTokenStreamContents(t, new String[]{"c:", "c:/a", "c:/a/b", "c:/a/b/c"}, Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilter.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilter.java (revision 1362220) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilter.java (working copy) @@ -23,7 +23,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.lucene.analysis.CharStream; import org.apache.lucene.analysis.charfilter.BaseCharFilter; /** @@ -54,7 +53,7 @@ private final String replacement; private Reader transformedInput; - public PatternReplaceCharFilter(Pattern pattern, String replacement, CharStream in) { + public PatternReplaceCharFilter(Pattern pattern, String replacement, Reader in) { super(in); this.pattern = pattern; this.replacement = replacement; @@ -64,18 +63,31 @@ public int read(char[] cbuf, int off, int len) throws IOException { // Buffer all input on the first call. if (transformedInput == null) { - StringBuilder buffered = new StringBuilder(); - char [] temp = new char [1024]; - for (int cnt = input.read(temp); cnt > 0; cnt = input.read(temp)) { - buffered.append(temp, 0, cnt); - } - transformedInput = new StringReader(processPattern(buffered).toString()); + fill(); } return transformedInput.read(cbuf, off, len); } + + private void fill() throws IOException { + StringBuilder buffered = new StringBuilder(); + char [] temp = new char [1024]; + for (int cnt = in.read(temp); cnt > 0; cnt = in.read(temp)) { + buffered.append(temp, 0, cnt); + } + transformedInput = new StringReader(processPattern(buffered).toString()); + } @Override + public int read() throws IOException { + if (transformedInput == null) { + fill(); + } + + return transformedInput.read(); + } + + @Override protected int correct(int currentOff) { return Math.max(0, super.correct(currentOff)); } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilter.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilter.java (revision 1362220) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilter.java (working copy) @@ -18,9 +18,9 @@ */ import java.io.IOException; +import java.io.Reader; -import org.apache.lucene.analysis.CharStream; -import org.apache.lucene.analysis.charfilter.CharFilter; +import org.apache.lucene.analysis.CharFilter; /** * CharFilter that replaces instances of Zero-width non-joiner with an @@ -28,7 +28,7 @@ */ public class PersianCharFilter extends CharFilter { - public PersianCharFilter(CharStream in) { + public PersianCharFilter(Reader in) { super(in); } @@ -45,4 +45,9 @@ } return charsRead; } + + @Override + protected int correct(int currentOff) { + return currentOff; // we don't change the length of the string + } } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java (revision 1362220) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java (working copy) @@ -21,7 +21,6 @@ import java.io.Reader; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.CharReader; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.ar.ArabicNormalizationFilter; @@ -134,6 +133,6 @@ */ @Override protected Reader initReader(String fieldName, Reader reader) { - return new PersianCharFilter(CharReader.get(reader)); + return new PersianCharFilter(reader); } } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java (revision 1362220) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java (working copy) @@ -114,9 +114,9 @@ tok = new StopFilter(matchVersion, tok, stopwords); return new TokenStreamComponents(src, tok) { @Override - protected void reset(final Reader reader) throws IOException { + protected void setReader(final Reader reader) throws IOException { src.setMaxTokenLength(ClassicAnalyzer.this.maxTokenLength); - super.reset(reader); + super.setReader(reader); } }; } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java (revision 1362220) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java (working copy) @@ -115,9 +115,9 @@ tok = new StopFilter(matchVersion, tok, stopwords); return new TokenStreamComponents(src, tok) { @Override - protected void reset(final Reader reader) throws IOException { + protected void setReader(final Reader reader) throws IOException { src.setMaxTokenLength(StandardAnalyzer.this.maxTokenLength); - super.reset(reader); + super.setReader(reader); } }; } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java (revision 1362220) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java (working copy) @@ -104,9 +104,9 @@ tok = new StopFilter(matchVersion, tok, stopwords); return new TokenStreamComponents(src, tok) { @Override - protected void reset(final Reader reader) throws IOException { + protected void setReader(final Reader reader) throws IOException { src.setMaxTokenLength(UAX29URLEmailAnalyzer.this.maxTokenLength); - super.reset(reader); + super.setReader(reader); } }; } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharFilterFactory.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharFilterFactory.java (revision 1362220) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharFilterFactory.java (working copy) @@ -17,13 +17,15 @@ * limitations under the License. */ -import org.apache.lucene.analysis.CharStream; +import java.io.Reader; +import org.apache.lucene.analysis.CharFilter; + /** - * Abstract parent class for analysis factories that create {@link CharStream} + * Abstract parent class for analysis factories that create {@link CharFilter} * instances. */ public abstract class CharFilterFactory extends AbstractAnalysisFactory { - public abstract CharStream create(CharStream input); + public abstract CharFilter create(Reader input); } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex (revision 1362220) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex (working copy) @@ -18,13 +18,13 @@ */ import java.io.IOException; +import java.io.Reader; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Set; import org.apache.lucene.util.Version; -import org.apache.lucene.analysis.CharStream; import org.apache.lucene.analysis.util.CharArrayMap; import org.apache.lucene.analysis.util.CharArraySet; import org.apache.lucene.analysis.util.OpenStringBuilder; @@ -173,7 +173,7 @@ /** * @param source */ - public HTMLStripCharFilter(CharStream source) { + public HTMLStripCharFilter(Reader source) { super(source); this.zzReader = source; } @@ -183,7 +183,7 @@ * @param escapedTags Tags in this set (both start and end tags) * will not be filtered out. */ - public HTMLStripCharFilter(CharStream source, Set escapedTags) { + public HTMLStripCharFilter(Reader source, Set escapedTags) { super(source); this.zzReader = source; if (null != escapedTags) { Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/CharFilter.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/CharFilter.java (revision 1362105) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/CharFilter.java (working copy) @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.lucene.analysis.charfilter; - -import java.io.IOException; - -import org.apache.lucene.analysis.CharStream; -import org.apache.lucene.analysis.Tokenizer; - -/** - * Subclasses of CharFilter can be chained to filter CharStream. - * They can be used as {@link java.io.Reader} with additional offset - * correction. {@link Tokenizer}s will automatically use {@link #correctOffset} - * if a CharFilter/CharStream subclass is used. - */ -public abstract class CharFilter extends CharStream { - - protected CharStream input; - - protected CharFilter(CharStream in) { - input = in; - } - - /** - * Subclass may want to override to correct the current offset. - * - * @param currentOff current offset - * @return corrected offset - */ - protected int correct(int currentOff) { - return currentOff; - } - - /** - * Chains the corrected offset through the input - * CharFilter. - */ - @Override - public final int correctOffset(int currentOff) { - return input.correctOffset(correct(currentOff)); - } - - @Override - public void close() throws IOException { - input.close(); - } - - @Override - public int read(char[] cbuf, int off, int len) throws IOException { - return input.read(cbuf, off, len); - } - - @Override - public boolean markSupported(){ - return input.markSupported(); - } - - @Override - public void mark( int readAheadLimit ) throws IOException { - input.mark(readAheadLimit); - } - - @Override - public void reset() throws IOException { - input.reset(); - } -} Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java (revision 1362220) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java (working copy) @@ -1,4 +1,4 @@ -/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 5/18/12 12:24 PM */ +/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 7/16/12 4:05 PM */ package org.apache.lucene.analysis.charfilter; @@ -20,13 +20,13 @@ */ import java.io.IOException; +import java.io.Reader; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Set; import org.apache.lucene.util.Version; -import org.apache.lucene.analysis.CharStream; import org.apache.lucene.analysis.util.CharArrayMap; import org.apache.lucene.analysis.util.CharArraySet; import org.apache.lucene.analysis.util.OpenStringBuilder; @@ -40,8 +40,8 @@ /** * This class is a scanner generated by * JFlex 1.5.0-SNAPSHOT - * on 5/18/12 12:24 PM from the specification file - * C:/svn/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex + * on 7/16/12 4:05 PM from the specification file + * /home/rmuir/workspace/lucene-trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex */ public final class HTMLStripCharFilter extends BaseCharFilter { @@ -30647,7 +30647,7 @@ /** * @param source */ - public HTMLStripCharFilter(CharStream source) { + public HTMLStripCharFilter(Reader source) { super(source); this.zzReader = source; } @@ -30657,7 +30657,7 @@ * @param escapedTags Tags in this set (both start and end tags) * will not be filtered out. */ - public HTMLStripCharFilter(CharStream source, Set escapedTags) { + public HTMLStripCharFilter(Reader source, Set escapedTags) { super(source); this.zzReader = source; if (null != escapedTags) { Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilter.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilter.java (revision 1362220) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilter.java (working copy) @@ -21,8 +21,7 @@ import java.io.Reader; import java.util.Map; -import org.apache.lucene.analysis.CharReader; -import org.apache.lucene.analysis.CharStream; +import org.apache.lucene.analysis.CharFilter; // javadocs import org.apache.lucene.util.CharsRef; import org.apache.lucene.util.RollingCharBuffer; import org.apache.lucene.util.fst.CharSequenceOutputs; @@ -51,8 +50,8 @@ private int replacementPointer; private int inputOff; - /** Default constructor that takes a {@link CharStream}. */ - public MappingCharFilter(NormalizeCharMap normMap, CharStream in) { + /** Default constructor that takes a {@link Reader}. */ + public MappingCharFilter(NormalizeCharMap normMap, Reader in) { super(in); buffer.reset(in); @@ -66,15 +65,10 @@ } } - /** Easy-use constructor that takes a {@link Reader}. */ - public MappingCharFilter(NormalizeCharMap normMap, Reader in) { - this(normMap, CharReader.get(in)); - } - @Override public void reset() throws IOException { super.reset(); - buffer.reset(input); + buffer.reset(in); replacement = null; inputOff = 0; } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/BaseCharFilter.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/BaseCharFilter.java (revision 1362220) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/BaseCharFilter.java (working copy) @@ -17,9 +17,10 @@ package org.apache.lucene.analysis.charfilter; -import org.apache.lucene.analysis.CharStream; +import org.apache.lucene.analysis.CharFilter; import org.apache.lucene.util.ArrayUtil; +import java.io.Reader; import java.util.Arrays; /** @@ -34,7 +35,7 @@ private int diffs[]; private int size = 0; - public BaseCharFilter(CharStream in) { + public BaseCharFilter(Reader in) { super(in); } Index: lucene/analysis/common/src/java/overview.html =================================================================== --- lucene/analysis/common/src/java/overview.html (revision 1362105) +++ lucene/analysis/common/src/java/overview.html (working copy) @@ -24,7 +24,7 @@ For an introduction to Lucene's analysis API, see the {@link org.apache.lucene.analysis} package documentation.

- This module contains concrete components ({@link org.apache.lucene.analysis.charfilter.CharFilter}s, + This module contains concrete components ({@link org.apache.lucene.analysis.CharFilter}s, {@link org.apache.lucene.analysis.Tokenizer}s, and ({@link org.apache.lucene.analysis.TokenFilter}s) for analyzing different types of content. It also provides a number of {@link org.apache.lucene.analysis.Analyzer}s for different languages that you can use to get started quickly. Index: lucene/test-framework/src/java/org/apache/lucene/analysis/MockCharFilter.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/analysis/MockCharFilter.java (revision 1362220) +++ lucene/test-framework/src/java/org/apache/lucene/analysis/MockCharFilter.java (working copy) @@ -24,13 +24,12 @@ /** the purpose of this charfilter is to send offsets out of bounds if the analyzer doesn't use correctOffset or does incorrect offset math. */ -public class MockCharFilter extends CharStream { - final CharStream in; +public class MockCharFilter extends CharFilter { final int remainder; // for testing only public MockCharFilter(Reader in, int remainder) { - this.in = CharReader.get(in); + super(in); // TODO: instead of fixed remainder... maybe a fixed // random seed? this.remainder = remainder; @@ -94,11 +93,11 @@ } @Override - public int correctOffset(int currentOff) { + public int correct(int currentOff) { SortedMap subMap = corrections.subMap(0, currentOff+1); int ret = subMap.isEmpty() ? currentOff : currentOff + subMap.get(subMap.lastKey()); assert ret >= 0 : "currentOff=" + currentOff + ",diff=" + (ret-currentOff); - return in.correctOffset(ret); // chain the call + return ret; } protected void addOffCorrectMap(int off, int cumulativeDiff) { Index: lucene/core/src/test/org/apache/lucene/analysis/TestCharFilter.java =================================================================== --- lucene/core/src/test/org/apache/lucene/analysis/TestCharFilter.java (revision 0) +++ lucene/core/src/test/org/apache/lucene/analysis/TestCharFilter.java (working copy) @@ -0,0 +1,70 @@ +package org.apache.lucene.analysis; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.Reader; +import java.io.StringReader; + +import org.apache.lucene.util.LuceneTestCase; + +public class TestCharFilter extends LuceneTestCase { + + public void testCharFilter1() throws Exception { + CharFilter cs = new CharFilter1(new StringReader("")); + assertEquals("corrected offset is invalid", 1, cs.correctOffset(0)); + } + + public void testCharFilter2() throws Exception { + CharFilter cs = new CharFilter2(new StringReader("")); + assertEquals("corrected offset is invalid", 2, cs.correctOffset(0)); + } + + public void testCharFilter12() throws Exception { + CharFilter cs = new CharFilter2(new CharFilter1(new StringReader(""))); + assertEquals( "corrected offset is invalid", 3, cs.correctOffset(0)); + } + + public void testCharFilter11() throws Exception { + CharFilter cs = new CharFilter1(new CharFilter1(new StringReader(""))); + assertEquals( "corrected offset is invalid", 2, cs.correctOffset(0)); + } + + static class CharFilter1 extends CharFilter { + + protected CharFilter1(Reader in) { + super(in); + } + + @Override + protected int correct(int currentOff) { + return currentOff + 1; + } + } + + static class CharFilter2 extends CharFilter { + + protected CharFilter2(Reader in) { + super(in); + } + + @Override + protected int correct(int currentOff) { + return currentOff + 2; + } + } +} Property changes on: lucene/core/src/test/org/apache/lucene/analysis/TestCharFilter.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native Index: lucene/core/src/test/org/apache/lucene/analysis/TestMockCharFilter.java =================================================================== --- lucene/core/src/test/org/apache/lucene/analysis/TestMockCharFilter.java (revision 1362220) +++ lucene/core/src/test/org/apache/lucene/analysis/TestMockCharFilter.java (working copy) @@ -33,7 +33,7 @@ @Override protected Reader initReader(String fieldName, Reader reader) { - return new MockCharFilter(CharReader.get(reader), 7); + return new MockCharFilter(reader, 7); } }; Index: lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java =================================================================== --- lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (revision 1362220) +++ lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (working copy) @@ -123,7 +123,7 @@ for (int i = 0; i < num; i++) { String s = _TestUtil.randomHtmlishString(random(), 20); StringReader reader = new StringReader(s); - MockCharFilter charfilter = new MockCharFilter(CharReader.get(reader), 2); + MockCharFilter charfilter = new MockCharFilter(reader, 2); MockAnalyzer analyzer = new MockAnalyzer(random()); TokenStream ts = analyzer.tokenStream("bogus", charfilter); ts.reset(); Index: lucene/core/src/java/org/apache/lucene/analysis/CharFilter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/analysis/CharFilter.java (revision 0) +++ lucene/core/src/java/org/apache/lucene/analysis/CharFilter.java (working copy) @@ -0,0 +1,55 @@ +package org.apache.lucene.analysis; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.FilterReader; +import java.io.Reader; + +/** + * Subclasses of CharFilter can be chained to filter a Reader + * They can be used as {@link java.io.Reader} with additional offset + * correction. {@link Tokenizer}s will automatically use {@link #correctOffset} + * if a CharFilter subclass is used. + */ +public abstract class CharFilter extends FilterReader { + + /** + * Create a new CharFilter wrapping the provided reader. + * @param in a Reader, can also be a CharFilter for chaining. + */ + public CharFilter(Reader in) { + super(in); + } + + /** + * Subclasses override to correct the current offset. + * + * @param currentOff current offset + * @return corrected offset + */ + protected abstract int correct(int currentOff); + + /** + * Chains the corrected offset through the input + * CharFilter(s). + */ + public final int correctOffset(int currentOff) { + final int corrected = correct(currentOff); + return (in instanceof CharFilter) ? ((CharFilter) in).correctOffset(corrected) : corrected; + } +} Property changes on: lucene/core/src/java/org/apache/lucene/analysis/CharFilter.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native Index: lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java (revision 1362220) +++ lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java (working copy) @@ -69,7 +69,7 @@ * instance of {@link TokenStreamComponents}. It returns the sink of the * components and stores the components internally. Subsequent calls to this * method will reuse the previously stored components after resetting them - * through {@link TokenStreamComponents#reset(Reader)}. + * through {@link TokenStreamComponents#setReader(Reader)}. *

* * @param fieldName the name of the field the created TokenStream is used for @@ -83,7 +83,7 @@ components = createComponents(fieldName, r); reuseStrategy.setReusableComponents(fieldName, components); } else { - components.reset(r); + components.setReader(r); } return components.getTokenStream(); } @@ -181,7 +181,7 @@ * @throws IOException * if the component's reset method throws an {@link IOException} */ - protected void reset(final Reader reader) throws IOException { + protected void setReader(final Reader reader) throws IOException { source.setReader(reader); } Index: lucene/core/src/java/org/apache/lucene/analysis/CharStream.java =================================================================== --- lucene/core/src/java/org/apache/lucene/analysis/CharStream.java (revision 1362105) +++ lucene/core/src/java/org/apache/lucene/analysis/CharStream.java (working copy) @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.lucene.analysis; - -import java.io.Reader; - -/** - * CharStream adds {@link #correctOffset} - * functionality over {@link Reader}. All Tokenizers accept a - * CharStream instead of {@link Reader} as input, which enables - * arbitrary character based filtering before tokenization. - * The {@link #correctOffset} method fixed offsets to account for - * removal or insertion of characters, so that the offsets - * reported in the tokens match the character offsets of the - * original Reader. - */ -public abstract class CharStream extends Reader { - - /** - * Called by CharFilter(s) and Tokenizer to correct token offset. - * - * @param currentOff offset as seen in the output - * @return corrected offset based on the input - */ - public abstract int correctOffset(int currentOff); -} Index: lucene/core/src/java/org/apache/lucene/analysis/CharReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/analysis/CharReader.java (revision 1362105) +++ lucene/core/src/java/org/apache/lucene/analysis/CharReader.java (working copy) @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.lucene.analysis; - -import java.io.IOException; -import java.io.Reader; - -/** - * CharReader is a Reader wrapper. It reads chars from - * Reader and outputs {@link CharStream}, defining an - * identify function {@link #correctOffset} method that - * simply returns the provided offset. - */ -public final class CharReader extends CharStream { - - private final Reader input; - - public static CharStream get(Reader input) { - return input instanceof CharStream ? - (CharStream)input : new CharReader(input); - } - - private CharReader(Reader in) { - input = in; - } - - @Override - public int correctOffset(int currentOff) { - return currentOff; - } - - @Override - public void close() throws IOException { - input.close(); - } - - @Override - public int read(char[] cbuf, int off, int len) throws IOException { - return input.read(cbuf, off, len); - } - - @Override - public int read() throws IOException { - return input.read(); - } - - @Override - public boolean markSupported(){ - return input.markSupported(); - } - - @Override - public void mark( int readAheadLimit ) throws IOException { - input.mark(readAheadLimit); - } - - @Override - public void reset() throws IOException { - input.reset(); - } -} Index: lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java (revision 1362220) +++ lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java (working copy) @@ -65,15 +65,15 @@ } } - /** Return the corrected offset. If {@link #input} is a {@link CharStream} subclass - * this method calls {@link CharStream#correctOffset}, else returns currentOff. + /** Return the corrected offset. If {@link #input} is a {@link CharFilter} subclass + * this method calls {@link CharFilter#correctOffset}, else returns currentOff. * @param currentOff offset as seen in the output * @return corrected offset based on the input - * @see CharStream#correctOffset + * @see CharFilter#correctOffset */ protected final int correctOffset(int currentOff) { assert input != null: "this tokenizer is closed"; - return (input instanceof CharStream) ? ((CharStream) input).correctOffset(currentOff) : currentOff; + return (input instanceof CharFilter) ? ((CharFilter) input).correctOffset(currentOff) : currentOff; } /** Expert: Reset the tokenizer to a new reader. Typically, an Index: solr/test-framework/src/java/org/apache/solr/analysis/MockCharFilterFactory.java =================================================================== --- solr/test-framework/src/java/org/apache/solr/analysis/MockCharFilterFactory.java (revision 1362105) +++ solr/test-framework/src/java/org/apache/solr/analysis/MockCharFilterFactory.java (working copy) @@ -17,9 +17,10 @@ * limitations under the License. */ +import java.io.Reader; import java.util.Map; -import org.apache.lucene.analysis.CharStream; +import org.apache.lucene.analysis.CharFilter; import org.apache.lucene.analysis.MockCharFilter; import org.apache.lucene.analysis.util.CharFilterFactory; @@ -40,7 +41,7 @@ } @Override - public CharStream create(CharStream input) { + public CharFilter create(Reader input) { return new MockCharFilter(input, remainder); } } Index: solr/core/src/test/org/apache/solr/analysis/TestArabicFilters.java =================================================================== --- solr/core/src/test/org/apache/solr/analysis/TestArabicFilters.java (revision 1362105) +++ solr/core/src/test/org/apache/solr/analysis/TestArabicFilters.java (working copy) @@ -23,7 +23,6 @@ import java.util.Map; import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.CharReader; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; @@ -78,7 +77,7 @@ tokenizerFactory.setLuceneMatchVersion(TEST_VERSION_CURRENT); Map args = Collections.emptyMap(); tokenizerFactory.init(args); - TokenStream stream = tokenizerFactory.create(charfilterFactory.create(CharReader.get(reader))); + TokenStream stream = tokenizerFactory.create(charfilterFactory.create(reader)); assertTokenStreamContents(stream, new String[] { "می", "خورد" }); } } Index: solr/core/src/test/org/apache/solr/analysis/TestJapaneseIterationMarkCharFilterFactory.java =================================================================== --- solr/core/src/test/org/apache/solr/analysis/TestJapaneseIterationMarkCharFilterFactory.java (revision 1362105) +++ solr/core/src/test/org/apache/solr/analysis/TestJapaneseIterationMarkCharFilterFactory.java (working copy) @@ -18,8 +18,7 @@ */ import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.CharReader; -import org.apache.lucene.analysis.CharStream; +import org.apache.lucene.analysis.CharFilter; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenStream; import org.apache.solr.core.SolrResourceLoader; @@ -38,7 +37,7 @@ public void testIterationMarksWithKeywordTokenizer() throws IOException { final String text = "時々馬鹿々々しいところゞゝゝミスヾ"; JapaneseIterationMarkCharFilterFactory filterFactory = new JapaneseIterationMarkCharFilterFactory(); - CharStream filter = filterFactory.create(CharReader.get(new StringReader(text))); + CharFilter filter = filterFactory.create(new StringReader(text)); TokenStream tokenStream = new MockTokenizer(filter, MockTokenizer.KEYWORD, false); assertTokenStreamContents(tokenStream, new String[]{"時時馬鹿馬鹿しいところどころミスズ"}); } @@ -53,8 +52,8 @@ Map filterArgs = Collections.emptyMap(); filterFactory.init(filterArgs); - CharStream filter = filterFactory.create( - CharReader.get(new StringReader("時々馬鹿々々しいところゞゝゝミスヾ")) + CharFilter filter = filterFactory.create( + new StringReader("時々馬鹿々々しいところゞゝゝミスヾ") ); TokenStream tokenStream = tokenizerFactory.create(filter); assertTokenStreamContents(tokenStream, new String[]{"時時", "馬鹿馬鹿しい", "ところどころ", "ミ", "スズ"}); @@ -72,8 +71,8 @@ filterArgs.put("normalizeKana", "false"); filterFactory.init(filterArgs); - CharStream filter = filterFactory.create( - CharReader.get(new StringReader("時々馬鹿々々しいところゞゝゝミスヾ")) + CharFilter filter = filterFactory.create( + new StringReader("時々馬鹿々々しいところゞゝゝミスヾ") ); TokenStream tokenStream = tokenizerFactory.create(filter); assertTokenStreamContents(tokenStream, new String[]{"時時", "馬鹿馬鹿しい", "ところ", "ゞ", "ゝ", "ゝ", "ミス", "ヾ"}); @@ -91,8 +90,8 @@ filterArgs.put("normalizeKana", "true"); filterFactory.init(filterArgs); - CharStream filter = filterFactory.create( - CharReader.get(new StringReader("時々馬鹿々々しいところゞゝゝミスヾ")) + CharFilter filter = filterFactory.create( + new StringReader("時々馬鹿々々しいところゞゝゝミスヾ") ); TokenStream tokenStream = tokenizerFactory.create(filter); assertTokenStreamContents(tokenStream, new String[]{"時々", "馬鹿", "々", "々", "しい", "ところどころ", "ミ", "スズ"}); Index: solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceCharFilterFactory.java =================================================================== --- solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceCharFilterFactory.java (revision 1362105) +++ solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceCharFilterFactory.java (working copy) @@ -39,8 +39,8 @@ args.put("pattern", "(aa)\\s+(bb)\\s+(cc)"); args.put("replacement", "$1$2$3"); factory.init(args); - CharStream cs = factory.create( - CharReader.get( new StringReader( BLOCK ) ) ); + CharFilter cs = factory.create( + new StringReader( BLOCK ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[] { "this", "is", "test." }, @@ -56,8 +56,8 @@ Map args = new HashMap(); args.put("pattern", "(aa)\\s+(bb)\\s+(cc)"); factory.init(args); - CharStream cs = factory.create( - CharReader.get( new StringReader( BLOCK ) ) ); + CharFilter cs = factory.create( + new StringReader( BLOCK ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); ts.reset(); assertFalse(ts.incrementToken()); @@ -75,8 +75,8 @@ args.put("pattern", "(aa)\\s+(bb)\\s+(cc)"); args.put("replacement", "$1#$2#$3"); factory.init(args); - CharStream cs = factory.create( - CharReader.get( new StringReader( BLOCK ) ) ); + CharFilter cs = factory.create( + new StringReader( BLOCK ) ); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[] { "aa#bb#cc" }, Index: solr/core/src/test/org/apache/solr/analysis/TestHTMLStripCharFilterFactory.java =================================================================== --- solr/core/src/test/org/apache/solr/analysis/TestHTMLStripCharFilterFactory.java (revision 1362105) +++ solr/core/src/test/org/apache/solr/analysis/TestHTMLStripCharFilterFactory.java (working copy) @@ -38,7 +38,7 @@ Map args = new HashMap(); args.put("escapedTags", "a, Title"); factory.init(args); - CharStream cs = factory.create(CharReader.get(new StringReader(text))); + CharFilter cs = factory.create(new StringReader(text)); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[] { "this", "is", "only", "a", "test." }, @@ -53,7 +53,7 @@ HTMLStripCharFilterFactory factory = new HTMLStripCharFilterFactory(); Map args = new HashMap(); factory.init(args); - CharStream cs = factory.create(CharReader.get(new StringReader(text))); + CharFilter cs = factory.create(new StringReader(text)); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[] { "this", "is", "only", "a", "test." }, @@ -69,7 +69,7 @@ Map args = new HashMap(); args.put("escapedTags", "U i"); factory.init(args); - CharStream cs = factory.create(CharReader.get(new StringReader(text))); + CharFilter cs = factory.create(new StringReader(text)); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[] { "this", "is", "only", "a", "test." }, @@ -85,7 +85,7 @@ Map args = new HashMap(); args.put("escapedTags", ",, , "); factory.init(args); - CharStream cs = factory.create(CharReader.get(new StringReader(text))); + CharFilter cs = factory.create(new StringReader(text)); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[] { "this", "is", "only", "a", "test." }, @@ -101,7 +101,7 @@ Map args = new HashMap(); args.put("escapedTags", ""); factory.init(args); - CharStream cs = factory.create(CharReader.get(new StringReader(text))); + CharFilter cs = factory.create(new StringReader(text)); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[] { "this", "is", "only", "a", "test." }, @@ -117,7 +117,7 @@ Map args = new HashMap(); args.put("escapedTags", ", B\r\n\t"); factory.init(args); - CharStream cs = factory.create(CharReader.get(new StringReader(text))); + CharFilter cs = factory.create(new StringReader(text)); TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false); assertTokenStreamContents(ts, new String[] { "this", "is", "only", "a", "test." }, Index: solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java =================================================================== --- solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java (revision 1362105) +++ solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java (working copy) @@ -28,7 +28,6 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.CharReader; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.util._TestUtil; @@ -45,7 +44,7 @@ String gold = " this is some text here is a link and " + "another link . " + "This is an entity: & plus a <. Here is an &. "; - LegacyHTMLStripCharFilter reader = new LegacyHTMLStripCharFilter(CharReader.get(new StringReader(html))); + LegacyHTMLStripCharFilter reader = new LegacyHTMLStripCharFilter(new StringReader(html)); StringBuilder builder = new StringBuilder(); int ch = -1; char [] goldArray = gold.toCharArray(); @@ -63,7 +62,7 @@ //Some sanity checks, but not a full-fledged check public void testHTML() throws Exception { InputStream stream = getClass().getResourceAsStream("htmlStripReaderTest.html"); - LegacyHTMLStripCharFilter reader = new LegacyHTMLStripCharFilter(CharReader.get(new InputStreamReader(stream, "UTF-8"))); + LegacyHTMLStripCharFilter reader = new LegacyHTMLStripCharFilter(new InputStreamReader(stream, "UTF-8")); StringBuilder builder = new StringBuilder(); int ch = -1; while ((ch = reader.read()) != -1){ @@ -83,7 +82,7 @@ String gold = "\u0393"; Set set = new HashSet(); set.add("reserved"); - Reader reader = new LegacyHTMLStripCharFilter(CharReader.get(new StringReader(test)), set); + Reader reader = new LegacyHTMLStripCharFilter(new StringReader(test), set); StringBuilder builder = new StringBuilder(); int ch = 0; while ((ch = reader.read()) != -1){ @@ -100,7 +99,7 @@ String gold = " \u00DCbermensch = \u0393 bar \u0393"; Set set = new HashSet(); set.add("reserved"); - Reader reader = new LegacyHTMLStripCharFilter(CharReader.get(new StringReader(test)), set); + Reader reader = new LegacyHTMLStripCharFilter(new StringReader(test), set); StringBuilder builder = new StringBuilder(); int ch = 0; while ((ch = reader.read()) != -1){ @@ -117,7 +116,7 @@ String gold = " ! @ and ’"; Set set = new HashSet(); set.add("reserved"); - Reader reader = new LegacyHTMLStripCharFilter(CharReader.get(new StringReader(test)), set); + Reader reader = new LegacyHTMLStripCharFilter(new StringReader(test), set); StringBuilder builder = new StringBuilder(); int ch = 0; while ((ch = reader.read()) != -1){ @@ -133,7 +132,7 @@ String test = "aaa bbb eeee ffff "; Set set = new HashSet(); set.add("reserved"); - Reader reader = new LegacyHTMLStripCharFilter(CharReader.get(new StringReader(test)), set); + Reader reader = new LegacyHTMLStripCharFilter(new StringReader(test), set); StringBuilder builder = new StringBuilder(); int ch = 0; while ((ch = reader.read()) != -1){ @@ -150,7 +149,7 @@ public void testMalformedHTML() throws Exception { String test = "a > "; String gold = "a out, CharStream input ){ + private String writeCharStream(NamedList out, Reader input ){ final int BUFFER_SIZE = 1024; char[] buf = new char[BUFFER_SIZE]; int len = 0; Index: solr/core/src/java/org/apache/solr/update/processor/HTMLStripFieldUpdateProcessorFactory.java =================================================================== --- solr/core/src/java/org/apache/solr/update/processor/HTMLStripFieldUpdateProcessorFactory.java (revision 1362105) +++ solr/core/src/java/org/apache/solr/update/processor/HTMLStripFieldUpdateProcessorFactory.java (working copy) @@ -21,7 +21,6 @@ import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; -import org.apache.lucene.analysis.CharReader; import org.apache.lucene.analysis.charfilter.HTMLStripCharFilter; import org.apache.commons.io.IOUtils; @@ -69,7 +68,7 @@ Reader in = null; try { in = new HTMLStripCharFilter - (CharReader.get(new StringReader(s.toString()))); + (new StringReader(s.toString())); IOUtils.copy(in, result); return result.toString(); } catch (IOException e) { Index: solr/core/src/java/org/apache/solr/analysis/JapaneseIterationMarkCharFilterFactory.java =================================================================== --- solr/core/src/java/org/apache/solr/analysis/JapaneseIterationMarkCharFilterFactory.java (revision 1362105) +++ solr/core/src/java/org/apache/solr/analysis/JapaneseIterationMarkCharFilterFactory.java (working copy) @@ -17,12 +17,13 @@ * limitations under the License. */ -import org.apache.lucene.analysis.CharStream; +import org.apache.lucene.analysis.CharFilter; import org.apache.lucene.analysis.ja.JapaneseIterationMarkCharFilter; import org.apache.lucene.analysis.util.AbstractAnalysisFactory; import org.apache.lucene.analysis.util.CharFilterFactory; import org.apache.lucene.analysis.util.MultiTermAwareComponent; +import java.io.Reader; import java.util.Map; /** @@ -46,7 +47,7 @@ private boolean normalizeKana = true; @Override - public CharStream create(CharStream input) { + public CharFilter create(Reader input) { return new JapaneseIterationMarkCharFilter(input, normalizeKanji, normalizeKana); } Index: solr/core/src/java/org/apache/solr/analysis/HTMLStripCharFilterFactory.java =================================================================== --- solr/core/src/java/org/apache/solr/analysis/HTMLStripCharFilterFactory.java (revision 1362105) +++ solr/core/src/java/org/apache/solr/analysis/HTMLStripCharFilterFactory.java (working copy) @@ -18,10 +18,10 @@ * limitations under the License. */ -import org.apache.lucene.analysis.CharStream; import org.apache.lucene.analysis.charfilter.HTMLStripCharFilter; import org.apache.lucene.analysis.util.CharFilterFactory; +import java.io.Reader; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -44,7 +44,7 @@ Set escapedTags = null; Pattern TAG_NAME_PATTERN = Pattern.compile("[^\\s,]+"); - public HTMLStripCharFilter create(CharStream input) { + public HTMLStripCharFilter create(Reader input) { HTMLStripCharFilter charFilter; if (null == escapedTags) { charFilter = new HTMLStripCharFilter(input); Index: solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java =================================================================== --- solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java (revision 1362105) +++ solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java (working copy) @@ -50,7 +50,7 @@ @Override public Reader initReader(String fieldName, Reader reader) { if (charFilters != null && charFilters.length > 0) { - CharStream cs = CharReader.get( reader ); + Reader cs = reader; for (CharFilterFactory charFilter : charFilters) { cs = charFilter.create(cs); } Index: solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilter.java =================================================================== --- solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilter.java (revision 1362105) +++ solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilter.java (working copy) @@ -26,8 +26,6 @@ import java.util.Set; import org.apache.lucene.analysis.charfilter.BaseCharFilter; -import org.apache.lucene.analysis.CharReader; -import org.apache.lucene.analysis.CharStream; /** *

@@ -72,21 +70,21 @@ public static void main(String[] args) throws IOException { Reader in = new LegacyHTMLStripCharFilter( - CharReader.get(new InputStreamReader(System.in, Charset.defaultCharset()))); + new InputStreamReader(System.in, Charset.defaultCharset())); int ch; while ( (ch=in.read()) != -1 ) System.out.print((char)ch); } - public LegacyHTMLStripCharFilter(CharStream source) { - super(source.markSupported() ? source : CharReader.get(new BufferedReader(source))); + public LegacyHTMLStripCharFilter(Reader source) { + super(source.markSupported() ? source : new BufferedReader(source)); } - public LegacyHTMLStripCharFilter(CharStream source, Set escapedTags){ + public LegacyHTMLStripCharFilter(Reader source, Set escapedTags){ this(source); this.escapedTags = escapedTags; } - public LegacyHTMLStripCharFilter(CharStream source, Set escapedTags, int readAheadLimit){ + public LegacyHTMLStripCharFilter(Reader source, Set escapedTags, int readAheadLimit){ this(source); this.escapedTags = escapedTags; this.readAheadLimit = readAheadLimit; @@ -105,7 +103,7 @@ return ch; } numRead++; - return input.read(); + return in.read(); } private int nextSkipWS() throws IOException { @@ -120,7 +118,7 @@ return pushed.charAt(len-1); } numRead++; - int ch = input.read(); + int ch = in.read(); push(ch); return ch; } @@ -182,11 +180,11 @@ private void saveState() throws IOException { lastMark = numRead; - input.mark(readAheadLimit); + in.mark(readAheadLimit); } private void restoreState() throws IOException { - input.reset(); + in.reset(); pushed.setLength(0); } @@ -775,12 +773,6 @@ return i; } - @Override - public void close() throws IOException { - input.close(); - } - - private static final HashMap entityTable; static { entityTable = new HashMap(); Index: solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java =================================================================== --- solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java (revision 1362105) +++ solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java (working copy) @@ -18,7 +18,8 @@ * limitations under the License. */ -import org.apache.lucene.analysis.CharStream; +import java.io.Reader; + import org.apache.lucene.analysis.util.CharFilterFactory; /** @@ -52,7 +53,7 @@ @Deprecated public class LegacyHTMLStripCharFilterFactory extends CharFilterFactory { - public LegacyHTMLStripCharFilter create(CharStream input) { + public LegacyHTMLStripCharFilter create(Reader input) { return new LegacyHTMLStripCharFilter(input); } Index: solr/core/src/java/org/apache/solr/analysis/PersianCharFilterFactory.java =================================================================== --- solr/core/src/java/org/apache/solr/analysis/PersianCharFilterFactory.java (revision 1362105) +++ solr/core/src/java/org/apache/solr/analysis/PersianCharFilterFactory.java (working copy) @@ -17,7 +17,9 @@ * limitations under the License. */ -import org.apache.lucene.analysis.CharStream; +import java.io.Reader; + +import org.apache.lucene.analysis.CharFilter; import org.apache.lucene.analysis.fa.PersianCharFilter; import org.apache.lucene.analysis.util.AbstractAnalysisFactory; import org.apache.lucene.analysis.util.CharFilterFactory; @@ -37,7 +39,7 @@ public class PersianCharFilterFactory extends CharFilterFactory implements MultiTermAwareComponent { @Override - public CharStream create(CharStream input) { + public CharFilter create(Reader input) { return new PersianCharFilter(input); } Index: solr/core/src/java/org/apache/solr/analysis/PatternReplaceCharFilterFactory.java =================================================================== --- solr/core/src/java/org/apache/solr/analysis/PatternReplaceCharFilterFactory.java (revision 1362105) +++ solr/core/src/java/org/apache/solr/analysis/PatternReplaceCharFilterFactory.java (working copy) @@ -17,10 +17,11 @@ package org.apache.solr.analysis; +import java.io.Reader; import java.util.Map; import java.util.regex.Pattern; -import org.apache.lucene.analysis.CharStream; +import org.apache.lucene.analysis.CharFilter; import org.apache.lucene.analysis.pattern.PatternReplaceCharFilter; import org.apache.lucene.analysis.util.CharFilterFactory; @@ -53,7 +54,7 @@ // TODO: throw exception if you set maxBlockChars or blockDelimiters ? } - public CharStream create(CharStream input) { + public CharFilter create(Reader input) { return new PatternReplaceCharFilter( p, replacement, input ); } } Index: solr/core/src/java/org/apache/solr/analysis/MappingCharFilterFactory.java =================================================================== --- solr/core/src/java/org/apache/solr/analysis/MappingCharFilterFactory.java (revision 1362105) +++ solr/core/src/java/org/apache/solr/analysis/MappingCharFilterFactory.java (working copy) @@ -19,12 +19,13 @@ import java.io.File; import java.io.IOException; +import java.io.Reader; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.lucene.analysis.CharStream; +import org.apache.lucene.analysis.CharFilter; import org.apache.lucene.analysis.charfilter.MappingCharFilter; import org.apache.lucene.analysis.charfilter.NormalizeCharMap; import org.apache.lucene.analysis.util.*; @@ -78,7 +79,7 @@ } } - public CharStream create(CharStream input) { + public CharFilter create(Reader input) { return new MappingCharFilter(normMap,input); } Index: solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java =================================================================== --- solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java (revision 1362105) +++ solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java (working copy) @@ -17,7 +17,6 @@ package org.apache.solr.handler.dataimport; import org.apache.lucene.analysis.charfilter.HTMLStripCharFilter; -import org.apache.lucene.analysis.CharReader; import java.io.IOException; import java.io.StringReader; @@ -73,7 +72,7 @@ StringBuilder out = new StringBuilder(); StringReader strReader = new StringReader(value); try { - HTMLStripCharFilter html = new HTMLStripCharFilter(CharReader.get(strReader.markSupported() ? strReader : new BufferedReader(strReader))); + HTMLStripCharFilter html = new HTMLStripCharFilter(strReader.markSupported() ? strReader : new BufferedReader(strReader)); char[] cbuf = new char[1024 * 10]; while (true) { int count = html.read(cbuf);