Index: modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java =================================================================== --- modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java (revision 1303167) +++ modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java (working copy) @@ -105,13 +105,19 @@ started = true; gramSize = minGram; char[] chars = new char[1024]; - charsRead = input.read(chars); - if (charsRead < 0) { - charsRead = inLen = 0; + charsRead = 0; + while (charsRead < chars.length) { + int inc = input.read(chars, charsRead, chars.length-charsRead); + if (inc == -1) { + break; + } + charsRead += inc; + } + inStr = new String(chars, 0, charsRead).trim(); // remove any trailing empty strings + inLen = inStr.length(); + if (inLen == 0) { return false; } - inStr = new String(chars).trim(); // remove any trailing empty strings - inLen = inStr.length(); } if (pos+gramSize > inLen) { // if we hit the end of the string @@ -140,7 +146,6 @@ @Override public void reset(Reader input) throws IOException { super.reset(input); - reset(); } @Override Index: modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java =================================================================== --- modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java (revision 1303167) +++ modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java (working copy) @@ -183,15 +183,21 @@ // if we are just starting, read the whole input if (!started) { started = true; + gramSize = minGram; char[] chars = new char[1024]; - charsRead = input.read(chars); - if (charsRead < 0) { - charsRead = inLen = 0; + charsRead = 0; + while (charsRead < chars.length) { + int inc = input.read(chars, charsRead, chars.length-charsRead); + if (inc == -1) { + break; + } + charsRead += inc; + } + inStr = new String(chars, 0, charsRead).trim(); // remove any trailing empty strings + inLen = inStr.length(); + if (inLen == 0) { return false; } - inStr = new String(chars, 0, charsRead).trim(); // remove any leading or trailing spaces - inLen = inStr.length(); - gramSize = minGram; } // if the remaining input is too short, we can't generate any n-grams @@ -223,7 +229,6 @@ @Override public void reset(Reader input) throws IOException { super.reset(input); - reset(); } @Override Index: lucene/test-framework/src/java/org/apache/lucene/analysis/MockReaderWrapper.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/analysis/MockReaderWrapper.java (revision 0) +++ lucene/test-framework/src/java/org/apache/lucene/analysis/MockReaderWrapper.java (working copy) @@ -0,0 +1,88 @@ +package org.apache.lucene.analysis; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.Reader; +import java.util.Random; + +/** Wraps a Reader, and can throw random or fixed + * exceptions, and spoon feed read chars. */ + +public class MockReaderWrapper extends Reader { + + private final Reader in; + // NOTE: not used yet... + private final Random random; + + private int excAtChar = -1; + private int readSoFar; + private boolean throwExcNext; + + public MockReaderWrapper(Random random, Reader in) { + this.in = in; + this.random = random; + } + + /** Throw an exception after reading this many chars. */ + public void throwExcAfterChar(int charUpto) { + excAtChar = charUpto; + // You should only call this on init!: + assert readSoFar == 0; + } + + public void throwExcNext() { + throwExcNext = true; + } + + @Override + public void close() throws IOException { + in.close(); + } + + @Override + public int read(char[] cbuf, int off, int len) throws IOException { + if (throwExcNext || (excAtChar != -1 && readSoFar >= excAtChar)) { + throw new RuntimeException("fake exception now!"); + } + final int read; + if (excAtChar != -1) { + final int left = excAtChar - readSoFar; + read = in.read(cbuf, off, Math.min(len, left)); + assert read != -1; + readSoFar += read; + } else { + read = in.read(cbuf, off, len); + } + return read; + } + + @Override + public boolean markSupported() { + return false; + } + + @Override + public boolean ready() { + return false; + } + + public static boolean isMyEvilException(Throwable t) { + return (t instanceof RuntimeException) && "fake exception no!".equals(t.getMessage()); + } +}; Property changes on: lucene/test-framework/src/java/org/apache/lucene/analysis/MockReaderWrapper.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native Index: lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (revision 1303167) +++ lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (working copy) @@ -177,8 +177,9 @@ } assertFalse("TokenStream has more tokens than expected", ts.incrementToken()); ts.end(); - if (finalOffset != null) + if (finalOffset != null) { assertEquals("finalOffset ", finalOffset.intValue(), offsetAtt.endOffset()); + } if (offsetAtt != null) { assertTrue("finalOffset must be >= 0", offsetAtt.endOffset() >= 0); } @@ -391,6 +392,8 @@ List startOffsets = new ArrayList(); List endOffsets = new ArrayList(); ts.reset(); + + // First pass: save away "correct" tokens while (ts.incrementToken()) { tokens.add(termAtt.toString()); if (typeAtt != null) types.add(typeAtt.type()); @@ -403,12 +406,88 @@ } ts.end(); ts.close(); + // verify reusing is "reproducable" and also get the normal tokenstream sanity checks if (!tokens.isEmpty()) { + + // KWTokenizer (for example) can produce a token + // even when input is length 0: + if (text.length() != 0) { + + // (Optional) second pass: do something evil: + final int evilness = random.nextInt(50); + if (evilness == 17) { + if (VERBOSE) { + System.out.println(Thread.currentThread().getName() + ": NOTE: BaseTokenStreamTestCase: re-run analysis w/ exception"); + } + // Throw an errant exception from the Reader: + + MockReaderWrapper evilReader = new MockReaderWrapper(random, new StringReader(text)); + evilReader.throwExcAfterChar(random.nextInt(text.length()+1)); + reader = evilReader; + + try { + // NOTE: some Tokenizers go and read characters + // when you call .setReader(Reader), eg + // PatternTokenizer. This is a bit + // iffy... (really, they should only + // pull from the Reader when you call + // .incremenToken(), I think?), but we + // currently allow it, so, we must call + // a.tokenStream inside the try since we may + // hit the exc on init: + ts = a.tokenStream("dummy", useCharFilter ? new MockCharFilter(evilReader, remainder) : evilReader); + ts.reset(); + while (ts.incrementToken()); + fail("did not hit exception"); + } catch (RuntimeException re) { + assertEquals("fake exception now!", re.getMessage()); + } + ts.end(); + ts.close(); + } else if (evilness == 7) { + // Only consume a subset of the tokens: + final int numTokensToRead = random.nextInt(tokens.size()); + if (VERBOSE) { + System.out.println(Thread.currentThread().getName() + ": NOTE: BaseTokenStreamTestCase: re-run analysis, only consuming " + numTokensToRead + " of " + tokens.size() + " tokens"); + } + + reader = new StringReader(text); + ts = a.tokenStream("dummy", useCharFilter ? new MockCharFilter(reader, remainder) : reader); + ts.reset(); + for(int tokenCount=0;tokenCount= 0) { off++; assert Character.isLowSurrogate((char) ch2); @@ -199,8 +217,11 @@ offsetAtt.setOffset(finalOffset, finalOffset); // some tokenizers, such as limiting tokenizers, call end() before incrementToken() returns false. // these tests should disable this check (in general you should consume the entire stream) - assert !enableChecks || streamState == State.INCREMENT_FALSE : "end() called before incrementToken() returned false!"; - streamState = State.END; + try { + assert !enableChecks || streamState == State.INCREMENT_FALSE : "end() called before incrementToken() returned false!"; + } finally { + streamState = State.END; + } } /**