Index: solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java =================================================================== --- solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java (revision 1162347) +++ solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java (revision ) @@ -195,7 +195,7 @@ source = analyzer.reusableTokenStream(field, new StringReader(part)); source.reset(); } catch (IOException e) { - source = analyzer.tokenStream(field, new StringReader(part)); + throw new RuntimeException("Unable to initialize TokenStream to analyze range part: " + part, e); } TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class); @@ -212,8 +212,11 @@ } try { + source.end(); source.close(); - } catch (IOException ignored) {} + } catch (IOException e) { + throw new RuntimeException("Unable to end & close TokenStream after analyzing range part: " + part, e); + } return new BytesRef(bytes); } Index: modules/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java =================================================================== --- modules/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java (revision 1145016) +++ modules/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java (revision ) @@ -477,28 +477,25 @@ source = analyzer.reusableTokenStream(field, new StringReader(queryText)); source.reset(); } catch (IOException e) { - source = analyzer.tokenStream(field, new StringReader(queryText)); + throw new ParseException("Unable to initialize TokenStream to analyze query text", e); } CachingTokenFilter buffer = new CachingTokenFilter(source); TermToBytesRefAttribute termAtt = null; PositionIncrementAttribute posIncrAtt = null; int numTokens = 0; - boolean success = false; try { buffer.reset(); - success = true; } catch (IOException e) { - // success==false if we hit an exception + throw new ParseException("Unable to initialize TokenStream to analyze query text", e); } - if (success) { + - if (buffer.hasAttribute(TermToBytesRefAttribute.class)) { - termAtt = buffer.getAttribute(TermToBytesRefAttribute.class); - } - if (buffer.hasAttribute(PositionIncrementAttribute.class)) { - posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class); - } + if (buffer.hasAttribute(TermToBytesRefAttribute.class)) { + termAtt = buffer.getAttribute(TermToBytesRefAttribute.class); + } + if (buffer.hasAttribute(PositionIncrementAttribute.class)) { + posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class); + } - } int positionCount = 0; boolean severalTokensAtSamePosition = false; @@ -529,7 +526,7 @@ source.close(); } catch (IOException e) { - // ignore + throw new ParseException("Cannot close TokenStream analyzing query text", e); } BytesRef bytes = termAtt == null ? null : termAtt.getBytesRef(); @@ -789,7 +786,7 @@ source = analyzer.reusableTokenStream(field, new StringReader(part)); source.reset(); } catch (IOException e) { - source = analyzer.tokenStream(field, new StringReader(part)); + throw new RuntimeException("Unable to initialize TokenStream to analyze range part: " + part, e); } TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class); @@ -808,7 +805,9 @@ try { source.end(); source.close(); - } catch (IOException ignored) {} + } catch (IOException e) { + throw new RuntimeException("Unable to end & close TokenStream after analyzing range part: " + part, e); + } return new BytesRef(bytes); } Index: modules/queryparser/src/java/org/apache/lucene/queryparser/classic/ParseException.java =================================================================== --- modules/queryparser/src/java/org/apache/lucene/queryparser/classic/ParseException.java (revision 1145016) +++ modules/queryparser/src/java/org/apache/lucene/queryparser/classic/ParseException.java (revision ) @@ -59,6 +59,18 @@ } /** + * Creates a new ParseException which is wrapping another Throwable with an + * additional message + * + * @param message Message for the Exception + * @param throwable Wrapped Throwable + */ + public ParseException(String message, Throwable throwable) { + super(message, throwable); + specialConstructor = false; + } + + /** * This variable determines which constructor was used to create * this object and thereby affects the semantics of the * "getMessage" method (see below). Index: solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java =================================================================== --- solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java (revision 1144761) +++ solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java (revision ) @@ -37,23 +37,25 @@ * * @since solr 1.3 **/ -class SimpleQueryConverter extends SpellingQueryConverter{ +class SimpleQueryConverter extends SpellingQueryConverter { + @Override public Collection convert(String origQuery) { + try { - Collection result = new HashSet(); - WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_40); + Collection result = new HashSet(); + WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_40); - TokenStream ts = analyzer.tokenStream("", new StringReader(origQuery)); + TokenStream ts = analyzer.reusableTokenStream("", new StringReader(origQuery)); - // TODO: support custom attributes - CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); - OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class); - TypeAttribute typeAtt = ts.addAttribute(TypeAttribute.class); - FlagsAttribute flagsAtt = ts.addAttribute(FlagsAttribute.class); - PayloadAttribute payloadAtt = ts.addAttribute(PayloadAttribute.class); - PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class); - + // TODO: support custom attributes + CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); + OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class); + TypeAttribute typeAtt = ts.addAttribute(TypeAttribute.class); + FlagsAttribute flagsAtt = ts.addAttribute(FlagsAttribute.class); + PayloadAttribute payloadAtt = ts.addAttribute(PayloadAttribute.class); + PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class); + - try { ts.reset(); + - while (ts.incrementToken()){ + while (ts.incrementToken()) { Token tok = new Token(); tok.copyBuffer(termAtt.buffer(), 0, termAtt.length()); tok.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset()); @@ -63,9 +65,12 @@ tok.setType(typeAtt.type()); result.add(tok); } + ts.end(); + ts.close(); + + return result; } catch (IOException e) { throw new RuntimeException(e); } - return result; } } Index: solr/core/src/java/org/apache/solr/schema/TextField.java =================================================================== --- solr/core/src/java/org/apache/solr/schema/TextField.java (revision 1162347) +++ solr/core/src/java/org/apache/solr/schema/TextField.java (revision ) @@ -112,28 +112,25 @@ source = analyzer.reusableTokenStream(field, new StringReader(queryText)); source.reset(); } catch (IOException e) { - source = analyzer.tokenStream(field, new StringReader(queryText)); + throw new RuntimeException("Unable to initialize TokenStream to analyze query text", e); } CachingTokenFilter buffer = new CachingTokenFilter(source); CharTermAttribute termAtt = null; PositionIncrementAttribute posIncrAtt = null; int numTokens = 0; - boolean success = false; try { buffer.reset(); - success = true; } catch (IOException e) { - // success==false if we hit an exception + throw new RuntimeException("Unable to initialize TokenStream to analyze query text", e); } - if (success) { + - if (buffer.hasAttribute(CharTermAttribute.class)) { - termAtt = buffer.getAttribute(CharTermAttribute.class); - } - if (buffer.hasAttribute(PositionIncrementAttribute.class)) { - posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class); - } + if (buffer.hasAttribute(CharTermAttribute.class)) { + termAtt = buffer.getAttribute(CharTermAttribute.class); + } + if (buffer.hasAttribute(PositionIncrementAttribute.class)) { + posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class); + } - } int positionCount = 0; boolean severalTokensAtSamePosition = false; Index: solr/core/src/java/org/apache/solr/schema/CollationField.java =================================================================== --- solr/core/src/java/org/apache/solr/schema/CollationField.java (revision 1162347) +++ solr/core/src/java/org/apache/solr/schema/CollationField.java (revision ) @@ -217,7 +217,7 @@ source = analyzer.reusableTokenStream(field, new StringReader(part)); source.reset(); } catch (IOException e) { - source = analyzer.tokenStream(field, new StringReader(part)); + throw new RuntimeException("Unable to initialize TokenStream to analyze range part: " + part, e); } TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class); @@ -234,8 +234,11 @@ } try { + source.end(); source.close(); - } catch (IOException ignored) {} + } catch (IOException e) { + throw new RuntimeException("Unable to end & close TokenStream after analyzing range part: " + part, e); + } return new BytesRef(bytes); } Index: solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java =================================================================== --- solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java (revision 1175297) +++ solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java (revision ) @@ -140,20 +140,25 @@ * @param analyzer The analyzer to use. */ protected Set getQueryTokenSet(String query, Analyzer analyzer) { + try { - final Set tokens = new HashSet(); + final Set tokens = new HashSet(); - final TokenStream tokenStream = analyzer.tokenStream("", new StringReader(query)); + final TokenStream tokenStream = analyzer.reusableTokenStream("", new StringReader(query)); - final TermToBytesRefAttribute bytesAtt = tokenStream.getAttribute(TermToBytesRefAttribute.class); - final BytesRef bytes = bytesAtt.getBytesRef(); + final TermToBytesRefAttribute bytesAtt = tokenStream.getAttribute(TermToBytesRefAttribute.class); + final BytesRef bytes = bytesAtt.getBytesRef(); - try { + tokenStream.reset(); + while (tokenStream.incrementToken()) { bytesAtt.fillBytesRef(); tokens.add(new BytesRef(bytes)); } + + tokenStream.end(); + tokenStream.close(); + return tokens; } catch (IOException ioe) { throw new RuntimeException("Error occured while iterating over tokenstream", ioe); } - return tokens; } /**