Index: lucene/src/java/org/apache/lucene/queryParser/QueryParser.java =================================================================== --- lucene/src/java/org/apache/lucene/queryParser/QueryParser.java (revision 965632) +++ lucene/src/java/org/apache/lucene/queryParser/QueryParser.java (working copy) @@ -17,7 +17,7 @@ import org.apache.lucene.analysis.CachingTokenFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; import org.apache.lucene.document.DateField; import org.apache.lucene.document.DateTools; import org.apache.lucene.index.Term; @@ -33,6 +33,7 @@ import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Version; import org.apache.lucene.util.VirtualMethod; @@ -574,7 +575,7 @@ source = analyzer.tokenStream(field, new StringReader(queryText)); } CachingTokenFilter buffer = new CachingTokenFilter(source); - CharTermAttribute termAtt = null; + TermToBytesRefAttribute termAtt = null; PositionIncrementAttribute posIncrAtt = null; int numTokens = 0; @@ -586,8 +587,8 @@ // success==false if we hit an exception } if (success) { - if (buffer.hasAttribute(CharTermAttribute.class)) { - termAtt = buffer.getAttribute(CharTermAttribute.class); + if (buffer.hasAttribute(TermToBytesRefAttribute.class)) { + termAtt = buffer.getAttribute(TermToBytesRefAttribute.class); } if (buffer.hasAttribute(PositionIncrementAttribute.class)) { posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class); @@ -629,11 +630,11 @@ if (numTokens == 0) return null; else if (numTokens == 1) { - String term = null; + BytesRef term = new BytesRef(); try { boolean hasNext = buffer.incrementToken(); assert hasNext == true; - term = termAtt.toString(); + termAtt.toBytesRef(term); } catch (IOException e) { // safe to ignore, because we know the number of tokens } @@ -648,11 +649,11 @@ BooleanClause.Occur.MUST : BooleanClause.Occur.SHOULD; for (int i = 0; i < numTokens; i++) { - String term = null; + BytesRef term = new BytesRef(); try { boolean hasNext = buffer.incrementToken(); assert hasNext == true; - term = termAtt.toString(); + termAtt.toBytesRef(term); } catch (IOException e) { // safe to ignore, because we know the number of tokens } @@ -670,12 +671,12 @@ List multiTerms = new ArrayList(); int position = -1; for (int i = 0; i < numTokens; i++) { - String term = null; + BytesRef term = new BytesRef(); int positionIncrement = 1; try { boolean hasNext = buffer.incrementToken(); assert hasNext == true; - term = termAtt.toString(); + termAtt.toBytesRef(term); if (posIncrAtt != null) { positionIncrement = posIncrAtt.getPositionIncrement(); } @@ -709,13 +710,13 @@ for (int i = 0; i < numTokens; i++) { - String term = null; + BytesRef term = new BytesRef(); int positionIncrement = 1; try { boolean hasNext = buffer.incrementToken(); assert hasNext == true; - term = termAtt.toString(); + termAtt.toBytesRef(term); if (posIncrAtt != null) { positionIncrement = posIncrAtt.getPositionIncrement(); } @@ -1568,12 +1569,6 @@ finally { jj_save(0, xla); } } - private boolean jj_3R_2() { - if (jj_scan_token(TERM)) return true; - if (jj_scan_token(COLON)) return true; - return false; - } - private boolean jj_3_1() { Token xsp; xsp = jj_scanpos; @@ -1590,6 +1585,12 @@ return false; } + private boolean jj_3R_2() { + if (jj_scan_token(TERM)) return true; + if (jj_scan_token(COLON)) return true; + return false; + } + /** Generated Token Manager. */ public QueryParserTokenManager token_source; /** Current token. */ Index: lucene/src/java/org/apache/lucene/queryParser/QueryParser.jj =================================================================== --- lucene/src/java/org/apache/lucene/queryParser/QueryParser.jj (revision 965632) +++ lucene/src/java/org/apache/lucene/queryParser/QueryParser.jj (working copy) @@ -41,7 +41,7 @@ import org.apache.lucene.analysis.CachingTokenFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; import org.apache.lucene.document.DateField; import org.apache.lucene.document.DateTools; import org.apache.lucene.index.Term; @@ -57,6 +57,7 @@ import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Version; import org.apache.lucene.util.VirtualMethod; @@ -598,7 +599,7 @@ source = analyzer.tokenStream(field, new StringReader(queryText)); } CachingTokenFilter buffer = new CachingTokenFilter(source); - CharTermAttribute termAtt = null; + TermToBytesRefAttribute termAtt = null; PositionIncrementAttribute posIncrAtt = null; int numTokens = 0; @@ -610,8 +611,8 @@ // success==false if we hit an exception } if (success) { - if (buffer.hasAttribute(CharTermAttribute.class)) { - termAtt = buffer.getAttribute(CharTermAttribute.class); + if (buffer.hasAttribute(TermToBytesRefAttribute.class)) { + termAtt = buffer.getAttribute(TermToBytesRefAttribute.class); } if (buffer.hasAttribute(PositionIncrementAttribute.class)) { posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class); @@ -653,11 +654,11 @@ if (numTokens == 0) return null; else if (numTokens == 1) { - String term = null; + BytesRef term = new BytesRef(); try { boolean hasNext = buffer.incrementToken(); assert hasNext == true; - term = termAtt.toString(); + termAtt.toBytesRef(term); } catch (IOException e) { // safe to ignore, because we know the number of tokens } @@ -672,11 +673,11 @@ BooleanClause.Occur.MUST : BooleanClause.Occur.SHOULD; for (int i = 0; i < numTokens; i++) { - String term = null; + BytesRef term = new BytesRef(); try { boolean hasNext = buffer.incrementToken(); assert hasNext == true; - term = termAtt.toString(); + termAtt.toBytesRef(term); } catch (IOException e) { // safe to ignore, because we know the number of tokens } @@ -694,12 +695,12 @@ List multiTerms = new ArrayList(); int position = -1; for (int i = 0; i < numTokens; i++) { - String term = null; + BytesRef term = new BytesRef(); int positionIncrement = 1; try { boolean hasNext = buffer.incrementToken(); assert hasNext == true; - term = termAtt.toString(); + termAtt.toBytesRef(term); if (posIncrAtt != null) { positionIncrement = posIncrAtt.getPositionIncrement(); } @@ -733,13 +734,13 @@ for (int i = 0; i < numTokens; i++) { - String term = null; + BytesRef term = new BytesRef(); int positionIncrement = 1; try { boolean hasNext = buffer.incrementToken(); assert hasNext == true; - term = termAtt.toString(); + termAtt.toBytesRef(term); if (posIncrAtt != null) { positionIncrement = posIncrAtt.getPositionIncrement(); } Index: lucene/src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java =================================================================== --- lucene/src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java (revision 965632) +++ lucene/src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java (working copy) @@ -15,7 +15,7 @@ import org.apache.lucene.analysis.CachingTokenFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; import org.apache.lucene.document.DateField; import org.apache.lucene.document.DateTools; import org.apache.lucene.index.Term; @@ -31,6 +31,7 @@ import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Version; import org.apache.lucene.util.VirtualMethod; Index: lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java =================================================================== --- lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java (revision 965632) +++ lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java (working copy) @@ -24,6 +24,7 @@ import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.AttributeSource; /** @@ -307,7 +308,7 @@ List list = new ArrayList(); int positionCount = 0; boolean severalTokensAtSamePosition = false; - CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); + TermToBytesRefAttribute termAtt = source.addAttribute(TermToBytesRefAttribute.class); PositionIncrementAttribute posincrAtt = source.addAttribute(PositionIncrementAttribute.class); try { @@ -328,7 +329,9 @@ return null; else if (list.size() == 1) { source.restoreState(list.get(0)); - return new TermQuery(new Term(field, termAtt.toString())); + BytesRef term = new BytesRef(); + termAtt.toBytesRef(term); + return new TermQuery(new Term(field, term)); } else { if (severalTokensAtSamePosition || !quoted) { if (positionCount == 1 || !quoted) { @@ -339,9 +342,11 @@ BooleanClause.Occur.MUST : BooleanClause.Occur.SHOULD; for (int i = 0; i < list.size(); i++) { + BytesRef term = new BytesRef(); source.restoreState(list.get(i)); + termAtt.toBytesRef(term); TermQuery currentQuery = new TermQuery( - new Term(field, termAtt.toString())); + new Term(field, term)); q.add(currentQuery, occur); } return q; @@ -351,12 +356,14 @@ MultiPhraseQuery mpq = new MultiPhraseQuery(); List multiTerms = new ArrayList(); for (int i = 0; i < list.size(); i++) { + BytesRef term = new BytesRef(); source.restoreState(list.get(i)); if (posincrAtt.getPositionIncrement() == 1 && multiTerms.size() > 0) { mpq.add(multiTerms.toArray(new Term[0])); multiTerms.clear(); } - multiTerms.add(new Term(field, termAtt.toString())); + termAtt.toBytesRef(term); + multiTerms.add(new Term(field, term)); } mpq.add(multiTerms.toArray(new Term[0])); return mpq; @@ -366,8 +373,10 @@ PhraseQuery q = new PhraseQuery(); q.setSlop(phraseSlop); for (int i = 0; i < list.size(); i++) { + BytesRef term = new BytesRef(); source.restoreState(list.get(i)); - q.add(new Term(field, termAtt.toString())); + termAtt.toBytesRef(term); + q.add(new Term(field, term)); } return q; } Index: lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj =================================================================== --- lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj (revision 965632) +++ lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj (working copy) @@ -48,6 +48,7 @@ import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.AttributeSource; /** @@ -331,7 +332,7 @@ List list = new ArrayList(); int positionCount = 0; boolean severalTokensAtSamePosition = false; - CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); + TermToBytesRefAttribute termAtt = source.addAttribute(TermToBytesRefAttribute.class); PositionIncrementAttribute posincrAtt = source.addAttribute(PositionIncrementAttribute.class); try { @@ -352,7 +353,9 @@ return null; else if (list.size() == 1) { source.restoreState(list.get(0)); - return new TermQuery(new Term(field, termAtt.toString())); + BytesRef term = new BytesRef(); + termAtt.toBytesRef(term); + return new TermQuery(new Term(field, term)); } else { if (severalTokensAtSamePosition || !quoted) { if (positionCount == 1 || !quoted) { @@ -363,9 +366,11 @@ BooleanClause.Occur.MUST : BooleanClause.Occur.SHOULD; for (int i = 0; i < list.size(); i++) { + BytesRef term = new BytesRef(); source.restoreState(list.get(i)); + termAtt.toBytesRef(term); TermQuery currentQuery = new TermQuery( - new Term(field, termAtt.toString())); + new Term(field, term)); q.add(currentQuery, occur); } return q; @@ -375,12 +380,14 @@ MultiPhraseQuery mpq = new MultiPhraseQuery(); List multiTerms = new ArrayList(); for (int i = 0; i < list.size(); i++) { + BytesRef term = new BytesRef(); source.restoreState(list.get(i)); if (posincrAtt.getPositionIncrement() == 1 && multiTerms.size() > 0) { mpq.add(multiTerms.toArray(new Term[0])); multiTerms.clear(); } - multiTerms.add(new Term(field, termAtt.toString())); + termAtt.toBytesRef(term); + multiTerms.add(new Term(field, term)); } mpq.add(multiTerms.toArray(new Term[0])); return mpq; @@ -390,8 +397,10 @@ PhraseQuery q = new PhraseQuery(); q.setSlop(phraseSlop); for (int i = 0; i < list.size(); i++) { + BytesRef term = new BytesRef(); source.restoreState(list.get(i)); - q.add(new Term(field, termAtt.toString())); + termAtt.toBytesRef(term); + q.add(new Term(field, term)); } return q; } Index: lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParserTokenManager.java =================================================================== --- lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParserTokenManager.java (revision 965632) +++ lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParserTokenManager.java (working copy) @@ -22,6 +22,7 @@ import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.AttributeSource; /** Token Manager. */ Index: lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java =================================================================== --- lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java (revision 965632) +++ lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java (working copy) @@ -5,10 +5,11 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; import org.apache.lucene.index.Term; import org.apache.lucene.search.Filter; import org.apache.lucene.search.TermsFilter; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.xmlparser.DOMUtils; import org.apache.lucene.xmlparser.FilterBuilder; import org.apache.lucene.xmlparser.ParserException; @@ -57,19 +58,21 @@ String text = DOMUtils.getNonBlankTextOrFail(e); String fieldName = DOMUtils.getAttributeWithInheritanceOrFail(e, "fieldName"); TokenStream ts = analyzer.tokenStream(fieldName, new StringReader(text)); - CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); + TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class); try { Term term = null; while (ts.incrementToken()) { + BytesRef bytes = new BytesRef(); + termAtt.toBytesRef(bytes); if (term == null) { - term = new Term(fieldName, termAtt.toString()); + term = new Term(fieldName, bytes); } else { // create from previous to save fieldName.intern overhead - term = term.createTerm(termAtt.toString()); + term = term.createTerm(bytes); } tf.addTerm(term); } Index: lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java =================================================================== --- lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java (revision 965632) +++ lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java (working copy) @@ -6,11 +6,12 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; import org.apache.lucene.index.Term; import org.apache.lucene.search.spans.SpanOrQuery; import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.search.spans.SpanTermQuery; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.xmlparser.DOMUtils; import org.apache.lucene.xmlparser.ParserException; import org.w3c.dom.Element; @@ -56,10 +57,12 @@ { ArrayList clausesList=new ArrayList(); TokenStream ts=analyzer.tokenStream(fieldName,new StringReader(value)); - CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); + TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class); while (ts.incrementToken()) { - SpanTermQuery stq=new SpanTermQuery(new Term(fieldName, termAtt.toString())); + BytesRef term = new BytesRef(); + termAtt.toBytesRef(term); + SpanTermQuery stq=new SpanTermQuery(new Term(fieldName, term)); clausesList.add(stq); } SpanOrQuery soq=new SpanOrQuery(clausesList.toArray(new SpanQuery[clausesList.size()])); Index: lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java =================================================================== --- lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java (revision 965632) +++ lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java (working copy) @@ -5,12 +5,13 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.xmlparser.DOMUtils; import org.apache.lucene.xmlparser.ParserException; import org.apache.lucene.xmlparser.QueryBuilder; @@ -57,16 +58,18 @@ TokenStream ts = analyzer.tokenStream(fieldName, new StringReader(text)); try { - CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); + TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class); Term term = null; while (ts.incrementToken()) { + BytesRef bytes = new BytesRef(); + termAtt.toBytesRef(bytes); if (term == null) { - term = new Term(fieldName, termAtt.toString()); + term = new Term(fieldName, bytes); } else { // create from previous to save fieldName.intern overhead - term = term.createTerm(termAtt.toString()); + term = term.createTerm(bytes); } bq.add(new BooleanClause(new TermQuery(term),BooleanClause.Occur.SHOULD)); }