Index: src/java/org/apache/solr/handler/AnalysisRequestHandler.java =================================================================== --- src/java/org/apache/solr/handler/AnalysisRequestHandler.java (revision 931099) +++ src/java/org/apache/solr/handler/AnalysisRequestHandler.java (working copy) @@ -20,12 +20,8 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.FlagsAttribute; -import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; -import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; -import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; -import org.apache.lucene.analysis.tokenattributes.TypeAttribute; +import org.apache.lucene.analysis.tokenattributes.*; +import org.apache.lucene.util.BytesRef; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.SolrParams; @@ -139,15 +135,29 @@ // outer is namedList since order of tokens is important NamedList> tokens = new NamedList>(); // TODO: support custom attributes - TermAttribute termAtt = (TermAttribute) tstream.addAttribute(TermAttribute.class); - OffsetAttribute offsetAtt = (OffsetAttribute) tstream.addAttribute(OffsetAttribute.class); - TypeAttribute typeAtt = (TypeAttribute) tstream.addAttribute(TypeAttribute.class); - PositionIncrementAttribute posIncAtt = (PositionIncrementAttribute) tstream.addAttribute(PositionIncrementAttribute.class); + TermAttribute termAtt = null; + TermToBytesRefAttribute bytesAtt = null; + if (tstream.hasAttribute(TermAttribute.class)) { + termAtt = tstream.getAttribute(TermAttribute.class); + } else if (tstream.hasAttribute(TermToBytesRefAttribute.class)) { + bytesAtt = tstream.getAttribute(TermToBytesRefAttribute.class); + } + final OffsetAttribute offsetAtt = tstream.addAttribute(OffsetAttribute.class); + final TypeAttribute typeAtt = tstream.addAttribute(TypeAttribute.class); + final PositionIncrementAttribute posIncAtt = tstream.addAttribute(PositionIncrementAttribute.class); + final BytesRef bytes = new BytesRef(); while (tstream.incrementToken()) { NamedList token = new SimpleOrderedMap(); tokens.add("token", token); - token.add("value", new String(termAtt.termBuffer(), 0, termAtt.termLength())); + if (termAtt != null) { + token.add("value", termAtt.term()); + } + if (bytesAtt != null) { + bytesAtt.toBytesRef(bytes); + // TODO: This is incorrect when numeric fields change in later lucene versions. It should use BytesRef directly! + token.add("value", bytes.utf8ToString()); + } token.add("start", offsetAtt.startOffset()); token.add("end", offsetAtt.endOffset()); token.add("posInc", posIncAtt.getPositionIncrement()); Index: src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java =================================================================== --- src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java (revision 931099) +++ src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java (working copy) @@ -22,12 +22,8 @@ import org.apache.lucene.analysis.CharStream; import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.FlagsAttribute; -import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; -import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; -import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; -import org.apache.lucene.analysis.tokenattributes.TypeAttribute; +import org.apache.lucene.analysis.tokenattributes.*; +import org.apache.lucene.util.BytesRef; import org.apache.solr.analysis.CharFilterFactory; import org.apache.solr.analysis.TokenFilterFactory; import org.apache.solr.analysis.TokenizerChain; @@ -147,25 +143,33 @@ */ private List analyzeTokenStream(TokenStream tokenStream) { List tokens = new ArrayList(); - + // TODO change this API to support custom attributes - TermAttribute termAtt = (TermAttribute) - tokenStream.addAttribute(TermAttribute.class); - OffsetAttribute offsetAtt = (OffsetAttribute) - tokenStream.addAttribute(OffsetAttribute.class); - TypeAttribute typeAtt = (TypeAttribute) - tokenStream.addAttribute(TypeAttribute.class); - FlagsAttribute flagsAtt = (FlagsAttribute) - tokenStream.addAttribute(FlagsAttribute.class); - PayloadAttribute payloadAtt = (PayloadAttribute) - tokenStream.addAttribute(PayloadAttribute.class); - PositionIncrementAttribute posIncAtt = (PositionIncrementAttribute) - tokenStream.addAttribute(PositionIncrementAttribute.class); + TermAttribute termAtt = null; + TermToBytesRefAttribute bytesAtt = null; + if (tokenStream.hasAttribute(TermAttribute.class)) { + termAtt = tokenStream.getAttribute(TermAttribute.class); + } else if (tokenStream.hasAttribute(TermToBytesRefAttribute.class)) { + bytesAtt = tokenStream.getAttribute(TermToBytesRefAttribute.class); + } + final OffsetAttribute offsetAtt = tokenStream.addAttribute(OffsetAttribute.class); + final TypeAttribute typeAtt = tokenStream.addAttribute(TypeAttribute.class); + final PositionIncrementAttribute posIncAtt = tokenStream.addAttribute(PositionIncrementAttribute.class); + final FlagsAttribute flagsAtt = tokenStream.addAttribute(FlagsAttribute.class); + final PayloadAttribute payloadAtt = tokenStream.addAttribute(PayloadAttribute.class); + final BytesRef bytes = new BytesRef(); try { while (tokenStream.incrementToken()) { Token token = new Token(); - token.setTermBuffer(termAtt.termBuffer(), 0, termAtt.termLength()); + if (termAtt != null) { + token.setTermBuffer(termAtt.term()); + } + if (bytesAtt != null) { + bytesAtt.toBytesRef(bytes); + // TODO: This is incorrect when numeric fields change in later lucene versions. It should use BytesRef directly! + token.setTermBuffer(bytes.utf8ToString()); + } token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset()); token.setType(typeAtt.type()); token.setFlags(flagsAtt.getFlags()); Index: src/java/org/apache/solr/response/PHPSerializedResponseWriter.java =================================================================== --- src/java/org/apache/solr/response/PHPSerializedResponseWriter.java (revision 931099) +++ src/java/org/apache/solr/response/PHPSerializedResponseWriter.java (working copy) @@ -23,6 +23,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Fieldable; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.UnicodeUtil; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.NamedList; @@ -80,12 +81,12 @@ class PHPSerializedWriter extends JSONWriter { final private boolean CESU8; - final UnicodeUtil.UTF8Result utf8; + final BytesRef utf8; public PHPSerializedWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp, boolean CESU8) { super(writer, req, rsp); this.CESU8 = CESU8; - this.utf8 = CESU8 ? null : new UnicodeUtil.UTF8Result(); + this.utf8 = CESU8 ? null : new BytesRef(10); // never indent serialized PHP data doIndent = false; } Index: src/java/org/apache/solr/schema/TrieDateField.java =================================================================== --- src/java/org/apache/solr/schema/TrieDateField.java (revision 931099) +++ src/java/org/apache/solr/schema/TrieDateField.java (working copy) @@ -32,6 +32,7 @@ import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.Query; import org.apache.lucene.search.NumericRangeQuery; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.NumericTokenStream; @@ -126,7 +127,10 @@ @Override public String readableToIndexed(String val) { - return NumericUtils.longToPrefixCoded(super.parseMath(null, val).getTime()); + // TODO: Numeric should never be handled as String, that may break in future lucene versions! Change to use BytesRef for term texts! + BytesRef bytes = new BytesRef(NumericUtils.BUF_SIZE_LONG); + NumericUtils.longToPrefixCoded(super.parseMath(null, val).getTime(), 0, bytes); + return bytes.utf8ToString(); } @Override @@ -142,7 +146,8 @@ } @Override - public String indexedToReadable(String indexedForm) { + public String indexedToReadable(String _indexedForm) { + final BytesRef indexedForm = new BytesRef(_indexedForm); return super.toExternal( new Date(NumericUtils.prefixCodedToLong(indexedForm)) ); } Index: src/java/org/apache/solr/schema/TrieField.java =================================================================== --- src/java/org/apache/solr/schema/TrieField.java (revision 931099) +++ src/java/org/apache/solr/schema/TrieField.java (working copy) @@ -19,6 +19,7 @@ import org.apache.lucene.document.Fieldable; import org.apache.lucene.document.Field; import org.apache.lucene.search.*; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.NumericTokenStream; @@ -322,20 +323,28 @@ @Override public String readableToIndexed(String val) { + // TODO: Numeric should never be handled as String, that may break in future lucene versions! Change to use BytesRef for term texts! + BytesRef bytes = new BytesRef(NumericUtils.BUF_SIZE_LONG); switch (type) { case INTEGER: - return NumericUtils.intToPrefixCoded(Integer.parseInt(val)); + NumericUtils.intToPrefixCoded(Integer.parseInt(val), 0, bytes); + break; case FLOAT: - return NumericUtils.intToPrefixCoded(NumericUtils.floatToSortableInt(Float.parseFloat(val))); + NumericUtils.intToPrefixCoded(NumericUtils.floatToSortableInt(Float.parseFloat(val)), 0, bytes); + break; case LONG: - return NumericUtils.longToPrefixCoded(Long.parseLong(val)); + NumericUtils.longToPrefixCoded(Long.parseLong(val), 0, bytes); + break; case DOUBLE: - return NumericUtils.longToPrefixCoded(NumericUtils.doubleToSortableLong(Double.parseDouble(val))); + NumericUtils.longToPrefixCoded(NumericUtils.doubleToSortableLong(Double.parseDouble(val)), 0, bytes); + break; case DATE: - return NumericUtils.longToPrefixCoded(dateField.parseMath(null, val).getTime()); + NumericUtils.longToPrefixCoded(dateField.parseMath(null, val).getTime(), 0, bytes); + break; default: throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type); } + return bytes.utf8ToString(); } @@ -371,7 +380,8 @@ } @Override - public String indexedToReadable(String indexedForm) { + public String indexedToReadable(String _indexedForm) { + final BytesRef indexedForm = new BytesRef(_indexedForm); switch (type) { case INTEGER: return Integer.toString( NumericUtils.prefixCodedToInt(indexedForm) ); Index: src/test/org/apache/solr/search/TestDocSet.java =================================================================== --- src/test/org/apache/solr/search/TestDocSet.java (revision 931099) +++ src/test/org/apache/solr/search/TestDocSet.java (working copy) @@ -355,7 +355,7 @@ return r; } - public IndexReader dummyMultiReader(int nSeg, int maxDoc) { + public IndexReader dummyMultiReader(int nSeg, int maxDoc) throws IOException { if (nSeg==1 && rand.nextBoolean()) return dummyIndexReader(rand.nextInt(maxDoc)); IndexReader[] subs = new IndexReader[rand.nextInt(nSeg)+1];