Index: lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java =================================================================== --- lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (revision 682416) +++ lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (working copy) @@ -15,19 +15,32 @@ * limitations under the License. */ +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; + import junit.framework.TestCase; + import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.index.*; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Payload; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermDocs; +import org.apache.lucene.index.TermEnum; +import org.apache.lucene.index.TermFreqVector; +import org.apache.lucene.index.TermPositionVector; +import org.apache.lucene.index.TermPositions; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; -import java.io.IOException; -import java.util.*; - /** * Asserts equality of content and behaviour of two index readers. */ @@ -151,21 +164,23 @@ document.add(f); if (i > 4) { final List tokens = new ArrayList(2); - Token t = new Token("the", 0, 2, "text"); + Token t = createToken("the", 0, 2, "text"); t.setPayload(new Payload(new byte[]{1, 2, 3})); tokens.add(t); - t = new Token("end", 3, 5, "text"); + t = createToken("end", 3, 5, "text"); t.setPayload(new Payload(new byte[]{2})); tokens.add(t); - tokens.add(new Token("fin", 7, 9)); + tokens.add(createToken("fin", 7, 9)); document.add(new Field("f", new TokenStream() { Iterator it = tokens.iterator(); - public Token next() throws IOException { + public Token next(Token token) throws IOException { if (!it.hasNext()) { return null; } - return it.next(); + // Resettable token streams need to return clones. + token = (Token) it.next(); + return (Token) token.clone(); } public void reset() throws IOException { @@ -466,4 +481,19 @@ testReader.close(); } + private static Token createToken(String term, int start, int offset) + { + Token token = new Token(start, offset); + token.setTermBuffer(term); + return token; + } + + private static Token createToken(String term, int start, int offset, String type) + { + Token token = new Token(start, offset, type); + token.setTermBuffer(term); + return token; + } + + } Index: lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java =================================================================== --- lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (revision 682416) +++ lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (working copy) @@ -520,12 +520,13 @@ } else { tokenStream = analyzer.tokenStream(field.name(), new StringReader(field.stringValue())); } - Token next = tokenStream.next(); + Token next = new Token(); + next = tokenStream.next(next); + while (next != null) { - next.setTermText(next.termText().intern()); // todo: not sure this needs to be interned? - tokens.add(next); // the vector will be built on commit. - next = tokenStream.next(); + tokens.add((Token) next.clone()); // the vector will be built on commit. + next = tokenStream.next(next); fieldSetting.fieldLength++; if (fieldSetting.fieldLength > maxFieldLength) { break; @@ -533,7 +534,10 @@ } } else { // untokenized - tokens.add(new Token(field.stringValue().intern(), 0, field.stringValue().length(), "untokenized")); + String fieldVal = field.stringValue(); + Token token = new Token(0, fieldVal.length(), "untokenized"); + token.setTermBuffer(fieldVal); + tokens.add(token); fieldSetting.fieldLength++; } } @@ -567,10 +571,10 @@ for (Token token : eField_Tokens.getValue()) { - TermDocumentInformationFactory termDocumentInformationFactory = termDocumentInformationFactoryByTermText.get(token.termText()); + TermDocumentInformationFactory termDocumentInformationFactory = termDocumentInformationFactoryByTermText.get(token.term()); if (termDocumentInformationFactory == null) { termDocumentInformationFactory = new TermDocumentInformationFactory(); - termDocumentInformationFactoryByTermText.put(token.termText(), termDocumentInformationFactory); + termDocumentInformationFactoryByTermText.put(token.term(), termDocumentInformationFactory); } //termDocumentInformationFactory.termFrequency++;