Index: lucene/contrib/CHANGES.txt =================================================================== --- lucene/contrib/CHANGES.txt (révision 1160156) +++ lucene/contrib/CHANGES.txt (copie de travail) @@ -7,6 +7,14 @@ New Features + * LUCENE-3392: Added ComboAnalyzer and ComboTokenStream + Multiplexes output of multiple analyzers and tokenstreams. + Easily permits to broaden the spectrum of searcheable terms, + like indexing both stemmed, normalized and original terms, + thus giving more results in case of misconfigured + (as wrong locale) search analysis. + (Olivier Favre) + * LUCENE-3234: provide a limit on phrase analysis in FastVectorHighlighter for highlighting speed up. Use FastVectorHighlighter.setPhraseLimit() to set limit (e.g. 5000). (Mike Sokolov via Koji Sekiguchi) Index: lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/TestComboAnalyzer.java =================================================================== --- lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/TestComboAnalyzer.java (révision 0) +++ lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/TestComboAnalyzer.java (révision 0) @@ -0,0 +1,54 @@ +package org.apache.lucene.analysis; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.analysis.standard.StandardAnalyzer; + +import java.io.IOException; +import java.io.StringReader; + +/** + * Testcase for {@link ComboAnalyzer} + */ +public class TestComboAnalyzer extends BaseTokenStreamTestCase { + + public void testSingleAnalyzer() throws IOException { + ComboAnalyzer cb = new ComboAnalyzer(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)); + for (int i = 0 ; i < 3 ; i++) + assertTokenStreamContents(cb.reusableTokenStream("field", new StringReader("just a little test "+i)), + new String[]{"just", "a", "little", "test", Integer.toString(i)}, + new int[]{ 0, 5, 7, 14, 19}, + new int[]{ 4, 6, 13, 18, 20}, + new int[]{ 1, 1, 1, 1, 1}); + } + + public void testMultipleAnalyzers() throws IOException { + ComboAnalyzer cb = new ComboAnalyzer(TEST_VERSION_CURRENT, + new WhitespaceAnalyzer(TEST_VERSION_CURRENT), + new StandardAnalyzer(TEST_VERSION_CURRENT), + new KeywordAnalyzer() + ); + for (int i = 0 ; i < 3 ; i++) + assertTokenStreamContents(cb.reusableTokenStream("field", new StringReader("just a little test "+i)), + new String[]{"just", "just", "just a little test "+i, "a", "little", "little", "test", "test", Integer.toString(i), Integer.toString(i)}, + new int[]{ 0, 0, 0, 5, 7, 7, 14, 14, 19, 19}, + new int[]{ 4, 4, 20, 6, 13, 13, 18, 18, 20, 20}, + new int[]{ 1, 0, 0, 1, 1, 0, 1, 0, 1, 0}); + } + +} Index: lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/TestComboTokenStream.java =================================================================== --- lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/TestComboTokenStream.java (révision 0) +++ lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/TestComboTokenStream.java (révision 0) @@ -0,0 +1,209 @@ +package org.apache.lucene.analysis; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; + +import java.io.IOException; + +/** + * Testcase for {@link ComboTokenStream}. + */ +public class TestComboTokenStream extends BaseTokenStreamTestCase { + + /** + * A TokenStream that takes the same input as the assertTokenStreamContents() function, + * and merely passes the corresponding assert. + */ + public final class ReplayTokenStream extends TokenStream { + + int index; + int length; + String[] outputs; + int[] positionIncrements; + int[] startOffsets; + int[] endOffsets; + CharTermAttribute output; + PositionIncrementAttribute positionIncrement; + OffsetAttribute offset; + + ReplayTokenStream(String[] outputs, int[] startOffsets, int[] endOffsets, int[] positionIncrements) { + index = 0; + this.outputs = outputs; + this.startOffsets = startOffsets; + this.endOffsets = endOffsets; + this.positionIncrements = positionIncrements; + if (outputs != null) { + this.length = outputs.length; + output = addAttribute(CharTermAttribute.class); + } else { + throw new NullPointerException("Outputs is null"); + } + if (startOffsets != null || endOffsets != null) { + if (startOffsets == null || startOffsets.length != length) throw new IllegalArgumentException("Bad startOffsets"); + if (endOffsets == null || endOffsets.length != length) throw new IllegalArgumentException("Bad endOffsets"); + offset = addAttribute(OffsetAttribute.class); + } + if (positionIncrements != null) { + if (positionIncrements.length != length) throw new IllegalArgumentException("Bad positionIncrements"); + positionIncrement = addAttribute(PositionIncrementAttribute.class); + } + } + + @Override + public final boolean incrementToken() throws IOException { + clearAttributes(); + if (index >= length) return false; + if (output != null) { + char[] buffer = outputs[index].toCharArray(); + output.copyBuffer(buffer, 0, buffer.length); + } + if (offset != null) + offset.setOffset(startOffsets[index], endOffsets[index]); + if (positionIncrement != null) + positionIncrement.setPositionIncrement(positionIncrements[index]); + index++; + return true; + } + + } + + + + public void testReplayTokenStream() throws IOException { + TokenStream ts = new ReplayTokenStream( + new String[]{"ab", "cd", "ef"}, + new int[]{ 0, 3, 5}, + new int[]{ 2, 4, 6}, + new int[]{ 1, 1, 1}); + assertTokenStreamContents(ts, + new String[]{"ab", "cd", "ef"}, + new int[]{ 0, 3, 5}, + new int[]{ 2, 4, 6}, + new int[]{ 1, 1, 1}); + } + + public void testSingleTokenStream() throws IOException { + ComboTokenStream cts = new ComboTokenStream( + new ReplayTokenStream( + new String[]{"ab", "cd", "ef"}, + new int[]{ 0, 3, 5}, + new int[]{ 2, 4, 6}, + new int[]{ 1, 1, 1}) + ); + CheckClearAttributesAttribute checkClearAtt = cts.addAttribute(CheckClearAttributesAttribute.class); + assertTokenStreamContents(cts, + new String[]{"ab", "cd", "ef"}, + new int[]{ 0, 3, 5}, + new int[]{ 2, 4, 6}, + new int[]{ 1, 1, 1}); + } + + public void testDoubleTokenStream() throws IOException { + ComboTokenStream cts = new ComboTokenStream( + new ReplayTokenStream( + new String[]{"ab", "cd", "ef"}, + new int[]{ 0, 3, 5}, + new int[]{ 2, 4, 6}, + new int[]{ 1, 1, 1}), + new ReplayTokenStream( + new String[]{"B", "D", "F"}, + new int[]{ 1, 4, 6}, + new int[]{ 2, 4, 6}, + new int[]{ 1, 1, 1}) + ); + assertTokenStreamContents(cts, + new String[]{"ab", "B", "cd", "D", "ef", "F"}, + new int[]{ 0, 1, 3, 4, 5, 6}, + new int[]{ 2, 2, 4, 4, 6, 6}, + new int[]{ 1, 0, 1, 0, 1, 0}); + // Now in reversed order + cts = new ComboTokenStream( + new ReplayTokenStream( + new String[]{"B", "D", "F"}, + new int[]{ 1, 4, 6}, + new int[]{ 2, 4, 6}, + new int[]{ 1, 1, 1}), + new ReplayTokenStream( + new String[]{"ab", "cd", "ef"}, + new int[]{ 0, 3, 5}, + new int[]{ 2, 4, 6}, + new int[]{ 1, 1, 1}) + ); + assertTokenStreamContents(cts, + new String[]{"ab", "B", "cd", "D", "ef", "F"}, + new int[]{ 0, 1, 3, 4, 5, 6}, + new int[]{ 2, 2, 4, 4, 6, 6}, + new int[]{ 1, 0, 1, 0, 1, 0}); + } + + public void testDoubleTokenStreamMultipleAtSamePosition() throws IOException { + ComboTokenStream cts = new ComboTokenStream( + new ReplayTokenStream( + new String[]{"ab", "cd", "ef"}, + new int[]{ 0, 3, 5}, + new int[]{ 2, 4, 6}, + new int[]{ 1, 1, 1}), + new ReplayTokenStream( + new String[]{"A", "B", "C", "D", "E", "F"}, + new int[]{ 0, 1, 3, 4, 5, 6}, + new int[]{ 1, 2, 3, 4, 5, 6}, + new int[]{ 1, 0, 1, 0, 1, 0}) + ); + if (ComboTokenStream.KEEP_STREAM_IF_SAME_POSITION) + assertTokenStreamContents(cts, + new String[]{"A", "B", "ab", "C", "D", "cd", "E", "F", "ef"}, + new int[]{ 0, 1, 0, 3, 4, 3, 5, 6, 5}, + new int[]{ 1, 2, 2, 3, 4, 4, 5, 6, 6}, + new int[]{ 1, 0, 0, 1, 0, 0, 1, 0, 0}); + else + assertTokenStreamContents(cts, + new String[]{"A", "ab", "B", "C", "cd", "D", "E", "ef", "F"}, + new int[]{ 0, 0, 1, 3, 3, 4, 5, 5, 6}, + new int[]{ 1, 2, 2, 3, 4, 4, 5, 6, 6}, + new int[]{ 1, 0, 0, 1, 0, 0, 1, 0, 0}); + // Now in reversed order + cts = new ComboTokenStream( + new ReplayTokenStream( + new String[]{"A", "B", "C", "D", "E", "F"}, + new int[]{ 0, 1, 3, 4, 5, 6}, + new int[]{ 1, 2, 3, 4, 5, 6}, + new int[]{ 1, 0, 1, 0, 1, 0}), + new ReplayTokenStream( + new String[]{"ab", "cd", "ef"}, + new int[]{ 0, 3, 5}, + new int[]{ 2, 4, 6}, + new int[]{ 1, 1, 1}) + ); + if (ComboTokenStream.KEEP_STREAM_IF_SAME_POSITION) + assertTokenStreamContents(cts, + new String[]{"A", "B", "ab", "C", "D", "cd", "E", "F", "ef"}, + new int[]{ 0, 1, 0, 3, 4, 3, 5, 6, 5}, + new int[]{ 1, 2, 2, 3, 4, 4, 5, 6, 6}, + new int[]{ 1, 0, 0, 1, 0, 0, 1, 0, 0}); + else + assertTokenStreamContents(cts, + new String[]{"A", "ab", "B", "C", "cd", "D", "E", "ef", "F"}, + new int[]{ 0, 0, 1, 3, 3, 4, 5, 5, 6}, + new int[]{ 1, 2, 2, 3, 4, 4, 5, 6, 6}, + new int[]{ 1, 0, 0, 1, 0, 0, 1, 0, 0}); + } + +} Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/PositionedTokenStream.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/PositionedTokenStream.java (révision 0) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/PositionedTokenStream.java (révision 0) @@ -0,0 +1,119 @@ +package org.apache.lucene.analysis; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; + +import java.io.IOException; + +/** + * A {@link TokenStream} wrapper that keeps track of + * the current term position of the given TokenStream, + * and defines a comparison order. + */ +public class PositionedTokenStream extends TokenFilter implements Comparable { + + // Attributes to track + private final OffsetAttribute offsetAttr; + private final PositionIncrementAttribute posAttr; + /** Position tracker. */ + private int position; + + public PositionedTokenStream(TokenStream input) { + super(input); + + // Force loading/adding these attributes + // won't do much bad if they're not read/written + offsetAttr = input.addAttribute(OffsetAttribute.class); + posAttr = input.addAttribute(PositionIncrementAttribute.class); + + this.position = 0; + } + + /** + * Returns the tracked current token position. + * @return The accumulated position increment attribute values. + */ + public int getPosition() { + return position; + } + + /* + * "TokenStream interface" + */ + + public final boolean incrementToken() throws IOException { + boolean rtn = input.incrementToken(); + if (!rtn) { + position = Integer.MAX_VALUE; + } + // Track accumulated position + position += posAttr.getPositionIncrement(); + return rtn; + } + + public void end() throws IOException { + input.end(); + position = 0; + } + + public void reset() throws IOException { + input.reset(); + position = 0; + } + + public void close() throws IOException { + input.close(); + position = 0; + } + + /** + * Permit ordering by reading order: term position, then term offsets (start, then end). + */ + public int compareTo(PositionedTokenStream that) { + // Nullity checks + if (that == null) + return 1; + // Position checks + if (this.position != that.position) + return this.position - that.position; + // TokenStream nullity checks + if (that.input == null) { + if (this.input == null) return 0; + else return 1; + } else if (this.input == null) return -1; + // Order by reading order, using offsets + if (this.offsetAttr != null && that.offsetAttr != null) { + int a = this.offsetAttr.startOffset(); + int b = that.offsetAttr.startOffset(); + if (a != b) { + return a-b; + } + a = this.offsetAttr.endOffset(); + b = that.offsetAttr.endOffset(); + return a-b; + } else if (that.offsetAttr == null) { + if (this.offsetAttr == null) return 0; + return 1; + } else { + return -1; + } + } + +} Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ComboAnalyzer.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ComboAnalyzer.java (révision 0) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ComboAnalyzer.java (révision 0) @@ -0,0 +1,134 @@ +package org.apache.lucene.analysis; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.util.CloseableThreadLocal; +import org.apache.lucene.util.ReaderCloneFactory; +import org.apache.lucene.util.Version; + +import java.io.IOException; +import java.io.Reader; +import java.util.Arrays; +import java.util.concurrent.atomic.AtomicReference; + +/** + * An analyzer that combines multiple sub-analyzers into one. + * + * It internally uses {@link ReaderCloneFactory} in order to feed the multiple + * sub-analyzers from a single input. + * If you analyzer big inputs or have a performance critical application, + * please see the remarks of the latter's documentation. + * + * The instances are thread safe with regards to the reused TokenStreams. + */ +public class ComboAnalyzer extends Analyzer { + + private Analyzer[] subAnalyzers; + private CloseableThreadLocal lastTokenStreams = new CloseableThreadLocal(); + private CloseableThreadLocal tempTokenStreams = new CloseableThreadLocal(); + private CloseableThreadLocal lastComboTokenStream = new CloseableThreadLocal(); + + public ComboAnalyzer(Version version, Analyzer... subAnalyzers) { + this.subAnalyzers = subAnalyzers; + } + + @Override public final TokenStream tokenStream(String fieldName, Reader originalReader) { + // Duplication of the original reader, to feed all sub-analyzers + ReaderCloneFactory.ReaderCloner readerCloner = null; + if (subAnalyzers.length <= 1) { + + // Can reuse the only reader we have, there will be no need of duplication + // Usage of the AtomicReference ensures that the same reader won't be duplicated. + ReaderCloneFactory.ReaderCloner useOnceReaderCloner = new ReaderCloneFactory.ReaderCloner() { + private AtomicReference singleUsageReference = null; + public void init(Reader originalReader) throws IOException { + singleUsageReference = new AtomicReference(originalReader); + } + public Reader giveAClone() { + return singleUsageReference.getAndSet(null); + } + }; + try { + useOnceReaderCloner.init(originalReader); + } catch (Throwable fail) { + useOnceReaderCloner = null; + } + readerCloner = useOnceReaderCloner; + + } else { + + readerCloner = ReaderCloneFactory.getCloner(originalReader); // internally uses the default "should always work" implementation + if (readerCloner == null) { + throw new IllegalArgumentException("Could not duplicate the original reader to feed multiple sub-readers"); + } + + } + + // We remember last used TokenStreams because many times Analyzers can provide a reusable TokenStream + // Detecting that all sub-TokenStreams are reusable permits to reuse our ComboTokenStream as well. + if (tempTokenStreams.get() == null) tempTokenStreams.set(new TokenStream[subAnalyzers.length]); // each time non reusability has been detected + if (lastTokenStreams.get() == null) lastTokenStreams.set(new TokenStream[subAnalyzers.length]); // only at first run + TokenStream[] tempTokenStreams_local = tempTokenStreams.get(); + TokenStream[] lastTokenStreams_local = lastTokenStreams.get(); + ComboTokenStream lastComboTokenStream_local = lastComboTokenStream.get(); + + // Get sub-TokenStreams from sub-analyzers + for (int i = subAnalyzers.length-1 ; i >= 0 ; --i) { + + // Feed the troll + Reader reader = readerCloner.giveAClone(); + // Try a reusable sub-TokenStream + try { + tempTokenStreams_local[i] = subAnalyzers[i].reusableTokenStream(fieldName, reader); + } catch (IOException ex) { + tempTokenStreams_local[i] = subAnalyzers[i].tokenStream(fieldName, reader); + } + // Detect non reusability + if (tempTokenStreams_local[i] != lastTokenStreams_local[i]) { + lastComboTokenStream_local = null; + } + + } + + // If last ComboTokenStream is not available create a new one + // This happens in the first call and in case of non reusability + if (lastComboTokenStream_local == null) { + // Clear old invalid references (preferred over allocating a new array) + Arrays.fill(lastTokenStreams_local, null); + // Swap temporary and last (non reusable) TokenStream references + lastTokenStreams.set(tempTokenStreams_local); + tempTokenStreams.set(lastTokenStreams_local); + // New ComboTokenStream to use + lastComboTokenStream_local = new ComboTokenStream(tempTokenStreams_local); + lastComboTokenStream.set(lastComboTokenStream_local); + } + return lastComboTokenStream_local; + } + + @Override + public final TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException { + return super.reusableTokenStream(fieldName, reader); + } + + @Override public void close() { + super.close(); + lastTokenStreams.close(); + tempTokenStreams.close(); + lastComboTokenStream.close(); + } +} Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ComboTokenStream.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ComboTokenStream.java (révision 0) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ComboTokenStream.java (révision 0) @@ -0,0 +1,184 @@ +package org.apache.lucene.analysis; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.util.Attribute; +import org.apache.lucene.util.AttributeImpl; +import org.apache.lucene.util.AttributeSource; + +import java.io.IOException; +import java.util.AbstractQueue; +import java.util.Iterator; +import java.util.PriorityQueue; + +/** + * A TokenStream combining the output of multiple sub-TokenStreams. + * + * This class copies the attributes from the last sub-TokenStream that + * was read from. If attributes are not uniform between sub-TokenStreams, + * extraneous attributes will stay untouched. + * + * @remark Copying is the only solution since most caller call + * get/addAttribute once and keep on reading the same + * updated instance returned the first (and only) time. + * Fortunately, {@link AttributeImpl}s have a method + * for giving their values to another instance. + */ +public class ComboTokenStream extends TokenStream { + + /** + * Whether or not to continue with the current TokenStream + * if it has multiple terms at same position, minimizing + * queue moves, or to enforce strip order (position, offsets) + */ + static final boolean KEEP_STREAM_IF_SAME_POSITION = false; + + private int lastPosition; + // Position tracked sub-TokenStreams + private final PositionedTokenStream[] positionedTokenStreams; + // Reading queue, using the reading order from PositionedTokenStream + private final AbstractQueue readQueue; + // Flag for lazy initialization and reset + private boolean readQueueResetted; + + public ComboTokenStream(TokenStream... tokenStreams) { + // Load the TokenStreams, track their position, and register their attributes + this.positionedTokenStreams = new PositionedTokenStream[tokenStreams.length]; + for (int i = tokenStreams.length-1 ; i >= 0 ; --i) { + if (tokenStreams[i] == null) continue; + this.positionedTokenStreams[i] = new PositionedTokenStream(tokenStreams[i]); + // Add each and every token seen in the current sub AttributeSource + Iterator> iterator = this.positionedTokenStreams[i].getAttributeClassesIterator(); + while (iterator.hasNext()) { + super.addAttribute(iterator.next()); + } + } + this.lastPosition = 0; + // Create an initially empty queue. + // It will be filled at first incrementToken() call, because + // it needs to call the same function on each sub-TokenStreams. + this.readQueue = new PriorityQueue(tokenStreams.length); + readQueueResetted = false; + } + + /* + * TokenStream multiplexed methods + */ + + @Override public final boolean incrementToken() throws IOException { + clearAttributes(); + + // Fill the queue on first call + if (!readQueueResetted) { + readQueueResetted = true; + readQueue.clear(); + for (PositionedTokenStream pts : positionedTokenStreams) { + if (pts == null) continue; + // Read first token + pts.clearAttributes(); + if (pts.incrementToken()) { + // PositionedTokenStream.incrementToken() initialized internal + // variables to perform proper ordering. + // Therefore we can only add it to the queue now! + readQueue.add(pts); + } // no token left (no token at all) + } + } + + // Read from the first token + PositionedTokenStream toRead = readQueue.peek(); + if (toRead == null) { + return false; // end of streams + } + // Look position to see if it will be increased, see usage a bit below + int pos = toRead.getPosition(); + + // Copy the current token attributes from the sub-TokenStream + // to our AttributeSource (see class javadoc remark) + AttributeSource currentAttributeSource = toRead; + Iterator> iter = toRead.getAttributeClassesIterator(); + while (iter.hasNext()) { + Class clazz = iter.next(); + @SuppressWarnings("unchecked") AttributeImpl attr = (AttributeImpl) currentAttributeSource.getAttribute(clazz); // forcefully an AttributeImpl, read Lucene source + if (this.hasAttribute(clazz)) { + @SuppressWarnings("unchecked") AttributeImpl attrLoc = (AttributeImpl) this.getAttribute(clazz); + attr.copyTo(attrLoc); + } // otherwise, leave untouched + } + // Override the PositionIncrementAttribute + this.getAttribute(PositionIncrementAttribute.class).setPositionIncrement(Math.max(0,pos - lastPosition)); + lastPosition = pos; + + // Prepare next read + // We did not remove the TokenStream from the queue yet, + // because if we have another token available at the same position, + // we can save a queue movement. + toRead.clearAttributes(); + if (!toRead.incrementToken()) { + // No more token to read, remove from the queue + readQueue.poll(); + } else { + // Check if token position changed + if (readQueue.size() > 1 && (!KEEP_STREAM_IF_SAME_POSITION || toRead.getPosition() != pos)) { + // If yes, re-enter in the priority queue + readQueue.add(readQueue.poll()); + } // Otherwise, next call will continue with the same TokenStream (less queue movements) + } + + return true; + } + + @Override public void end() throws IOException { + super.end(); + lastPosition = 0; + // Apply on each sub-TokenStream + for (PositionedTokenStream pts : positionedTokenStreams) { + if (pts == null) continue; + pts.end(); + } + readQueueResetted = false; + readQueue.clear(); + } + + @Override public void reset() throws IOException { + super.reset(); + super.clearAttributes(); + lastPosition = 0; + // Apply on each sub-TokenStream + for (PositionedTokenStream pts : positionedTokenStreams) { + if (pts == null) continue; + pts.reset(); + } + readQueueResetted = false; + readQueue.clear(); + } + + @Override public void close() throws IOException { + super.close(); + lastPosition = 0; + // Apply on each sub-TokenStream + for (PositionedTokenStream pts : positionedTokenStreams) { + if (pts == null) continue; + pts.close(); + } + readQueueResetted = false; + readQueue.clear(); + } + +} Index: lucene/src/test/org/apache/lucene/index/TestReusableStringReaderCloner.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestReusableStringReaderCloner.java (révision 0) +++ lucene/src/test/org/apache/lucene/index/TestReusableStringReaderCloner.java (révision 0) @@ -0,0 +1,82 @@ +package org.apache.lucene.index; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.analysis.BaseTokenStreamTestCase; +import org.apache.lucene.util.ReaderCloneFactory; +import org.apache.lucene.util.TestReaderCloneFactory; + +import java.io.IOException; +import java.io.Reader; + +/** + * Testcase for {@link ReusableStringReaderCloner} + */ +public class TestReusableStringReaderCloner extends BaseTokenStreamTestCase { + + public void testCloningReusableStringReader() throws IOException, ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InstantiationException { + // This test cannot be located inside TestReaderCloneFactory + // because of the ReusableStringReader class being package private + // (and it's a real pain to use Java reflection to gain access to + // a package private constructor) + Reader clone; + ReusableStringReader reader = new ReusableStringReader(); + reader.init("test string"); + ReaderCloneFactory.ReaderCloner cloner = ReaderCloneFactory.getCloner(reader); + assertNotNull(cloner); + assertEquals(cloner.getClass().getName(), ReusableStringReaderCloner.class.getName()); + clone = cloner.giveAClone(); + TestReaderCloneFactory.assertReaderContent(clone, "test string"); + clone = cloner.giveAClone(); + TestReaderCloneFactory.assertReaderContent(clone, "test string"); + + // Test reusability + ReaderCloneFactory.ReaderCloner forClassClonerStrict = ReaderCloneFactory.getClonerStrict(ReusableStringReader.class); + assertNotNull(forClassClonerStrict); + assertEquals(forClassClonerStrict.getClass().getName(), ReusableStringReaderCloner.class.getName()); + reader.init("another test string"); + forClassClonerStrict.init(reader); + clone = forClassClonerStrict.giveAClone(); + TestReaderCloneFactory.assertReaderContent(clone, "another test string"); + clone = forClassClonerStrict.giveAClone(); + TestReaderCloneFactory.assertReaderContent(clone, "another test string"); + reader.init("test string"); + forClassClonerStrict.init(reader); + clone = forClassClonerStrict.giveAClone(); + TestReaderCloneFactory.assertReaderContent(clone, "test string"); + clone = forClassClonerStrict.giveAClone(); + TestReaderCloneFactory.assertReaderContent(clone, "test string"); + + ReaderCloneFactory.ReaderCloner forClassCloner = ReaderCloneFactory.getCloner(ReusableStringReader.class); + assertNotNull(forClassCloner); + assertEquals(forClassCloner.getClass().getName(), ReusableStringReaderCloner.class.getName()); + reader.init("another test string"); + forClassCloner.init(reader); + clone = forClassCloner.giveAClone(); + TestReaderCloneFactory.assertReaderContent(clone, "another test string"); + clone = forClassCloner.giveAClone(); + TestReaderCloneFactory.assertReaderContent(clone, "another test string"); + reader.init("test string"); + forClassCloner.init(reader); + clone = forClassCloner.giveAClone(); + TestReaderCloneFactory.assertReaderContent(clone, "test string"); + clone = forClassCloner.giveAClone(); + TestReaderCloneFactory.assertReaderContent(clone, "test string"); + } + +} Index: lucene/src/test/org/apache/lucene/util/TestReaderCloneFactory.java =================================================================== --- lucene/src/test/org/apache/lucene/util/TestReaderCloneFactory.java (révision 0) +++ lucene/src/test/org/apache/lucene/util/TestReaderCloneFactory.java (révision 0) @@ -0,0 +1,244 @@ +package org.apache.lucene.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.BufferedReader; +import java.io.CharArrayReader; +import java.io.FilterReader; +import java.io.IOException; +import java.io.PushbackReader; +import java.io.Reader; +import java.io.StringReader; + +/** + * Testcase for {@link ReaderCloneFactory}. + */ +public class TestReaderCloneFactory extends LuceneTestCase { + + public static void assertReaderContent(Reader reader, String content) throws IOException { + int len = content.length(); + int index = 0; + int read; + while (index < len && (read = reader.read()) != -1) + assertEquals(read, content.charAt(index++)); + assertEquals(index, len); + reader.close(); + } + + public void testCloningStringReader() throws IOException { + StringReader reader = new StringReader("test string"); + ReaderCloneFactory.ReaderCloner cloner = ReaderCloneFactory.getCloner(reader); + assertNotNull(cloner); + assertEquals(cloner.getClass().getName(), StringReaderCloner.class.getName()); + Reader clone; + clone = cloner.giveAClone(); + assertReaderContent(clone, "test string"); + clone = cloner.giveAClone(); + assertReaderContent(clone, "test string"); + + // Test reusability + ReaderCloneFactory.ReaderCloner forClassClonerStrict = ReaderCloneFactory.getClonerStrict(StringReader.class); + assertNotNull(forClassClonerStrict); + assertEquals(forClassClonerStrict.getClass().getName(), StringReaderCloner.class.getName()); + forClassClonerStrict.init(new StringReader("test string")); + clone = forClassClonerStrict.giveAClone(); + assertReaderContent(clone, "test string"); + clone = forClassClonerStrict.giveAClone(); + assertReaderContent(clone, "test string"); + forClassClonerStrict.init(new StringReader("another test string")); + clone = forClassClonerStrict.giveAClone(); + assertReaderContent(clone, "another test string"); + clone = forClassClonerStrict.giveAClone(); + assertReaderContent(clone, "another test string"); + + ReaderCloneFactory.ReaderCloner forClassCloner = ReaderCloneFactory.getCloner(StringReader.class); + assertNotNull(forClassCloner); + assertEquals(forClassCloner.getClass().getName(), StringReaderCloner.class.getName()); + forClassCloner.init(new StringReader("test string")); + clone = forClassCloner.giveAClone(); + assertReaderContent(clone, "test string"); + clone = forClassCloner.giveAClone(); + assertReaderContent(clone, "test string"); + forClassCloner.init(new StringReader("another test string")); + clone = forClassCloner.giveAClone(); + assertReaderContent(clone, "another test string"); + clone = forClassCloner.giveAClone(); + assertReaderContent(clone, "another test string"); + } + + public void testCloningCharArrayReader() throws IOException { + CharArrayReader reader = new CharArrayReader("test string".toCharArray()); + ReaderCloneFactory.ReaderCloner cloner = ReaderCloneFactory.getCloner(reader); + assertNotNull(cloner); + assertEquals(cloner.getClass().getName(), CharArrayReaderCloner.class.getName()); + Reader clone; + clone = cloner.giveAClone(); + assertReaderContent(clone, "test string"); + clone = cloner.giveAClone(); + assertReaderContent(clone, "test string"); + + // Test reusability + ReaderCloneFactory.ReaderCloner forClassClonerStrict = ReaderCloneFactory.getClonerStrict(CharArrayReader.class); + assertNotNull(forClassClonerStrict); + assertEquals(cloner.getClass().getName(), CharArrayReaderCloner.class.getName()); + forClassClonerStrict.init(new CharArrayReader("test string".toCharArray())); + clone = forClassClonerStrict.giveAClone(); + assertReaderContent(clone, "test string"); + clone = forClassClonerStrict.giveAClone(); + assertReaderContent(clone, "test string"); + forClassClonerStrict.init(new CharArrayReader("another test string".toCharArray())); + clone = forClassClonerStrict.giveAClone(); + assertReaderContent(clone, "another test string"); + clone = forClassClonerStrict.giveAClone(); + assertReaderContent(clone, "another test string"); + + ReaderCloneFactory.ReaderCloner forClassCloner = ReaderCloneFactory.getCloner(CharArrayReader.class); + assertNotNull(forClassCloner); + assertEquals(cloner.getClass().getName(), CharArrayReaderCloner.class.getName()); + forClassCloner.init(new CharArrayReader("test string".toCharArray())); + clone = forClassCloner.giveAClone(); + assertReaderContent(clone, "test string"); + clone = forClassCloner.giveAClone(); + assertReaderContent(clone, "test string"); + forClassCloner.init(new CharArrayReader("another test string".toCharArray())); + clone = forClassCloner.giveAClone(); + assertReaderContent(clone, "another test string"); + clone = forClassCloner.giveAClone(); + assertReaderContent(clone, "another test string"); + } + + public void testCloningBufferedStringReader() throws IOException { + // The (useless) BufferedReader should be unwrapped, and a StringReaderCloner should be returned + StringReader stringReader = new StringReader("test string"); + BufferedReader reader = new BufferedReader(stringReader); + ReaderCloneFactory.ReaderCloner cloner = ReaderCloneFactory.getCloner(reader); + assertNotNull(cloner); + assertEquals(cloner.getClass().getName(), StringReaderCloner.class.getName()); + Reader clone; + clone = cloner.giveAClone(); + assertReaderContent(clone, "test string"); + clone = cloner.giveAClone(); + assertReaderContent(clone, "test string"); + + // Test reusability (does not use unwrapping) + ReaderCloneFactory.ReaderCloner forClassClonerStrict = ReaderCloneFactory.getClonerStrict(BufferedReader.class); + assertNull(forClassClonerStrict); + + ReaderCloneFactory.ReaderCloner forClassCloner = ReaderCloneFactory.getCloner(BufferedReader.class); + assertNotNull(forClassCloner); + assertEquals(forClassCloner.getClass().getName(), ReaderClonerDefaultImpl.class.getName()); + forClassCloner.init(new BufferedReader(new StringReader("test string"))); + clone = forClassCloner.giveAClone(); + assertReaderContent(clone, "test string"); + clone = forClassCloner.giveAClone(); + assertReaderContent(clone, "test string"); + forClassCloner.init(new BufferedReader(new StringReader("another test string"))); + clone = forClassCloner.giveAClone(); + assertReaderContent(clone, "another test string"); + clone = forClassCloner.giveAClone(); + assertReaderContent(clone, "another test string"); + } + + public void testCloningFilterStringReader() throws IOException { + // The (useless) FilterReader should be unwrapped, and a StringReaderCloner should be returned + StringReader stringReader = new StringReader("test string"); + FilterReader reader = new PushbackReader(stringReader); + ReaderCloneFactory.ReaderCloner cloner = ReaderCloneFactory.getCloner(reader); + assertNotNull(cloner); + assertEquals(cloner.getClass().getName(), StringReaderCloner.class.getName()); + Reader clone; + clone = cloner.giveAClone(); + assertReaderContent(clone, "test string"); + clone = cloner.giveAClone(); + assertReaderContent(clone, "test string"); + + // Test reusability (does not use unwrapping) + ReaderCloneFactory.ReaderCloner forClassClonerStrict = ReaderCloneFactory.getClonerStrict(FilterReader.class); + assertNull(forClassClonerStrict); + + ReaderCloneFactory.ReaderCloner forClassCloner = ReaderCloneFactory.getCloner(FilterReader.class); + assertNotNull(forClassCloner); + assertEquals(forClassCloner.getClass().getName(), ReaderClonerDefaultImpl.class.getName()); + forClassCloner.init(new BufferedReader(new StringReader("test string"))); + clone = cloner.giveAClone(); + assertReaderContent(clone, "test string"); + clone = cloner.giveAClone(); + assertReaderContent(clone, "test string"); + forClassCloner.init(new BufferedReader(new StringReader("another test string"))); + clone = forClassCloner.giveAClone(); + assertReaderContent(clone, "another test string"); + clone = forClassCloner.giveAClone(); + assertReaderContent(clone, "another test string"); + } + + public void testCloningAnonymousReader() throws IOException { + final StringReader delegated1 = new StringReader("test string"); + Reader reader = new Reader() { + @Override public int read(char[] cbuf, int off, int len) throws IOException { + return delegated1.read(cbuf, off, len); + } + @Override public void close() throws IOException { + delegated1.close(); + } + }; + ReaderCloneFactory.ReaderCloner cloner = ReaderCloneFactory.getCloner(reader); + assertEquals(cloner.getClass().getName(), ReaderClonerDefaultImpl.class.getName()); + Reader clone; + clone = cloner.giveAClone(); + assertReaderContent(clone, "test string"); + clone = cloner.giveAClone(); + assertReaderContent(clone, "test string"); + + // Test reusability (does not use unwrapping) + ReaderCloneFactory.ReaderCloner/**/ forClassClonerStrict = ReaderCloneFactory.getClonerStrict(reader.getClass()); + assertNull(forClassClonerStrict); + + ReaderCloneFactory.ReaderCloner forClassCloner = ReaderCloneFactory.getCloner(reader.getClass()); + assertNotNull(forClassCloner); + assertEquals(forClassCloner.getClass().getName(), ReaderClonerDefaultImpl.class.getName()); + final StringReader delegated2 = new StringReader("test string"); + Reader reader2 = new Reader() { + @Override public int read(char[] cbuf, int off, int len) throws IOException { + return delegated2.read(cbuf, off, len); + } + @Override public void close() throws IOException { + delegated2.close(); + } + }; + forClassCloner.init(reader2); + clone = cloner.giveAClone(); + assertReaderContent(clone, "test string"); + clone = cloner.giveAClone(); + assertReaderContent(clone, "test string"); + final StringReader delegated3 = new StringReader("another test string"); + Reader reader3 = new Reader() { + @Override public int read(char[] cbuf, int off, int len) throws IOException { + return delegated3.read(cbuf, off, len); + } + @Override public void close() throws IOException { + delegated3.close(); + } + }; + forClassCloner.init(reader3); + clone = forClassCloner.giveAClone(); + assertReaderContent(clone, "another test string"); + clone = forClassCloner.giveAClone(); + assertReaderContent(clone, "another test string"); + } + +} Index: lucene/src/java/org/apache/lucene/index/ReusableStringReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/ReusableStringReader.java (révision 1160156) +++ lucene/src/java/org/apache/lucene/index/ReusableStringReader.java (copie de travail) @@ -25,7 +25,7 @@ final class ReusableStringReader extends Reader { int upto; int left; - String s; + String s; // watch for ReusableStringReaderCloner's access to this content void init(String s) { this.s = s; left = s.length(); Index: lucene/src/java/org/apache/lucene/index/ReusableStringReaderCloner.java =================================================================== --- lucene/src/java/org/apache/lucene/index/ReusableStringReaderCloner.java (révision 0) +++ lucene/src/java/org/apache/lucene/index/ReusableStringReaderCloner.java (révision 0) @@ -0,0 +1,75 @@ +package org.apache.lucene.index; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.util.ReaderCloneFactory; + +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; + +/** + * A ReaderCloner specialized in duplicating Lucene's {@link org.apache.lucene.index.ReusableStringReader}. + * + * As this class is package private, this cloner has an additional function + * to perform an {@code instanceof} check for you. + * + * The implementation exploits the fact that ReusableStringReader has a package + * private field {@code String s}, storing the original content. + * It is therefore sensitive to Lucene implementation changes. + */ +public class ReusableStringReaderCloner implements ReaderCloneFactory.ReaderCloner { + + private ReusableStringReader original; + private String originalContent; + + /** + * Binds this ReaderCloner with the package-private {@link ReusableStringReader} class + * into the {@link ReaderCloneFactory}, without giving access to the hidden class. + */ + public static void registerCloner() { + ReaderCloneFactory.bindCloner(ReusableStringReader.class, ReusableStringReaderCloner.class); + } + + /** + * @param original Must pass the canHandleReader(Reader) test, otherwise an IllegalArgumentException will be thrown. + */ + public void init(ReusableStringReader original) throws IOException { + this.original = original; + this.originalContent = null; + try { + // Exploit package private access to the original String + this.originalContent = original.s; + } catch (Throwable ex) { // Extra sanity check in case the implementation changes, and this class still can be used + throw new IllegalArgumentException("The org.apache.lucene.index.ReusableStringReader no longer propose an access to the package private String s field, please consider updating this class to a newer version or use a fallback ReaderCloner"); + } + } + + /** + * First call will return the original Reader provided. + */ + public Reader giveAClone() { + if (original != null) { + Reader rtn = original; + original = null; // no longer hold a reference + return rtn; + } + return new StringReader(originalContent); + } + +} Index: lucene/src/java/org/apache/lucene/util/FilterReaderUnwrapper.java =================================================================== --- lucene/src/java/org/apache/lucene/util/FilterReaderUnwrapper.java (révision 0) +++ lucene/src/java/org/apache/lucene/util/FilterReaderUnwrapper.java (révision 0) @@ -0,0 +1,50 @@ +package org.apache.lucene.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.FilterReader; +import java.io.Reader; +import java.lang.reflect.Field; + +/** + * A {@link java.io.FilterReader} ReaderUnwrapper that + * returns the Reader wrapped inside the FilterReader + * (and all its subclasses) + */ +public class FilterReaderUnwrapper implements ReaderCloneFactory.ReaderUnwrapper { + + private static Field internalField; + + static { + try { + internalField = FilterReader.class.getDeclaredField("in"); + internalField.setAccessible(true); + } catch (Exception ex) { + throw new IllegalArgumentException("Could not give accessibility to private \"in\" field of the given FilterReader", ex); + } + } + + public Reader unwrap(FilterReader originalReader) throws IllegalArgumentException { + try { + return (Reader) internalField.get(originalReader); + } catch (Exception ex) { + throw new IllegalArgumentException("Could not access private \"in\" field of the given FilterReader (actual class: "+originalReader.getClass().getCanonicalName()+")", ex); + } + } + +} Index: lucene/src/java/org/apache/lucene/util/StringReaderCloner.java =================================================================== --- lucene/src/java/org/apache/lucene/util/StringReaderCloner.java (révision 0) +++ lucene/src/java/org/apache/lucene/util/StringReaderCloner.java (révision 0) @@ -0,0 +1,75 @@ +package org.apache.lucene.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.lang.reflect.Field; + +/** + * A ReaderCloner specialized for StringReader. + * + * The only efficient mean of retrieving the original content + * from a StringReader is to use introspection and access the + * {@code private String str} field. + * + * Apart from being efficient, this code is also very sensitive + * to the used JVM implementation. + * If the introspection does not work, an {@link IllegalArgumentException} + * is thrown. + */ +public class StringReaderCloner implements ReaderCloneFactory.ReaderCloner { + + private static Field internalField; + + private StringReader original; + private String originalContent; + + static { + try { + internalField = StringReader.class.getDeclaredField("str"); + internalField.setAccessible(true); + } catch (Exception ex) { + throw new IllegalArgumentException("Could not give accessibility to private \"str\" field of the given StringReader", ex); + } + } + + public void init(StringReader originalReader) throws IOException { + this.originalContent = null; + try { + this.original = originalReader; + this.originalContent = (String) internalField.get(original); + } catch (Exception ex) { + throw new IllegalArgumentException("Could not access private \"str\" field of the given StringReader (actual class: "+original.getClass().getCanonicalName()+")", ex); + } + } + + /** + * First call will return the original Reader provided. + */ + public Reader giveAClone() { + if (original != null) { + Reader rtn = original; + original = null; // no longer hold a reference + return rtn; + } + return new StringReader(originalContent); + } + +} Index: lucene/src/java/org/apache/lucene/util/CharArrayReaderCloner.java =================================================================== --- lucene/src/java/org/apache/lucene/util/CharArrayReaderCloner.java (révision 0) +++ lucene/src/java/org/apache/lucene/util/CharArrayReaderCloner.java (révision 0) @@ -0,0 +1,75 @@ +package org.apache.lucene.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.CharArrayReader; +import java.io.IOException; +import java.io.Reader; +import java.lang.reflect.Field; + +/** + * A ReaderCloner specialized for CharArrayReader. + * + * The only efficient mean of retrieving the original content + * from a CharArrayReader is to use introspection and access the + * {@code private String str} field. + * + * Apart from being efficient, this code is also very sensitive + * to the used JVM implementation. + * If the introspection does not work, an {@link IllegalArgumentException} + * is thrown. + */ +public class CharArrayReaderCloner implements ReaderCloneFactory.ReaderCloner { + + private static Field internalField; + + private CharArrayReader original; + private char[] originalContent; + + static { + try { + internalField = CharArrayReader.class.getDeclaredField("buf"); + internalField.setAccessible(true); + } catch (Exception ex) { + throw new IllegalArgumentException("Could not give accessibility to private \"buf\" field of the given CharArrayReader", ex); + } + } + + public void init(CharArrayReader originalReader) throws IOException { + this.original = originalReader; + this.originalContent = null; + try { + this.originalContent = (char[]) internalField.get(original); + } catch (Exception ex) { + throw new IllegalArgumentException("Could not access private \"buf\" field of the given CharArrayReader (actual class: "+original.getClass().getCanonicalName()+")", ex); + } + } + + /** + * First call will return the original Reader provided. + */ + public Reader giveAClone() { + if (original != null) { + Reader rtn = original; + original = null; // no longer hold a reference + return rtn; + } + return new CharArrayReader(originalContent); + } + +} Index: lucene/src/java/org/apache/lucene/util/BufferedReaderUnwrapper.java =================================================================== --- lucene/src/java/org/apache/lucene/util/BufferedReaderUnwrapper.java (révision 0) +++ lucene/src/java/org/apache/lucene/util/BufferedReaderUnwrapper.java (révision 0) @@ -0,0 +1,49 @@ +package org.apache.lucene.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.BufferedReader; +import java.io.Reader; +import java.lang.reflect.Field; + +/** + * A {@link java.io.BufferedReader} ReaderUnwrapper that + * returns the Reader wrapped inside the BufferReader. + */ +public class BufferedReaderUnwrapper implements ReaderCloneFactory.ReaderUnwrapper { + + private static Field internalField; + + static { + try { + internalField = BufferedReader.class.getDeclaredField("in"); + internalField.setAccessible(true); + } catch (Exception ex) { + throw new IllegalArgumentException("Could not give accessibility to private \"in\" field of the given BufferedReader", ex); + } + } + + public Reader unwrap(BufferedReader originalReader) throws IllegalArgumentException { + try { + return (Reader) internalField.get(originalReader); + } catch (Exception ex) { + throw new IllegalArgumentException("Could not access private \"in\" field of the given BufferedReader (actual class: "+originalReader.getClass().getCanonicalName()+")", ex); + } + } + +} Index: lucene/src/java/org/apache/lucene/util/ReaderCloneFactory.java =================================================================== --- lucene/src/java/org/apache/lucene/util/ReaderCloneFactory.java (révision 0) +++ lucene/src/java/org/apache/lucene/util/ReaderCloneFactory.java (révision 0) @@ -0,0 +1,308 @@ +package org.apache.lucene.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.ReusableStringReaderCloner; + +import java.io.BufferedReader; +import java.io.CharArrayReader; +import java.io.FilterReader; +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.lang.ref.WeakReference; +import java.util.WeakHashMap; + +/** + * Duplicates {@link Reader}s in order to feed multiple consumers. + * + * This class registers multiple implementations, and tries to resolve which one to use, + * looking at the actual class of the Reader to clone, and matching with the most bond + * handled classes for each {@link ReaderCloner} implementation. + * + * By default, a few {@link Reader} implementations are handled, including the + * most used inside Lucene ({@link StringReader}), and a default, fallback implementation + * that merely reads all the available content, and creates a String out of it. + * + * Therefore you should understand the importance of having a proper implementation for + * any optimizable {@link Reader}. For instance, {@link StringReaderCloner} gains access + * to the underlying String in order to avoid copies. A generic BufferedReader + */ +public class ReaderCloneFactory { + + /** + * Interface for a utility class, able to unwrap a {@link java.io.Reader} + * inside another {@link Reader}. + * @param The base class handled. + */ + public static interface ReaderUnwrapper { + /** + * Unwraps a {@link Reader} from another, simplifying an eventual chain. + */ + public Reader unwrap(T originalReader) throws IllegalArgumentException; + } + + /** + * Interface for a utility class, able to clone the content of a {@link java.io.Reader}, + * possibly in an optimized way (such as gaining access to a package private field, + * or through reflection using {@link java.lang.reflect.Field.setAccessible(boolean)}). + * @param The base class handled. + */ + public static interface ReaderCloner { + /** + * Initialize or reinitialize the cloner with the given reader. + * The implementing class should have a default no arguments constructor. + * @remark The given Reader is now controlled by this ReaderCloner, it may + * be closed during a call to this method, or it may be returned + * at first call to {@link giveAClone()}. + * @see giveAClone() + */ + public void init(T originalReader) throws IOException; + + /** + * Returns a new {@link Reader}. + * @remark The returned Reader should be closed. + * The original Reader, if not consumed by the {@link init(T)} method, + * should be returned at first call. Therefore it is important to + * call this method at least once, or to be prepared to face possible + * exceptions when closing the original Reader. + */ + public Reader giveAClone(); + } + + /** Map storing the mapping between a handled class and a handling class, for {@link ReaderCloner}s */ + private static final WeakHashMap, WeakReference>> typeMap = + new WeakHashMap, WeakReference>>(); + /** Map storing the mapping between a handled class and a handling instance, for {@link ReaderUnwrapper}s */ + private static final WeakHashMap, WeakReference> unwrapperTypeMap = + new WeakHashMap, WeakReference>(); + + /** + * Add the association between a (handled) class and its handling {@link ReaderCloner}. + * @param handledClass The base class that is handled by clonerImplClass. + * Using this parameter, you can further restrict the usage of a more generic cloner. + * @param clonerImplClass The class of the associated cloner. + * @param The base handled class of the ReaderCloner. + * @return The previously associated ReaderCloner for the handledClass. + */ + public static WeakReference> bindCloner( + Class handledClass, Class> clonerImplClass) { + return typeMap.put(handledClass, new WeakReference>(clonerImplClass)); + } + + /** + * Add the association between a (handled) class and its handling {@link ReaderUnwrapper} instance. + * @param handledClass The base class that is handled by clonerImplClass. + * Using this parameter, you can further restrict the usage of a more generic cloner. + * @param unwrapperImpl The instance of the associated unwrapper. + * @param The base handled class of the ReaderUnwrapper. + * @return The previously associated ReaderUnwrapper instance for the handledClass. + */ + public static WeakReference bindUnwrapper( + Class handledClass, ReaderUnwrapper unwrapperImpl) { + return unwrapperTypeMap.put(handledClass, new WeakReference(unwrapperImpl)); + } + + /** + * Static initialization registering default associations + */ + static { + // General purpose Reader handling + bindCloner(Reader.class, ReaderClonerDefaultImpl.class); + bindUnwrapper(BufferedReader.class, new BufferedReaderUnwrapper()); + bindUnwrapper(FilterReader.class, new FilterReaderUnwrapper()); + // Often used Java Readers + bindCloner(StringReader.class, StringReaderCloner.class); // very, very used inside Lucene + bindCloner(CharArrayReader.class, CharArrayReaderCloner.class); + // Lucene specific handling + ReusableStringReaderCloner.registerCloner(); + } + + /** + * (Expert) Returns the ReaderUnwrapper associated with the exact given class. + * @param forClass The handled class bond to the ReaderUnwrapper to return. + * @param The base handled class of the ReaderUnwrapper to return. + * @return The bond ReaderUnwrapper, or null. + */ + public static ReaderUnwrapper getUnwrapperStrict(Class forClass) { + WeakReference refUnwrapper = unwrapperTypeMap.get(forClass); + if (refUnwrapper != null) + return refUnwrapper.get(); + return null; + } + + /** + * Returns the ReaderCloner associated with the exact given class. + * @param forClass The handled class bond to the ReaderCloner to return. + * @param The base handled class of the ReaderCloner to return. + * @return The bond ReaderCloner, or null. + */ + public static ReaderCloner getClonerStrict(Class forClass) { + WeakReference> refClonerClass = typeMap.get(forClass); + if (refClonerClass != null) { + Class clazz = refClonerClass.get(); + if (clazz != null) { + try { + ReaderCloner cloner = (ReaderCloner) clazz.newInstance(); + return cloner; + } catch (Throwable ignored) { + } + } + } + return null; + } + + /** + * (Advanced) Returns an initialized ReaderCloner, associated with the exact class of the given Reader. + * If the initialization fails, this function returns null. + * @param forReader The handled class bond to the ReaderCloner to return. + * @param The base handled class of the ReaderCloner to return. + * @return The bond, initialized ReaderCloner, or null. + */ + public static ReaderCloner getClonerStrict(T forReader) { + ReaderCloner rtn = ReaderCloneFactory.getClonerStrict((Class) forReader.getClass()); + if (rtn != null) { + try { + rtn.init(forReader); + } catch (Throwable fail) { + return null; + } + } + return rtn; + } + + /** + * (Advanced) Returns an initialized ReaderCloner, associated with the given base class, for the given Reader. + * If the initialization fails, this function returns null. + * + * The function first tries to match the exact class of forReader, and initialize the ReaderCloner. + * If no ReaderCloner or (tested second) ReaderUnwrapper matches, the resolution continues with the super class, + * until the baseClass is reached, and tested. + * + * If this process is not successful, null is returned. + * + * @param baseClass The baseClass, above which the resolution will not try to continue with the super class. + * @param forClass The class to start with, should be the class of forReader (but the latter can be null, hence this parameter) + * @param forReader The Reader instance to return and initialize a ReaderCloner for. Can be null. + * @param The base handled class of the ReaderCloner to return + * @param The class of the given Reader to handle + * @return An initialized ReaderCloner suitable for the givenReader, or null. + */ + public static ReaderCloner getCloner(Class baseClass, Class forClass, S forReader) { + // Loop through each super class + while (forClass != null) { + // Try first a matching cloner + ReaderCloner cloner = ReaderCloneFactory.getClonerStrict(forClass); + if (cloner != null) { + if (forReader != null) { + try { + cloner.init(forReader); + } catch (Throwable fail) { + cloner = null; + } + } + if (cloner != null) + return cloner; + } + // Try then a matching unwrapper, for better suitability of the used cloner + if (forReader != null) { + ReaderUnwrapper unwrapper = ReaderCloneFactory.getUnwrapperStrict(forClass); + if (unwrapper != null) + try { + // Recursive resolution + Reader unwrapped = unwrapper.unwrap(forReader); + if (unwrapped != null) + return (ReaderCloner)ReaderCloneFactory.getCloner(Reader.class, (Class)unwrapped.getClass(), unwrapped); + } catch (Throwable ignore) { + // in case of errors, simply continue the began process and forget about this failed attempt + } + } + // Continue resolution with super class... + Class clazz = forClass.getSuperclass(); + // ... checking ancestry with the given base class + forClass = null; + if (baseClass.isAssignableFrom(clazz)) + forClass = clazz; + } + return null; + } + + /** + * Returns a ReaderCloner suitable for handling general Ss instances (inheriting T, itself + * inheriting {@link java.io.Reader}). + * + * Resolution starts on forClass (S), and does not go further than baseClass. + * + * Not all optimizations can be ran, like unwrapping and failing initialization fallback. + * However, for standard cases, when performance is really critical, + * using this function can reduce a possible resolution overhead + * because ReaderCloner are reusable. + * + * @param baseClass The baseClass, above which the resolution will not try to continue with the super class. + * @param forClass The class to start with, should be the class of forReader (but the latter can be null, hence this parameter) + * @param The base handled class of the ReaderCloner to return + * @param The class of the given Reader to handle + * @return An uninitialized ReaderCloner suitable for any T Readers, or null. + */ + public static ReaderCloner getCloner(Class baseClass, Class forClass) { + return ReaderCloneFactory.getCloner(baseClass, forClass, null); + } + + /** + * Returns a ReaderCloner suitable for handling general Ss instances (inheriting {@link java.io.Reader}). + * + * Calls ReaderCloneFactory.getCloner(Reader.class, forClass, (S)null). + * + * Not all optimizations can be ran, like unwrapping and failing initialization fallback. + * However, for standard cases, when performance is really critical, + * using this function can reduce a possible resolution overhead + * because ReaderCloner are reusable. + * + * @param forClass The class to start with, should be the class of forReader (but the latter can be null, hence this parameter) + * @param The class of the given Reader to handle + * @return An uninitialized ReaderCloner suitable for any S, or null. + */ + public static ReaderCloner getCloner(Class forClass) { + return ReaderCloneFactory.getCloner(Reader.class, forClass, (S)null); + } + + /** + * Returns an initialized ReaderCloner, for the given Reader. + * + * Calls ReaderCloneFactory.getCloner(Reader.class, (Class)forReader.getClass(), forReader). + * If forReader is null, works as {@link ReaderCloneFactory.getGenericCloner()}. + * + * @param forReader The Reader instance to return and initialize a ReaderCloner for. Can be null. + * @param The class of the given Reader + * @return An initialized ReaderCloner suitable for given Reader, or null. + */ + public static ReaderCloner getCloner(S forReader) { + if (forReader != null) + return ReaderCloneFactory.getCloner(Reader.class, (Class)forReader.getClass(), forReader); + else + return ReaderCloneFactory.getGenericCloner(); + } + + /** + * Returns a {@link ReaderCloner} suitable for any {@link java.io.Reader} instance. + */ + public static ReaderCloner getGenericCloner() { + return ReaderCloneFactory.getCloner(Reader.class, Reader.class, (Reader)null); + } + +} Index: lucene/src/java/org/apache/lucene/util/ReaderClonerDefaultImpl.java =================================================================== --- lucene/src/java/org/apache/lucene/util/ReaderClonerDefaultImpl.java (révision 0) +++ lucene/src/java/org/apache/lucene/util/ReaderClonerDefaultImpl.java (révision 0) @@ -0,0 +1,95 @@ +package org.apache.lucene.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; + +/** + * Default, memory costly but generic implementation of a {@link java.io.Reader} duplicator. + * + * This implementation makes no assumption on the initial Reader. + * Therefore, only the read() functions are available to figure out + * what was the original content provided to the initial Reader. + * + * After having read and filled a buffer with the whole content, + * a String-based Reader implementation will be used and returned. + * + * This implementation is memory costly because the initial content is + * forcefully duplicated once. Moreover, buffer size growth may cost + * some more memory too. + */ +public class ReaderClonerDefaultImpl implements ReaderCloneFactory.ReaderCloner { + + public static final int DEFAULT_INITIAL_CAPACITY = 64 * 1024; + public static final int DEFAULT_READ_BUFFER_SIZE = 16 * 1024; + + protected int initialCapacity; + protected int readBufferSize; + + private String originalContent; + + public ReaderClonerDefaultImpl() { + this(DEFAULT_INITIAL_CAPACITY, DEFAULT_READ_BUFFER_SIZE); + } + + public ReaderClonerDefaultImpl(int initialCapacity) { + this(initialCapacity, DEFAULT_READ_BUFFER_SIZE); + } + + /** + * Extracts the original content from a generic Reader instance + * by repeatedly calling {@link Reader.read(char[])} on it, + * feeding a {@link StringBuilder}. + * + * @param initialCapacity Initial StringBuilder capacity + * @param readBufferSize Size of the char[] read buffer at each read() call + * @throws IOException + */ + public ReaderClonerDefaultImpl(int initialCapacity, int readBufferSize) { + this.initialCapacity = initialCapacity; + this.readBufferSize = readBufferSize; + } + + public void init(Reader originalReader) throws IOException { + this.originalContent = null; + StringBuilder sb = null; + if (initialCapacity < 0) + sb = new StringBuilder(); + else + sb = new StringBuilder(initialCapacity); + char[] buffer = new char[readBufferSize]; + int read = -1; + while((read = originalReader.read(buffer)) != -1){ + sb.append(buffer, 0, read); + } + this.originalContent = sb.toString(); + originalReader.close(); + } + + /** + * Returns a new {@link StringReader} instance, + * directly based on the extracted original content. + * @return A {@link StringReader} + */ + public Reader giveAClone() { + return new StringReader(originalContent); + } + +}