Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountAnalyzer.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountAnalyzer.java (revision 1513769) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountAnalyzer.java (working copy) @@ -46,6 +46,7 @@ * @param consumeAllTokens whether all tokens from the delegate should be consumed even if maxTokenCount is reached. */ public LimitTokenCountAnalyzer(Analyzer delegate, int maxTokenCount, boolean consumeAllTokens) { + super(delegate.getReuseStrategy()); this.delegate = delegate; this.maxTokenCount = maxTokenCount; this.consumeAllTokens = consumeAllTokens; Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java (revision 1513769) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java (working copy) @@ -73,6 +73,7 @@ */ public PerFieldAnalyzerWrapper(Analyzer defaultAnalyzer, Map fieldAnalyzers) { + super(PER_FIELD_REUSE_STRATEGY); this.defaultAnalyzer = defaultAnalyzer; this.fieldAnalyzers = (fieldAnalyzers != null) ? fieldAnalyzers : Collections.emptyMap(); } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java (revision 1513769) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java (working copy) @@ -148,6 +148,7 @@ IndexReader indexReader, Collection fields, int maxDocFreq) throws IOException { + super(delegate.getReuseStrategy()); this.matchVersion = matchVersion; this.delegate = delegate; Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapper.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapper.java (revision 1513769) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapper.java (working copy) @@ -30,7 +30,7 @@ */ public final class ShingleAnalyzerWrapper extends AnalyzerWrapper { - private final Analyzer defaultAnalyzer; + private final Analyzer delegate; private final int maxShingleSize; private final int minShingleSize; private final String tokenSeparator; @@ -52,7 +52,7 @@ /** * Creates a new ShingleAnalyzerWrapper * - * @param defaultAnalyzer Analyzer whose TokenStream is to be filtered + * @param delegate Analyzer whose TokenStream is to be filtered * @param minShingleSize Min shingle (token ngram) size * @param maxShingleSize Max shingle size * @param tokenSeparator Used to separate input stream tokens in output shingles @@ -65,13 +65,14 @@ * regardless of whether any shingles are available. */ public ShingleAnalyzerWrapper( - Analyzer defaultAnalyzer, + Analyzer delegate, int minShingleSize, int maxShingleSize, String tokenSeparator, boolean outputUnigrams, boolean outputUnigramsIfNoShingles) { - this.defaultAnalyzer = defaultAnalyzer; + super(delegate.getReuseStrategy()); + this.delegate = delegate; if (maxShingleSize < 2) { throw new IllegalArgumentException("Max shingle size must be >= 2"); @@ -138,7 +139,7 @@ @Override protected Analyzer getWrappedAnalyzer(String fieldName) { - return defaultAnalyzer; + return delegate; } @Override Index: lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java (revision 1513769) +++ lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java (working copy) @@ -72,12 +72,15 @@ private final ReuseStrategy reuseStrategy; + // non final as it gets nulled if closed; pkg private for access by ReuseStrategy's final helper methods: + CloseableThreadLocal storedValue = new CloseableThreadLocal(); + /** * Create a new Analyzer, reusing the same set of components per-thread * across calls to {@link #tokenStream(String, Reader)}. */ public Analyzer() { - this(new GlobalReuseStrategy()); + this(GLOBAL_REUSE_STRATEGY); } /** @@ -133,11 +136,11 @@ */ public final TokenStream tokenStream(final String fieldName, final Reader reader) throws IOException { - TokenStreamComponents components = reuseStrategy.getReusableComponents(fieldName); + TokenStreamComponents components = reuseStrategy.getReusableComponents(this, fieldName); final Reader r = initReader(fieldName, reader); if (components == null) { components = createComponents(fieldName, r); - reuseStrategy.setReusableComponents(fieldName, components); + reuseStrategy.setReusableComponents(this, fieldName, components); } else { components.setReader(r); } @@ -167,7 +170,7 @@ * @see #tokenStream(String, Reader) */ public final TokenStream tokenStream(final String fieldName, final String text) throws IOException { - TokenStreamComponents components = reuseStrategy.getReusableComponents(fieldName); + TokenStreamComponents components = reuseStrategy.getReusableComponents(this, fieldName); @SuppressWarnings("resource") final ReusableStringReader strReader = (components == null || components.reusableStringReader == null) ? new ReusableStringReader() : components.reusableStringReader; @@ -175,7 +178,7 @@ final Reader r = initReader(fieldName, strReader); if (components == null) { components = createComponents(fieldName, r); - reuseStrategy.setReusableComponents(fieldName, components); + reuseStrategy.setReusableComponents(this, fieldName, components); } else { components.setReader(r); } @@ -229,10 +232,20 @@ return 1; } + /** + * Returns the used {@link ReuseStrategy}. + */ + public final ReuseStrategy getReuseStrategy() { + return reuseStrategy; + } + /** Frees persistent resources used by this Analyzer */ @Override public void close() { - reuseStrategy.close(); + if (storedValue != null) { + storedValue.close(); + storedValue = null; + } } /** @@ -317,43 +330,44 @@ * Strategy defining how TokenStreamComponents are reused per call to * {@link Analyzer#tokenStream(String, java.io.Reader)}. */ - public static abstract class ReuseStrategy implements Closeable { + public static abstract class ReuseStrategy { - private CloseableThreadLocal storedValue = new CloseableThreadLocal(); - /** Sole constructor. (For invocation by subclass constructors, typically implicit.) */ public ReuseStrategy() {} /** - * Gets the reusable TokenStreamComponents for the field with the given name + * Gets the reusable TokenStreamComponents for the field with the given name. * + * @param analyzer Analyzer from which to get the reused components. Use + * {@link #getStoredValue(Analyzer)} and {@link #setStoredValue(Analyzer, Object)} + * to access the data on the Analyzer. * @param fieldName Name of the field whose reusable TokenStreamComponents * are to be retrieved * @return Reusable TokenStreamComponents for the field, or {@code null} * if there was no previous components for the field */ - public abstract TokenStreamComponents getReusableComponents(String fieldName); + public abstract TokenStreamComponents getReusableComponents(Analyzer analyzer, String fieldName); /** * Stores the given TokenStreamComponents as the reusable components for the - * field with the give name + * field with the give name. * * @param fieldName Name of the field whose TokenStreamComponents are being set * @param components TokenStreamComponents which are to be reused for the field */ - public abstract void setReusableComponents(String fieldName, TokenStreamComponents components); + public abstract void setReusableComponents(Analyzer analyzer, String fieldName, TokenStreamComponents components); /** - * Returns the currently stored value + * Returns the currently stored value. * * @return Currently stored value or {@code null} if no value is stored - * @throws AlreadyClosedException if the ReuseStrategy is closed. + * @throws AlreadyClosedException if the Analyzer is closed. */ - protected final Object getStoredValue() { + protected final Object getStoredValue(Analyzer analyzer) { try { - return storedValue.get(); + return analyzer.storedValue.get(); } catch (NullPointerException npe) { - if (storedValue == null) { + if (analyzer.storedValue == null) { throw new AlreadyClosedException("this Analyzer is closed"); } else { throw npe; @@ -362,16 +376,16 @@ } /** - * Sets the stored value + * Sets the stored value. * * @param storedValue Value to store - * @throws AlreadyClosedException if the ReuseStrategy is closed. + * @throws AlreadyClosedException if the Analyzer is closed. */ - protected final void setStoredValue(Object storedValue) { + protected final void setStoredValue(Analyzer analyzer, Object storedValue) { try { - this.storedValue.set(storedValue); + analyzer.storedValue.set(storedValue); } catch (NullPointerException npe) { - if (storedValue == null) { + if (analyzer.storedValue == null) { throw new AlreadyClosedException("this Analyzer is closed"); } else { throw npe; @@ -379,61 +393,73 @@ } } - /** - * Closes the ReuseStrategy, freeing any resources - */ - @Override - public void close() { - if (storedValue != null) { - storedValue.close(); - storedValue = null; - } - } } /** + * A predefined {@link ReuseStrategy} that reuses the same components for + * every field. + */ + public static final ReuseStrategy GLOBAL_REUSE_STRATEGY = new GlobalReuseStrategy(); + + /** * Implementation of {@link ReuseStrategy} that reuses the same components for * every field. + * @deprecated This implementation class will be hidden in Lucene 5.0. + * Use {@link Analyzer#GLOBAL_REUSE_STRATEGY} instead! */ + @Deprecated public final static class GlobalReuseStrategy extends ReuseStrategy { - /** Creates a new instance, with empty per-thread values */ + /** Sole constructor. (For invocation by subclass constructors, typically implicit.) + * @deprecated Don't create instances of this class, use {@link Analyzer#GLOBAL_REUSE_STRATEGY} */ + @Deprecated public GlobalReuseStrategy() {} @Override - public TokenStreamComponents getReusableComponents(String fieldName) { - return (TokenStreamComponents) getStoredValue(); + public TokenStreamComponents getReusableComponents(Analyzer analyzer, String fieldName) { + return (TokenStreamComponents) getStoredValue(analyzer); } @Override - public void setReusableComponents(String fieldName, TokenStreamComponents components) { - setStoredValue(components); + public void setReusableComponents(Analyzer analyzer, String fieldName, TokenStreamComponents components) { + setStoredValue(analyzer, components); } } /** + * A predefined {@link ReuseStrategy} that reuses components per-field by + * maintaining a Map of TokenStreamComponent per field name. + */ + public static final ReuseStrategy PER_FIELD_REUSE_STRATEGY = new PerFieldReuseStrategy(); + + /** * Implementation of {@link ReuseStrategy} that reuses components per-field by * maintaining a Map of TokenStreamComponent per field name. + * @deprecated This implementation class will be hidden in Lucene 5.0. + * Use {@link Analyzer#PER_FIELD_REUSE_STRATEGY} instead! */ + @Deprecated public static class PerFieldReuseStrategy extends ReuseStrategy { - /** Creates a new instance, with empty per-thread-per-field values */ + /** Sole constructor. (For invocation by subclass constructors, typically implicit.) + * @deprecated Don't create instances of this class, use {@link Analyzer#PER_FIELD_REUSE_STRATEGY} */ + @Deprecated public PerFieldReuseStrategy() {} @SuppressWarnings("unchecked") @Override - public TokenStreamComponents getReusableComponents(String fieldName) { - Map componentsPerField = (Map) getStoredValue(); + public TokenStreamComponents getReusableComponents(Analyzer analyzer, String fieldName) { + Map componentsPerField = (Map) getStoredValue(analyzer); return componentsPerField != null ? componentsPerField.get(fieldName) : null; } @SuppressWarnings("unchecked") @Override - public void setReusableComponents(String fieldName, TokenStreamComponents components) { - Map componentsPerField = (Map) getStoredValue(); + public void setReusableComponents(Analyzer analyzer, String fieldName, TokenStreamComponents components) { + Map componentsPerField = (Map) getStoredValue(analyzer); if (componentsPerField == null) { componentsPerField = new HashMap(); - setStoredValue(componentsPerField); + setStoredValue(analyzer, componentsPerField); } componentsPerField.put(fieldName, components); } Index: lucene/core/src/java/org/apache/lucene/analysis/AnalyzerWrapper.java =================================================================== --- lucene/core/src/java/org/apache/lucene/analysis/AnalyzerWrapper.java (revision 1513769) +++ lucene/core/src/java/org/apache/lucene/analysis/AnalyzerWrapper.java (working copy) @@ -34,13 +34,30 @@ /** * Creates a new AnalyzerWrapper. Since the {@link Analyzer.ReuseStrategy} of - * the wrapped Analyzers are unknown, {@link Analyzer.PerFieldReuseStrategy} is assumed + * the wrapped Analyzers are unknown, {@link #PER_FIELD_REUSE_STRATEGY} is assumed. + * @deprecated Use {@link #AnalyzerWrapper(Analyzer.ReuseStrategy)} + * and specify a valid {@link Analyzer.ReuseStrategy}, probably retrieved from the + * wrapped analyzer using {@link #getReuseStrategy()}. */ + @Deprecated protected AnalyzerWrapper() { - super(new PerFieldReuseStrategy()); + this(PER_FIELD_REUSE_STRATEGY); } /** + * Creates a new AnalyzerWrapper with the given reuse strategy. + *

If you want to wrap a single delegate Analyzer you can probably + * reuse its strategy when instantiating this subclass: + * {@code super(delegate.getReuseStrategy());}. + *

If you choose different analyzers per field, use + * {@link #PER_FIELD_REUSE_STRATEGY}. + * @see #getReuseStrategy() + */ + protected AnalyzerWrapper(ReuseStrategy reuseStrategy) { + super(reuseStrategy); + } + + /** * Retrieves the wrapped Analyzer appropriate for analyzing the field with * the given name * Index: lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java =================================================================== --- lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (revision 1513769) +++ lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (working copy) @@ -135,7 +135,8 @@ // LUCENE-5153: test that wrapping an analyzer's reader is allowed final Random random = random(); - Analyzer a = new AnalyzerWrapper() { + final Analyzer delegate = new MockAnalyzer(random); + Analyzer a = new AnalyzerWrapper(delegate.getReuseStrategy()) { @Override protected Reader wrapReader(String fieldName, Reader reader) { @@ -149,7 +150,7 @@ @Override protected Analyzer getWrappedAnalyzer(String fieldName) { - return new MockAnalyzer(random); + return delegate; } }; Index: lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat3.java =================================================================== --- lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat3.java (revision 1513769) +++ lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat3.java (working copy) @@ -67,7 +67,7 @@ // creates 8 fields with different options and does "duels" of fields against each other public void test() throws Exception { Directory dir = newDirectory(); - Analyzer analyzer = new Analyzer(new Analyzer.PerFieldReuseStrategy()) { + Analyzer analyzer = new Analyzer(Analyzer.PER_FIELD_REUSE_STRATEGY) { @Override protected TokenStreamComponents createComponents(String fieldName, Reader reader) { Tokenizer tokenizer = new MockTokenizer(reader); Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (revision 1513769) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (working copy) @@ -381,7 +381,7 @@ doc.add(newTextField("field", "a field", Field.Store.YES)); w.addDocument(doc); - Analyzer analyzer = new Analyzer(new Analyzer.PerFieldReuseStrategy()) { + Analyzer analyzer = new Analyzer(Analyzer.PER_FIELD_REUSE_STRATEGY) { @Override public TokenStreamComponents createComponents(String fieldName, Reader reader) { MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); @@ -590,7 +590,7 @@ } public void testDocumentsWriterExceptions() throws IOException { - Analyzer analyzer = new Analyzer(new Analyzer.PerFieldReuseStrategy()) { + Analyzer analyzer = new Analyzer(Analyzer.PER_FIELD_REUSE_STRATEGY) { @Override public TokenStreamComponents createComponents(String fieldName, Reader reader) { MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); @@ -685,7 +685,7 @@ } public void testDocumentsWriterExceptionThreads() throws Exception { - Analyzer analyzer = new Analyzer(new Analyzer.PerFieldReuseStrategy()) { + Analyzer analyzer = new Analyzer(Analyzer.PER_FIELD_REUSE_STRATEGY) { @Override public TokenStreamComponents createComponents(String fieldName, Reader reader) { MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); Index: lucene/core/src/test/org/apache/lucene/index/TestPayloads.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestPayloads.java (revision 1513769) +++ lucene/core/src/test/org/apache/lucene/index/TestPayloads.java (working copy) @@ -363,11 +363,11 @@ Map fieldToData = new HashMap(); public PayloadAnalyzer() { - super(new PerFieldReuseStrategy()); + super(PER_FIELD_REUSE_STRATEGY); } public PayloadAnalyzer(String field, byte[] data, int offset, int length) { - super(new PerFieldReuseStrategy()); + super(PER_FIELD_REUSE_STRATEGY); setPayloadData(field, data, offset, length); } Index: lucene/core/src/test/org/apache/lucene/search/payloads/PayloadHelper.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/payloads/PayloadHelper.java (revision 1513769) +++ lucene/core/src/test/org/apache/lucene/search/payloads/PayloadHelper.java (working copy) @@ -59,7 +59,7 @@ public final class PayloadAnalyzer extends Analyzer { public PayloadAnalyzer() { - super(new PerFieldReuseStrategy()); + super(PER_FIELD_REUSE_STRATEGY); } @Override Index: lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (revision 1513769) +++ lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (working copy) @@ -64,7 +64,7 @@ private static class PayloadAnalyzer extends Analyzer { private PayloadAnalyzer() { - super(new PerFieldReuseStrategy()); + super(PER_FIELD_REUSE_STRATEGY); } @Override Index: lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java =================================================================== --- lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java (revision 1513769) +++ lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java (working copy) @@ -311,7 +311,7 @@ MockAnalyzer stdAnalyzer = new MockAnalyzer(random()); public AnalyzerReturningNull() { - super(new PerFieldReuseStrategy()); + super(PER_FIELD_REUSE_STRATEGY); } @Override Index: lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java =================================================================== --- lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java (revision 1513769) +++ lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java (working copy) @@ -347,7 +347,7 @@ MockAnalyzer stdAnalyzer = new MockAnalyzer(random()); public AnalyzerReturningNull() { - super(new PerFieldReuseStrategy()); + super(PER_FIELD_REUSE_STRATEGY); } @Override Index: lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java =================================================================== --- lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java (revision 1513769) +++ lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java (working copy) @@ -99,10 +99,10 @@ protected final static String TEXT_FIELD_NAME = "text"; private final Analyzer queryAnalyzer; - private final Analyzer indexAnalyzer; - private final Version matchVersion; + final Analyzer indexAnalyzer; + final Version matchVersion; private final File indexPath; - private final int minPrefixChars; + final int minPrefixChars; private Directory dir; /** {@link IndexSearcher} used for lookups. */ @@ -193,7 +193,7 @@ AtomicReader r = null; boolean success = false; try { - Analyzer gramAnalyzer = new AnalyzerWrapper() { + Analyzer gramAnalyzer = new AnalyzerWrapper(Analyzer.PER_FIELD_REUSE_STRATEGY) { @Override protected Analyzer getWrappedAnalyzer(String fieldName) { return indexAnalyzer; Index: lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java (revision 1513769) +++ lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java (working copy) @@ -60,7 +60,7 @@ * @param filter DFA describing how terms should be filtered (set of stopwords, etc) */ public MockAnalyzer(Random random, CharacterRunAutomaton runAutomaton, boolean lowerCase, CharacterRunAutomaton filter) { - super(new PerFieldReuseStrategy()); + super(PER_FIELD_REUSE_STRATEGY); // TODO: this should be solved in a different way; Random should not be shared (!). this.random = new Random(random.nextLong()); this.runAutomaton = runAutomaton; Index: solr/core/src/java/org/apache/solr/schema/IndexSchema.java =================================================================== --- solr/core/src/java/org/apache/solr/schema/IndexSchema.java (revision 1513769) +++ solr/core/src/java/org/apache/solr/schema/IndexSchema.java (working copy) @@ -379,6 +379,7 @@ protected final HashMap analyzers; SolrIndexAnalyzer() { + super(PER_FIELD_REUSE_STRATEGY); analyzers = analyzerCache(); } @@ -400,6 +401,8 @@ } private class SolrQueryAnalyzer extends SolrIndexAnalyzer { + SolrQueryAnalyzer() {} + @Override protected HashMap analyzerCache() { HashMap cache = new HashMap();