Index: src/java/org/apache/lucene/index/SegmentReader.java =================================================================== --- src/java/org/apache/lucene/index/SegmentReader.java (revision 927380) +++ src/java/org/apache/lucene/index/SegmentReader.java (working copy) @@ -58,6 +58,9 @@ private boolean normsDirty = false; private int pendingDeleteCount; + // The registered plugins + private Map plugins = new HashMap(); + private boolean rollbackHasChanges = false; private boolean rollbackDeletedDocsDirty = false; private boolean rollbackNormsDirty = false; @@ -290,6 +293,13 @@ } /** + * Get a plugin by name. + */ + public SegmentPlugin getPlugin(String name) { + return plugins.get(name); + } + + /** * Sets the initial value */ private class FieldsReaderLocal extends CloseableThreadLocal { @@ -558,6 +568,14 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ + public static SegmentReader get(boolean readOnly, SegmentInfo si, int termInfosIndexDivisor, Map plugins) throws CorruptIndexException, IOException { + return get(readOnly, si.dir, si, BufferedIndexInput.BUFFER_SIZE, true, termInfosIndexDivisor, plugins); + } + + /** + * @throws CorruptIndexException if the index is corrupt + * @throws IOException if there is a low-level IO error + */ public static SegmentReader get(boolean readOnly, Directory dir, SegmentInfo si, @@ -565,6 +583,22 @@ boolean doOpenStores, int termInfosIndexDivisor) throws CorruptIndexException, IOException { + return get(readOnly, dir, si, readBufferSize, doOpenStores, termInfosIndexDivisor, null); + } + + + /** + * @throws CorruptIndexException if the index is corrupt + * @throws IOException if there is a low-level IO error + */ + public static SegmentReader get(boolean readOnly, + Directory dir, + SegmentInfo si, + int readBufferSize, + boolean doOpenStores, + int termInfosIndexDivisor, + Map plugins) + throws CorruptIndexException, IOException { SegmentReader instance = readOnly ? new ReadOnlySegmentReader() : new SegmentReader(); instance.readOnly = readOnly; instance.si = si; @@ -579,6 +613,17 @@ } instance.loadDeletedDocs(); instance.openNorms(instance.core.cfsDir, readBufferSize); + + // Initialize plugins + if (plugins != null) { + for (Map.Entry entry : plugins.entrySet()) { + final SegmentPlugin plugin = entry.getValue().open(instance); + if (plugin != null) { + instance.plugins.put(entry.getKey(), plugin); + } + } + } + success = true; } finally { @@ -730,6 +775,14 @@ // any norms that have changed: clone.openNorms(si.getUseCompoundFile() ? core.getCFSReader() : directory(), readBufferSize); + // Reopen plugins + for (Map.Entry entry : plugins.entrySet()) { + final SegmentPlugin plugin = entry.getValue().reopen(clone); + if (plugin != null) { + clone.plugins.put(entry.getKey(), entry.getValue().reopen(clone)); + } + } + success = true; } finally { if (!success) { @@ -795,6 +848,11 @@ if (core != null) { core.decRef(); } + + // Close the plugins + for (SegmentPlugin plugin : plugins.values()) { + plugin.close(); + } } static boolean hasDeletions(SegmentInfo si) throws IOException { Index: src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java =================================================================== --- src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java (revision 927380) +++ src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java (working copy) @@ -23,13 +23,13 @@ import java.util.Map; class ReadOnlyDirectoryReader extends DirectoryReader { - ReadOnlyDirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, int termInfosIndexDivisor) throws IOException { - super(directory, sis, deletionPolicy, true, termInfosIndexDivisor); + ReadOnlyDirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, int termInfosIndexDivisor, final Map plugins) throws IOException { + super(directory, sis, deletionPolicy, true, termInfosIndexDivisor, plugins); } ReadOnlyDirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, int[] oldStarts, Map oldNormsCache, boolean doClone, - int termInfosIndexDivisor) throws IOException { - super(directory, infos, oldReaders, oldStarts, oldNormsCache, true, doClone, termInfosIndexDivisor); + int termInfosIndexDivisor, final Map plugins) throws IOException { + super(directory, infos, oldReaders, oldStarts, oldNormsCache, true, doClone, termInfosIndexDivisor, plugins); } ReadOnlyDirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor) throws IOException { Index: src/java/org/apache/lucene/index/SegmentPluginFactory.java =================================================================== --- src/java/org/apache/lucene/index/SegmentPluginFactory.java (revision 0) +++ src/java/org/apache/lucene/index/SegmentPluginFactory.java (revision 0) @@ -0,0 +1,15 @@ +package org.apache.lucene.index; + +import java.io.IOException; + +/** + * Factory for creating a plugin for SegmentReader. + */ +public abstract class SegmentPluginFactory { + + /** + * Initialize a segment plugin for reader. + */ + public abstract SegmentPlugin open(SegmentReader reader) throws IOException; + +} \ No newline at end of file Property changes on: src/java/org/apache/lucene/index/SegmentPluginFactory.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/DirectoryReader.java =================================================================== --- src/java/org/apache/lucene/index/DirectoryReader.java (revision 927380) +++ src/java/org/apache/lucene/index/DirectoryReader.java (working copy) @@ -63,25 +63,32 @@ private int maxDoc = 0; private int numDocs = -1; private boolean hasDeletions = false; + private Map plugins; // factory for initializing SegmentReaders static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, final int termInfosIndexDivisor) throws CorruptIndexException, IOException { + return open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor, null); + } + + static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, + final int termInfosIndexDivisor, final Map plugins) throws CorruptIndexException, IOException { return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) { @Override protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException { SegmentInfos infos = new SegmentInfos(); infos.read(directory, segmentFileName); if (readOnly) - return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor); + return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor, plugins); else - return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor); + return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor, plugins); } }.run(commit); } /** Construct reading the named set of readers. */ - DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws IOException { + DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, final Map plugins) throws IOException { this.directory = directory; + this.plugins = plugins; this.readOnly = readOnly; this.segmentInfos = sis; this.deletionPolicy = deletionPolicy; @@ -102,7 +109,7 @@ for (int i = sis.size()-1; i >= 0; i--) { boolean success = false; try { - readers[i] = SegmentReader.get(readOnly, sis.info(i), termInfosIndexDivisor); + readers[i] = SegmentReader.get(readOnly, sis.info(i), termInfosIndexDivisor, plugins); success = true; } finally { if (!success) { @@ -125,6 +132,7 @@ DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor) throws IOException { this.directory = writer.getDirectory(); this.readOnly = true; + this.plugins = writer.getSegmentPlugins(); this.segmentInfos = infos; segmentInfosStart = (SegmentInfos) infos.clone(); this.termInfosIndexDivisor = termInfosIndexDivisor; @@ -178,9 +186,10 @@ /** This constructor is only used for {@link #reopen()} */ DirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, int[] oldStarts, - Map oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor) throws IOException { + Map oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor, final Map plugins) throws IOException { this.directory = directory; this.readOnly = readOnly; + this.plugins = plugins; this.segmentInfos = infos; this.termInfosIndexDivisor = termInfosIndexDivisor; if (!readOnly) { @@ -226,7 +235,7 @@ assert !doClone; // this is a new reader; in case we hit an exception we can close it safely - newReader = SegmentReader.get(readOnly, infos.info(i), termInfosIndexDivisor); + newReader = SegmentReader.get(readOnly, infos.info(i), termInfosIndexDivisor, plugins); } else { newReader = newReaders[i].reopenSegment(infos.info(i), doClone, readOnly); } @@ -438,9 +447,9 @@ private synchronized DirectoryReader doReopen(SegmentInfos infos, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException { DirectoryReader reader; if (openReadOnly) { - reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache, doClone, termInfosIndexDivisor); + reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache, doClone, termInfosIndexDivisor, plugins); } else { - reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false, doClone, termInfosIndexDivisor); + reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false, doClone, termInfosIndexDivisor, plugins); } return reader; } Index: src/java/org/apache/lucene/index/SegmentPlugin.java =================================================================== --- src/java/org/apache/lucene/index/SegmentPlugin.java (revision 0) +++ src/java/org/apache/lucene/index/SegmentPlugin.java (revision 0) @@ -0,0 +1,24 @@ +package org.apache.lucene.index; + +import java.io.IOException; + +/** + * Base class for plugins for SegmentReader. + */ +public abstract class SegmentPlugin { + + + /** + * Reopen this SegmentPlugin for newReader. + */ + public abstract SegmentPlugin reopen(SegmentReader newReader) throws IOException; + + + /** + * Close this SegmentPlugin. + */ + public abstract void close() throws IOException; + + +} + Property changes on: src/java/org/apache/lucene/index/SegmentPlugin.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/index/IndexReader.java =================================================================== --- src/java/org/apache/lucene/index/IndexReader.java (revision 927380) +++ src/java/org/apache/lucene/index/IndexReader.java (working copy) @@ -263,6 +263,36 @@ } /** Expert: returns an IndexReader reading the index in + * the given Directory, with a custom {@link + * IndexDeletionPolicy}. You should pass readOnly=true, + * since it gives much better concurrent performance, + * unless you intend to do write operations (delete + * documents or change norms) with the reader. + * @param directory the index directory + * @param deletionPolicy a custom deletion policy (only used + * if you use this reader to perform deletes or to set + * norms); see {@link IndexWriter} for details. + * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader + * @param termInfosIndexDivisor Subsamples which indexed + * terms are loaded into RAM. This has the same effect as {@link + * IndexWriter#setTermIndexInterval} except that setting + * must be done at indexing time while this setting can be + * set per reader. When set to N, then one in every + * N*termIndexInterval terms in the index is loaded into + * memory. By setting this to a value > 1 you can reduce + * memory usage, at the expense of higher latency when + * loading a TermInfo. The default value is 1. Set this + * to -1 to skip loading the terms index entirely. + * @param plugins map of plugins to initialize for SegmentReaders + * @throws CorruptIndexException if the index is corrupt + * @throws IOException if there is a low-level IO error + */ + public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, final Map plugins) throws CorruptIndexException, IOException { + return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor, plugins); + } + + + /** Expert: returns an IndexReader reading the index in * the given Directory, using a specific commit and with * a custom {@link IndexDeletionPolicy}. You should pass * readOnly=true, since it gives much better concurrent @@ -312,10 +342,46 @@ return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor); } + /** Expert: returns an IndexReader reading the index in + * the given Directory, using a specific commit and with + * a custom {@link IndexDeletionPolicy}. You should pass + * readOnly=true, since it gives much better concurrent + * performance, unless you intend to do write operations + * (delete documents or change norms) with the reader. + * @param commit the specific {@link IndexCommit} to open; + * see {@link IndexReader#listCommits} to list all commits + * in a directory + * @param deletionPolicy a custom deletion policy (only used + * if you use this reader to perform deletes or to set + * norms); see {@link IndexWriter} for details. + * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader + * @param termInfosIndexDivisor Subsamples which indexed + * terms are loaded into RAM. This has the same effect as {@link + * IndexWriter#setTermIndexInterval} except that setting + * must be done at indexing time while this setting can be + * set per reader. When set to N, then one in every + * N*termIndexInterval terms in the index is loaded into + * memory. By setting this to a value > 1 you can reduce + * memory usage, at the expense of higher latency when + * loading a TermInfo. The default value is 1. Set this + * to -1 to skip loading the terms index entirely. + * @param plugins map of plugins to initialize for segments + * @throws CorruptIndexException if the index is corrupt + * @throws IOException if there is a low-level IO error + */ + public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, final Map plugins) throws CorruptIndexException, IOException { + return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor, plugins); + } + + private static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return DirectoryReader.open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor); + return DirectoryReader.open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor, null); } + private static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, int termInfosIndexDivisor, final Map plugins) throws CorruptIndexException, IOException { + return DirectoryReader.open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor, plugins); + } + /** * Refreshes an IndexReader if the index has changed since this instance * was (re)opened. Index: src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- src/java/org/apache/lucene/index/IndexWriter.java (revision 927380) +++ src/java/org/apache/lucene/index/IndexWriter.java (working copy) @@ -265,6 +265,7 @@ private DocumentsWriter docWriter; private IndexFileDeleter deleter; + private Map segmentPlugins = java.util.Collections.emptyMap(); private Set segmentsToOptimize = new HashSet(); // used by optimize to note those needing optimization @@ -606,7 +607,7 @@ // TODO: we may want to avoid doing this while // synchronized // Returns a ref, which we xfer to readerMap: - sr = SegmentReader.get(false, info.dir, info, readBufferSize, doOpenStores, termsIndexDivisor); + sr = SegmentReader.get(false, info.dir, info, readBufferSize, doOpenStores, termsIndexDivisor, segmentPlugins); readerMap.put(info, sr); } else { if (doOpenStores) { @@ -4850,6 +4851,21 @@ return mergedSegmentWarmer; } + /** + * Set the plugins to initialize for a newly opened segment. + */ + public void setSegmentPlugins(Map plugins) { + segmentPlugins = plugins; + } + + /** + * Get the plugins for newly opened segments + */ + public Map getSegmentPlugins() { + return segmentPlugins; + } + + private void handleOOM(OutOfMemoryError oom, String location) { if (infoStream != null) { message("hit OutOfMemoryError inside " + location);