Index: lucene/contrib/misc/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java =================================================================== --- lucene/contrib/misc/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java (revision 1158642) +++ lucene/contrib/misc/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java (working copy) @@ -1,120 +0,0 @@ -package org.apache.lucene.store; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.File; -import java.util.ArrayList; -import java.util.List; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LineFileDocs; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.Version; -import org.apache.lucene.util._TestUtil; - -public class TestNRTCachingDirectory extends LuceneTestCase { - - public void testNRTAndCommit() throws Exception { - Directory dir = newDirectory(); - NRTCachingDirectory cachedDir = new NRTCachingDirectory(dir, 2.0, 25.0); - IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); - RandomIndexWriter w = new RandomIndexWriter(random, cachedDir, conf); - w.w.setInfoStream(VERBOSE ? System.out : null); - final LineFileDocs docs = new LineFileDocs(random); - final int numDocs = _TestUtil.nextInt(random, 100, 400); - - if (VERBOSE) { - System.out.println("TEST: numDocs=" + numDocs); - } - - final List ids = new ArrayList(); - IndexReader r = null; - for(int docCount=0;docCountThis class is likely only useful in a near-real-time - * context, where indexing rate is lowish but reopen - * rate is highish, resulting in many tiny files being - * written. This directory keeps such segments (as well as - * the segments produced by merging them, as long as they - * are small enough), in RAM.

- * - *

This is safe to use: when your app calls {IndexWriter#commit}, - * all cached files will be flushed from the cached and sync'd.

- * - *

Here's a simple example usage: - * - * 

- *   Directory fsDir = FSDirectory.open(new File("/path/to/index"));
- *   NRTCachingDirectory cachedFSDir = new NRTCachingDirectory(fsDir, 5.0, 60.0);
- *   IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_32, analyzer);
- *   IndexWriter writer = new IndexWriter(cachedFSDir, conf);
- *
- * - *

This will cache all newly flushed segments, all merges - * whose expected segment size is <= 5 MB, unless the net - * cached bytes exceeds 60 MB at which point all writes will - * not be cached (until the net bytes falls below 60 MB).

- * - * @lucene.experimental - */ - -public class NRTCachingDirectory extends Directory { - - private final RAMDirectory cache = new RAMDirectory(); - - private final Directory delegate; - - private final long maxMergeSizeBytes; - private final long maxCachedBytes; - - private static final boolean VERBOSE = false; - - /** - * We will cache a newly created output if 1) it's a - * flush or a merge and the estimated size of the merged segment is <= - * maxMergeSizeMB, and 2) the total cached bytes is <= - * maxCachedMB */ - public NRTCachingDirectory(Directory delegate, double maxMergeSizeMB, double maxCachedMB) { - this.delegate = delegate; - maxMergeSizeBytes = (long) (maxMergeSizeMB*1024*1024); - maxCachedBytes = (long) (maxCachedMB*1024*1024); - } - - @Override - public LockFactory getLockFactory() { - return delegate.getLockFactory(); - } - - @Override - public void setLockFactory(LockFactory lf) throws IOException { - delegate.setLockFactory(lf); - } - - @Override - public String getLockID() { - return delegate.getLockID(); - } - - @Override - public Lock makeLock(String name) { - return delegate.makeLock(name); - } - - @Override - public void clearLock(String name) throws IOException { - delegate.clearLock(name); - } - - @Override - public String toString() { - return "NRTCachingDirectory(" + delegate + "; maxCacheMB=" + (maxCachedBytes/1024/1024.) + " maxMergeSizeMB=" + (maxMergeSizeBytes/1024/1024.) + ")"; - } - - @Override - public synchronized String[] listAll() throws IOException { - final Set files = new HashSet(); - for(String f : cache.listAll()) { - files.add(f); - } - for(String f : delegate.listAll()) { - // Cannot do this -- if lucene calls createOutput but - // file already exists then this falsely trips: - //assert !files.contains(f): "file \"" + f + "\" is in both dirs"; - files.add(f); - } - return files.toArray(new String[files.size()]); - } - - /** Returns how many bytes are being used by the - * RAMDirectory cache */ - public long sizeInBytes() { - return cache.sizeInBytes(); - } - - @Override - public synchronized boolean fileExists(String name) throws IOException { - return cache.fileExists(name) || delegate.fileExists(name); - } - - @Override - public synchronized long fileModified(String name) throws IOException { - if (cache.fileExists(name)) { - return cache.fileModified(name); - } else { - return delegate.fileModified(name); - } - } - - @Override - public synchronized void deleteFile(String name) throws IOException { - if (VERBOSE) { - System.out.println("nrtdir.deleteFile name=" + name); - } - if (cache.fileExists(name)) { - assert !delegate.fileExists(name); - cache.deleteFile(name); - } else { - delegate.deleteFile(name); - } - } - - @Override - public synchronized long fileLength(String name) throws IOException { - if (cache.fileExists(name)) { - return cache.fileLength(name); - } else { - return delegate.fileLength(name); - } - } - - public String[] listCachedFiles() { - return cache.listAll(); - } - - @Override - public IndexOutput createOutput(String name, IOContext context) throws IOException { - if (VERBOSE) { - System.out.println("nrtdir.createOutput name=" + name); - } - if (doCacheWrite(name, context)) { - if (VERBOSE) { - System.out.println(" to cache"); - } - return cache.createOutput(name, context); - } else { - return delegate.createOutput(name, context); - } - } - - @Override - public void sync(Collection fileNames) throws IOException { - if (VERBOSE) { - System.out.println("nrtdir.sync files=" + fileNames); - } - for(String fileName : fileNames) { - unCache(fileName); - } - delegate.sync(fileNames); - } - - @Override - public synchronized IndexInput openInput(String name, IOContext context) throws IOException { - if (VERBOSE) { - System.out.println("nrtdir.openInput name=" + name); - } - if (cache.fileExists(name)) { - if (VERBOSE) { - System.out.println(" from cache"); - } - return cache.openInput(name, context); - } else { - return delegate.openInput(name, context); - } - } - - @Override - public synchronized CompoundFileDirectory openCompoundInput(String name, IOContext context) throws IOException { - if (cache.fileExists(name)) { - return cache.openCompoundInput(name, context); - } else { - return delegate.openCompoundInput(name, context); - } - } - - @Override - public synchronized CompoundFileDirectory createCompoundOutput(String name, IOContext context) - throws IOException { - if (cache.fileExists(name)) { - throw new IOException("File " + name + "already exists"); - } else { - return delegate.createCompoundOutput(name, context); - } - } - - - /** Close this directory, which flushes any cached files - * to the delegate and then closes the delegate. */ - @Override - public void close() throws IOException { - for(String fileName : cache.listAll()) { - unCache(fileName); - } - cache.close(); - delegate.close(); - } - - /** Subclass can override this to customize logic; return - * true if this file should be written to the RAMDirectory. */ - protected boolean doCacheWrite(String name, IOContext context) { - final MergeInfo merge = context.mergeInfo; - //System.out.println(Thread.currentThread().getName() + ": CACHE check merge=" + merge + " size=" + (merge==null ? 0 : merge.estimatedMergeBytes)); - return !name.equals(IndexFileNames.SEGMENTS_GEN) && (merge == null || merge.estimatedMergeBytes <= maxMergeSizeBytes) && cache.sizeInBytes() <= maxCachedBytes; - } - - private void unCache(String fileName) throws IOException { - final IndexOutput out; - IOContext context = IOContext.DEFAULT; - synchronized(this) { - if (!delegate.fileExists(fileName)) { - assert cache.fileExists(fileName); - out = delegate.createOutput(fileName, context); - } else { - out = null; - } - } - - if (out != null) { - IndexInput in = null; - try { - in = cache.openInput(fileName, context); - in.copyBytes(out, in.length()); - } finally { - IOUtils.closeSafely(false, in, out); - } - synchronized(this) { - cache.deleteFile(fileName); - } - } - } -} Index: lucene/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 1158662) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -1450,7 +1450,7 @@ // Tests that if FSDir is opened w/ a NoLockFactory (or SingleInstanceLF), // then IndexWriter ctor succeeds. Previously (LUCENE-2386) it failed // when listAll() was called in IndexFileDeleter. - Directory dir = newFSDirectory(_TestUtil.getTempDir("emptyFSDirNoLock"), NoLockFactory.getNoLockFactory()); + Directory dir = newFSDirectory(_TestUtil.getTempDir("emptyFSDirNoLock"), NoLockFactory.getNoLockFactory(), true); new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))).close(); dir.close(); } Index: lucene/src/test/org/apache/lucene/index/TestCrash.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestCrash.java (revision 1158662) +++ lucene/src/test/org/apache/lucene/index/TestCrash.java (working copy) @@ -30,7 +30,7 @@ public class TestCrash extends LuceneTestCase { private IndexWriter initIndex(Random random, boolean initialCommit) throws IOException { - return initIndex(random, newDirectory(), initialCommit); + return initIndex(random, newDirectory(random, false), initialCommit); } private IndexWriter initIndex(Random random, MockDirectoryWrapper dir, boolean initialCommit) throws IOException { Index: lucene/src/test/org/apache/lucene/store/TestLockFactory.java =================================================================== --- lucene/src/test/org/apache/lucene/store/TestLockFactory.java (revision 1158662) +++ lucene/src/test/org/apache/lucene/store/TestLockFactory.java (working copy) @@ -145,7 +145,7 @@ } public void _testStressLocks(LockFactory lockFactory, File indexDir) throws Exception { - Directory dir = newFSDirectory(indexDir, lockFactory); + Directory dir = newFSDirectory(indexDir, lockFactory, true); // First create a 1 doc index: IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE)); @@ -235,9 +235,9 @@ File fdir1 = _TestUtil.getTempDir("TestLockFactory.8"); File fdir2 = _TestUtil.getTempDir("TestLockFactory.8.Lockdir"); - Directory dir1 = newFSDirectory(fdir1, new NativeFSLockFactory(fdir1)); + Directory dir1 = newFSDirectory(fdir1, new NativeFSLockFactory(fdir1), true); // same directory, but locks are stored somewhere else. The prefix of the lock factory should != null - Directory dir2 = newFSDirectory(fdir1, new NativeFSLockFactory(fdir2)); + Directory dir2 = newFSDirectory(fdir1, new NativeFSLockFactory(fdir2), true); String prefix1 = dir1.getLockFactory().getLockPrefix(); assertNull("Lock prefix for lockDir same as directory should be null", prefix1); Index: lucene/src/java/org/apache/lucene/store/NRTCachingDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/NRTCachingDirectory.java (revision 1158662) +++ lucene/src/java/org/apache/lucene/store/NRTCachingDirectory.java (working copy) @@ -217,25 +217,15 @@ } @Override - public synchronized CompoundFileDirectory openCompoundInput(String name, IOContext context) throws IOException { - if (cache.fileExists(name)) { - return cache.openCompoundInput(name, context); - } else { - return delegate.openCompoundInput(name, context); - } + public final CompoundFileDirectory openCompoundInput(String name, IOContext context) throws IOException { + return super.openCompoundInput(name, context); } @Override - public synchronized CompoundFileDirectory createCompoundOutput(String name, IOContext context) - throws IOException { - if (cache.fileExists(name)) { - throw new IOException("File " + name + "already exists"); - } else { - return delegate.createCompoundOutput(name, context); - } + public final CompoundFileDirectory createCompoundOutput(String name, IOContext context) throws IOException { + return super.createCompoundOutput(name, context); } - /** Close this directory, which flushes any cached files * to the delegate and then closes the delegate. */ @Override Index: lucene/src/test-framework/org/apache/lucene/store/MockCompoundFileDirectoryWrapper.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/store/MockCompoundFileDirectoryWrapper.java (revision 1158662) +++ lucene/src/test-framework/org/apache/lucene/store/MockCompoundFileDirectoryWrapper.java (working copy) @@ -122,7 +122,7 @@ @Override public String toString() { - return "MockCompoundFileDirectoryWrapper(" + super.toString() + ")"; + return "MockCompoundFileDirectoryWrapper(" + delegate.toString() + ")"; } @Override Index: lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (revision 1158662) +++ lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (working copy) @@ -66,6 +66,7 @@ import org.apache.lucene.store.MergeInfo; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.store.MockDirectoryWrapper.Throttling; +import org.apache.lucene.store.NRTCachingDirectory; import org.apache.lucene.util.FieldCacheSanityChecker.Insanity; import org.junit.*; import org.junit.rules.TestWatchman; @@ -1034,12 +1035,16 @@ * See {@link #newDirectory()} for more information. */ public static MockDirectoryWrapper newDirectory(Random r) throws IOException { - Directory impl = newDirectoryImpl(r, TEST_DIRECTORY); + return newDirectory(r, true); + } + + public static MockDirectoryWrapper newDirectory(Random r, boolean maybeWrap) throws IOException { + Directory impl = newDirectoryImpl(r, TEST_DIRECTORY, maybeWrap); MockDirectoryWrapper dir = new MockDirectoryWrapper(r, impl); stores.put(dir, Thread.currentThread().getStackTrace()); dir.setThrottling(TEST_THROTTLING); return dir; - } + } /** * Returns a new Directory instance, with contents copied from the @@ -1052,11 +1057,11 @@ /** Returns a new FSDirectory instance over the given file, which must be a folder. */ public static MockDirectoryWrapper newFSDirectory(File f) throws IOException { - return newFSDirectory(f, null); + return newFSDirectory(f, null, true); } /** Returns a new FSDirectory instance over the given file, which must be a folder. */ - public static MockDirectoryWrapper newFSDirectory(File f, LockFactory lf) throws IOException { + public static MockDirectoryWrapper newFSDirectory(File f, LockFactory lf, boolean maybeWrap) throws IOException { String fsdirClass = TEST_DIRECTORY; if (fsdirClass.equals("random")) { fsdirClass = FS_DIRECTORIES[random.nextInt(FS_DIRECTORIES.length)]; @@ -1080,7 +1085,8 @@ clazz = Class.forName(fsdirClass).asSubclass(FSDirectory.class); } - MockDirectoryWrapper dir = new MockDirectoryWrapper(random, newFSDirectoryImpl(clazz, f)); + Directory fsdir = newFSDirectoryImpl(clazz, f); + MockDirectoryWrapper dir = new MockDirectoryWrapper(random, maybeWrap ? maybeNRTWrap(random, fsdir) : fsdir); if (lf != null) { dir.setLockFactory(lf); } @@ -1098,7 +1104,11 @@ * {@link #newDirectory()} for more information. */ public static MockDirectoryWrapper newDirectory(Random r, Directory d) throws IOException { - Directory impl = newDirectoryImpl(r, TEST_DIRECTORY); + return newDirectory(r, d, true); + } + + public static MockDirectoryWrapper newDirectory(Random r, Directory d, boolean maybeWrap) throws IOException { + Directory impl = newDirectoryImpl(r, TEST_DIRECTORY, maybeWrap); for (String file : d.listAll()) { d.copy(impl, file, file, newIOContext(r)); } @@ -1249,7 +1259,15 @@ tempDirs.put(tmpFile.getAbsoluteFile(), Thread.currentThread().getStackTrace()); } - static Directory newDirectoryImpl(Random random, String clazzName) { + private static Directory maybeNRTWrap(Random random, Directory directory) { + if (true || /* nocommit */ rarely(random)) { + return new NRTCachingDirectory(directory, random.nextDouble(), random.nextDouble()); + } else { + return directory; + } + } + + static Directory newDirectoryImpl(Random random, String clazzName, boolean maybeWrap) { if (clazzName.equals("random")) clazzName = randomDirectory(random); if (clazzName.indexOf(".") == -1) // if not fully qualified, assume .store @@ -1262,11 +1280,12 @@ tmpFile.delete(); tmpFile.mkdir(); registerTempFile(tmpFile); - return newFSDirectoryImpl(clazz.asSubclass(FSDirectory.class), tmpFile); + Directory dir = newFSDirectoryImpl(clazz.asSubclass(FSDirectory.class), tmpFile); + return maybeWrap ? maybeNRTWrap(random, dir) : dir; } // try empty ctor - return clazz.newInstance(); + return maybeWrap ? maybeNRTWrap(random, clazz.newInstance()) : clazz.newInstance(); } catch (Exception e) { throw new RuntimeException(e); }