Index: lucene/contrib/misc/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java =================================================================== --- lucene/contrib/misc/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java (revision 1157834) +++ lucene/contrib/misc/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java (working copy) @@ -1,120 +0,0 @@ -package org.apache.lucene.store; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.File; -import java.util.ArrayList; -import java.util.List; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LineFileDocs; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.Version; -import org.apache.lucene.util._TestUtil; - -public class TestNRTCachingDirectory extends LuceneTestCase { - - public void testNRTAndCommit() throws Exception { - Directory dir = newDirectory(); - NRTCachingDirectory cachedDir = new NRTCachingDirectory(dir, 2.0, 25.0); - IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); - RandomIndexWriter w = new RandomIndexWriter(random, cachedDir, conf); - w.w.setInfoStream(VERBOSE ? System.out : null); - final LineFileDocs docs = new LineFileDocs(random); - final int numDocs = _TestUtil.nextInt(random, 100, 400); - - if (VERBOSE) { - System.out.println("TEST: numDocs=" + numDocs); - } - - final List ids = new ArrayList(); - IndexReader r = null; - for(int docCount=0;docCountThis class is likely only useful in a near-real-time - * context, where indexing rate is lowish but reopen - * rate is highish, resulting in many tiny files being - * written. This directory keeps such segments (as well as - * the segments produced by merging them, as long as they - * are small enough), in RAM.

- * - *

This is safe to use: when your app calls {IndexWriter#commit}, - * all cached files will be flushed from the cached and sync'd.

- * - *

Here's a simple example usage: - * - * 

- *   Directory fsDir = FSDirectory.open(new File("/path/to/index"));
- *   NRTCachingDirectory cachedFSDir = new NRTCachingDirectory(fsDir, 5.0, 60.0);
- *   IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_32, analyzer);
- *   IndexWriter writer = new IndexWriter(cachedFSDir, conf);
- *
- * - *

This will cache all newly flushed segments, all merges - * whose expected segment size is <= 5 MB, unless the net - * cached bytes exceeds 60 MB at which point all writes will - * not be cached (until the net bytes falls below 60 MB).

- * - * @lucene.experimental - */ - -public class NRTCachingDirectory extends Directory { - - private final RAMDirectory cache = new RAMDirectory(); - - private final Directory delegate; - - private final long maxMergeSizeBytes; - private final long maxCachedBytes; - - private static final boolean VERBOSE = false; - - /** - * We will cache a newly created output if 1) it's a - * flush or a merge and the estimated size of the merged segment is <= - * maxMergeSizeMB, and 2) the total cached bytes is <= - * maxCachedMB */ - public NRTCachingDirectory(Directory delegate, double maxMergeSizeMB, double maxCachedMB) { - this.delegate = delegate; - maxMergeSizeBytes = (long) (maxMergeSizeMB*1024*1024); - maxCachedBytes = (long) (maxCachedMB*1024*1024); - } - - @Override - public LockFactory getLockFactory() { - return delegate.getLockFactory(); - } - - @Override - public void setLockFactory(LockFactory lf) throws IOException { - delegate.setLockFactory(lf); - } - - @Override - public String getLockID() { - return delegate.getLockID(); - } - - @Override - public Lock makeLock(String name) { - return delegate.makeLock(name); - } - - @Override - public void clearLock(String name) throws IOException { - delegate.clearLock(name); - } - - @Override - public String toString() { - return "NRTCachingDirectory(" + delegate + "; maxCacheMB=" + (maxCachedBytes/1024/1024.) + " maxMergeSizeMB=" + (maxMergeSizeBytes/1024/1024.) + ")"; - } - - @Override - public synchronized String[] listAll() throws IOException { - final Set files = new HashSet(); - for(String f : cache.listAll()) { - files.add(f); - } - for(String f : delegate.listAll()) { - // Cannot do this -- if lucene calls createOutput but - // file already exists then this falsely trips: - //assert !files.contains(f): "file \"" + f + "\" is in both dirs"; - files.add(f); - } - return files.toArray(new String[files.size()]); - } - - /** Returns how many bytes are being used by the - * RAMDirectory cache */ - public long sizeInBytes() { - return cache.sizeInBytes(); - } - - @Override - public synchronized boolean fileExists(String name) throws IOException { - return cache.fileExists(name) || delegate.fileExists(name); - } - - @Override - public synchronized long fileModified(String name) throws IOException { - if (cache.fileExists(name)) { - return cache.fileModified(name); - } else { - return delegate.fileModified(name); - } - } - - @Override - public synchronized void deleteFile(String name) throws IOException { - if (VERBOSE) { - System.out.println("nrtdir.deleteFile name=" + name); - } - if (cache.fileExists(name)) { - assert !delegate.fileExists(name); - cache.deleteFile(name); - } else { - delegate.deleteFile(name); - } - } - - @Override - public synchronized long fileLength(String name) throws IOException { - if (cache.fileExists(name)) { - return cache.fileLength(name); - } else { - return delegate.fileLength(name); - } - } - - public String[] listCachedFiles() { - return cache.listAll(); - } - - @Override - public IndexOutput createOutput(String name, IOContext context) throws IOException { - if (VERBOSE) { - System.out.println("nrtdir.createOutput name=" + name); - } - if (doCacheWrite(name, context)) { - if (VERBOSE) { - System.out.println(" to cache"); - } - return cache.createOutput(name, context); - } else { - return delegate.createOutput(name, context); - } - } - - @Override - public void sync(Collection fileNames) throws IOException { - if (VERBOSE) { - System.out.println("nrtdir.sync files=" + fileNames); - } - for(String fileName : fileNames) { - unCache(fileName); - } - delegate.sync(fileNames); - } - - @Override - public synchronized IndexInput openInput(String name, IOContext context) throws IOException { - if (VERBOSE) { - System.out.println("nrtdir.openInput name=" + name); - } - if (cache.fileExists(name)) { - if (VERBOSE) { - System.out.println(" from cache"); - } - return cache.openInput(name, context); - } else { - return delegate.openInput(name, context); - } - } - - @Override - public synchronized CompoundFileDirectory openCompoundInput(String name, IOContext context) throws IOException { - if (cache.fileExists(name)) { - return cache.openCompoundInput(name, context); - } else { - return delegate.openCompoundInput(name, context); - } - } - - @Override - public synchronized CompoundFileDirectory createCompoundOutput(String name, IOContext context) - throws IOException { - if (cache.fileExists(name)) { - throw new IOException("File " + name + "already exists"); - } else { - return delegate.createCompoundOutput(name, context); - } - } - - - /** Close this directory, which flushes any cached files - * to the delegate and then closes the delegate. */ - @Override - public void close() throws IOException { - for(String fileName : cache.listAll()) { - unCache(fileName); - } - cache.close(); - delegate.close(); - } - - /** Subclass can override this to customize logic; return - * true if this file should be written to the RAMDirectory. */ - protected boolean doCacheWrite(String name, IOContext context) { - final MergeInfo merge = context.mergeInfo; - //System.out.println(Thread.currentThread().getName() + ": CACHE check merge=" + merge + " size=" + (merge==null ? 0 : merge.estimatedMergeBytes)); - return !name.equals(IndexFileNames.SEGMENTS_GEN) && (merge == null || merge.estimatedMergeBytes <= maxMergeSizeBytes) && cache.sizeInBytes() <= maxCachedBytes; - } - - private void unCache(String fileName) throws IOException { - final IndexOutput out; - IOContext context = IOContext.DEFAULT; - synchronized(this) { - if (!delegate.fileExists(fileName)) { - assert cache.fileExists(fileName); - out = delegate.createOutput(fileName, context); - } else { - out = null; - } - } - - if (out != null) { - IndexInput in = null; - try { - in = cache.openInput(fileName, context); - in.copyBytes(out, in.length()); - } finally { - IOUtils.closeSafely(false, in, out); - } - synchronized(this) { - cache.deleteFile(fileName); - } - } - } -} Index: lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (revision 1157834) +++ lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (working copy) @@ -66,6 +66,7 @@ import org.apache.lucene.store.MergeInfo; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.store.MockDirectoryWrapper.Throttling; +import org.apache.lucene.store.NRTCachingDirectory; import org.apache.lucene.util.FieldCacheSanityChecker.Insanity; import org.junit.*; import org.junit.rules.TestWatchman; @@ -1080,7 +1081,7 @@ clazz = Class.forName(fsdirClass).asSubclass(FSDirectory.class); } - MockDirectoryWrapper dir = new MockDirectoryWrapper(random, newFSDirectoryImpl(clazz, f)); + MockDirectoryWrapper dir = new MockDirectoryWrapper(random, maybeNRTWrap(random, newFSDirectoryImpl(clazz, f))); if (lf != null) { dir.setLockFactory(lf); } @@ -1249,6 +1250,14 @@ tempDirs.put(tmpFile.getAbsoluteFile(), Thread.currentThread().getStackTrace()); } + private static Directory maybeNRTWrap(Random random, Directory directory) { + if (rarely(random)) { + return new NRTCachingDirectory(directory, random.nextDouble(), random.nextDouble()); + } else { + return directory; + } + } + static Directory newDirectoryImpl(Random random, String clazzName) { if (clazzName.equals("random")) clazzName = randomDirectory(random); @@ -1262,11 +1271,11 @@ tmpFile.delete(); tmpFile.mkdir(); registerTempFile(tmpFile); - return newFSDirectoryImpl(clazz.asSubclass(FSDirectory.class), tmpFile); + return maybeNRTWrap(random, newFSDirectoryImpl(clazz.asSubclass(FSDirectory.class), tmpFile)); } // try empty ctor - return clazz.newInstance(); + return maybeNRTWrap(random, clazz.newInstance()); } catch (Exception e) { throw new RuntimeException(e); }