Index: CHANGES.txt =================================================================== --- CHANGES.txt (revision 778452) +++ CHANGES.txt (working copy) @@ -138,6 +138,11 @@ 14. LUCENE-1636: Make TokenFilter.input final so it's set only once. (Wouter Heijke, Uwe Schindler via Mike McCandless). +15. LUCENE-1658: Refactored FSDirectory into a base class (still named + FSDirectory) and SimpleFSDirectory. Added an FSDirectory.open + static method to pick a good default FSDirectory implementation + given the OS. (Michael McCandless) + Bug fixes 1. LUCENE-1415: MultiPhraseQuery has incorrect hashCode() and equals() Index: src/test/org/apache/lucene/index/TestDoc.java =================================================================== --- src/test/org/apache/lucene/index/TestDoc.java (revision 778452) +++ src/test/org/apache/lucene/index/TestDoc.java (working copy) @@ -25,7 +25,6 @@ import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; import org.apache.lucene.demo.FileDocument; import java.io.*; @@ -105,7 +104,7 @@ StringWriter sw = new StringWriter(); PrintWriter out = new PrintWriter(sw, true); - Directory directory = FSDirectory.getDirectory(indexDir); + Directory directory = FSDirectory.open(indexDir); IndexWriter writer = new IndexWriter(directory, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); SegmentInfo si1 = indexDoc(writer, "test.txt"); @@ -133,7 +132,7 @@ sw = new StringWriter(); out = new PrintWriter(sw, true); - directory = FSDirectory.getDirectory(indexDir); + directory = FSDirectory.open(indexDir); writer = new IndexWriter(directory, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); si1 = indexDoc(writer, "test.txt"); @@ -173,12 +172,10 @@ private SegmentInfo merge(SegmentInfo si1, SegmentInfo si2, String merged, boolean useCompoundFile) throws Exception { - Directory directory = FSDirectory.getDirectory(indexDir, false); - SegmentReader r1 = SegmentReader.get(si1); SegmentReader r2 = SegmentReader.get(si2); - SegmentMerger merger = new SegmentMerger(directory, merged); + SegmentMerger merger = new SegmentMerger(si1.dir, merged); merger.add(r1); merger.add(r2); @@ -188,17 +185,15 @@ if (useCompoundFile) { List filesToDelete = merger.createCompoundFile(merged + ".cfs"); for (Iterator iter = filesToDelete.iterator(); iter.hasNext();) - directory.deleteFile((String) iter.next()); + si1.dir.deleteFile((String) iter.next()); } - directory.close(); - return new SegmentInfo(merged, si1.docCount + si2.docCount, directory, useCompoundFile, true); + return new SegmentInfo(merged, si1.docCount + si2.docCount, si1.dir, useCompoundFile, true); } private void printSegment(PrintWriter out, SegmentInfo si) throws Exception { - Directory directory = FSDirectory.getDirectory(indexDir, false); SegmentReader reader = SegmentReader.get(si); for (int i = 0; i < reader.numDocs(); i++) @@ -226,6 +221,5 @@ } tis.close(); reader.close(); - directory.close(); } } Index: src/test/org/apache/lucene/index/TestIndexReader.java =================================================================== --- src/test/org/apache/lucene/index/TestIndexReader.java (revision 778452) +++ src/test/org/apache/lucene/index/TestIndexReader.java (working copy) @@ -535,7 +535,7 @@ throw new IOException("tempDir undefined, cannot run test"); File indexDir = new File(tempDir, "lucenetestnormwriter"); - Directory dir = FSDirectory.getDirectory(indexDir); + Directory dir = FSDirectory.open(indexDir); IndexWriter writer; IndexReader reader; Term searchTerm = new Term("content", "aaa"); @@ -715,7 +715,7 @@ } private Directory getDirectory() throws IOException { - return FSDirectory.getDirectory(new File(System.getProperty("tempDir"), "testIndex")); + return FSDirectory.open(new File(System.getProperty("tempDir"), "testIndex")); } public void testFilesOpenClose() throws IOException @@ -1183,7 +1183,7 @@ public void testOpenReaderAfterDelete() throws IOException { File dirFile = new File(System.getProperty("tempDir"), "deletetest"); - Directory dir = FSDirectory.getDirectory(dirFile); + Directory dir = FSDirectory.open(dirFile); try { IndexReader.open(dir); fail("expected FileNotFoundException"); @@ -1541,7 +1541,7 @@ File indexDir = new File(tempDir, "lucenetestdiralreadyclosed"); try { - FSDirectory dir = FSDirectory.getDirectory(indexDir); + FSDirectory dir = FSDirectory.open(indexDir); IndexWriter w = new IndexWriter(indexDir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED); w.setUseCompoundFile(false); Document doc = new Document(); @@ -1622,7 +1622,7 @@ // good exception public void testNoDir() throws Throwable { String tempDir = System.getProperty("java.io.tmpdir"); - Directory dir = FSDirectory.getDirectory(new File(tempDir, "doesnotexist"), null); + Directory dir = FSDirectory.open(new File(tempDir, "doesnotexist")); try { IndexReader.open(dir); fail("did not hit expected exception"); Index: src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- src/test/org/apache/lucene/index/TestIndexWriter.java (revision 778452) +++ src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -745,7 +745,7 @@ File indexDir = new File(tempDir, "lucenetestindexwriter"); try { - Directory dir = FSDirectory.getDirectory(indexDir); + Directory dir = FSDirectory.open(indexDir); // add one document & close writer IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); @@ -4048,7 +4048,7 @@ // LUCENE-1374 public void testMergeCompressedFields() throws IOException { File indexDir = new File(System.getProperty("tempDir"), "mergecompressedfields"); - Directory dir = FSDirectory.getDirectory(indexDir); + Directory dir = FSDirectory.open(indexDir); try { for(int i=0;i<5;i++) { // Must make a new writer & doc each time, w/ Index: src/test/org/apache/lucene/index/store/TestRAMDirectory.java =================================================================== --- src/test/org/apache/lucene/index/store/TestRAMDirectory.java (revision 778452) +++ src/test/org/apache/lucene/index/store/TestRAMDirectory.java (working copy) @@ -77,7 +77,7 @@ public void testRAMDirectory () throws IOException { - Directory dir = FSDirectory.getDirectory(indexDir); + Directory dir = FSDirectory.open(indexDir); MockRAMDirectory ramDir = new MockRAMDirectory(dir); // close the underlaying directory Index: src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java =================================================================== --- src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (revision 778452) +++ src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (working copy) @@ -86,7 +86,7 @@ // test with a single index: index1 File indexDir1 = new File(tempDir, "lucenetestindex1"); - Directory dir1 = FSDirectory.getDirectory(indexDir1); + Directory dir1 = FSDirectory.open(indexDir1); IndexWriter.unlock(dir1); norms = new ArrayList(); @@ -105,14 +105,14 @@ numDocNorms = 0; File indexDir2 = new File(tempDir, "lucenetestindex2"); - Directory dir2 = FSDirectory.getDirectory(indexDir2); + Directory dir2 = FSDirectory.open(indexDir2); createIndex(dir2); doTestNorms(dir2); // add index1 and index2 to a third index: index3 File indexDir3 = new File(tempDir, "lucenetestindex3"); - Directory dir3 = FSDirectory.getDirectory(indexDir3); + Directory dir3 = FSDirectory.open(indexDir3); createIndex(dir3); IndexWriter iw = new IndexWriter(dir3, anlzr, false, Index: src/test/org/apache/lucene/index/TestThreadedOptimize.java =================================================================== --- src/test/org/apache/lucene/index/TestThreadedOptimize.java (revision 778452) +++ src/test/org/apache/lucene/index/TestThreadedOptimize.java (working copy) @@ -149,7 +149,7 @@ throw new IOException("tempDir undefined, cannot run test"); String dirName = tempDir + "/luceneTestThreadedOptimize"; - directory = FSDirectory.getDirectory(dirName); + directory = FSDirectory.open(new File(dirName)); runTest(directory, false, new SerialMergeScheduler()); runTest(directory, true, new SerialMergeScheduler()); runTest(directory, false, new ConcurrentMergeScheduler()); Index: src/test/org/apache/lucene/index/TestAtomicUpdate.java =================================================================== --- src/test/org/apache/lucene/index/TestAtomicUpdate.java (revision 778452) +++ src/test/org/apache/lucene/index/TestAtomicUpdate.java (working copy) @@ -190,7 +190,7 @@ // Second in an FSDirectory: String tempDir = System.getProperty("java.io.tmpdir"); File dirPath = new File(tempDir, "lucene.test.atomic"); - directory = FSDirectory.getDirectory(dirPath); + directory = FSDirectory.open(dirPath); runTest(directory); directory.close(); _TestUtil.rmDir(dirPath); Index: src/test/org/apache/lucene/index/TestCompoundFile.java =================================================================== --- src/test/org/apache/lucene/index/TestCompoundFile.java (revision 778452) +++ src/test/org/apache/lucene/index/TestCompoundFile.java (working copy) @@ -62,7 +62,7 @@ super.setUp(); File file = new File(System.getProperty("tempDir"), "testIndex"); _TestUtil.rmDir(file); - dir = FSDirectory.getDirectory(file); + dir = FSDirectory.open(file); } Index: src/test/org/apache/lucene/index/TestFieldsReader.java =================================================================== --- src/test/org/apache/lucene/index/TestFieldsReader.java (revision 778452) +++ src/test/org/apache/lucene/index/TestFieldsReader.java (working copy) @@ -215,7 +215,7 @@ String path = tmpIODir + File.separator + "lazyDir" + userName; File file = new File(path); _TestUtil.rmDir(file); - FSDirectory tmpDir = FSDirectory.getDirectory(file); + FSDirectory tmpDir = FSDirectory.open(file); assertTrue(tmpDir != null); IndexWriter writer = new IndexWriter(tmpDir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); @@ -314,7 +314,7 @@ FSDirectory fsDir; public FaultyFSDirectory(File dir) throws IOException { - fsDir = FSDirectory.getDirectory(dir); + fsDir = FSDirectory.open(dir); lockFactory = fsDir.getLockFactory(); } public IndexInput openInput(String name) throws IOException { Index: src/test/org/apache/lucene/index/TestTransactionRollback.java =================================================================== --- src/test/org/apache/lucene/index/TestTransactionRollback.java (revision 778452) +++ src/test/org/apache/lucene/index/TestTransactionRollback.java (working copy) @@ -117,11 +117,6 @@ protected void setUp() throws Exception { dir = new MockRAMDirectory(); - // dir=FSDirectory.getDirectory("/indexes/testDeletionPolicy"); - // String[] files = dir.list(); - // for (String string : files) { - // dir.deleteFile(string); - // } //Build index, of records 1 to 100, committing after each batch of 10 IndexDeletionPolicy sdp=new KeepAllDeletionPolicy(); Index: src/test/org/apache/lucene/index/TestIndexReaderReopen.java =================================================================== --- src/test/org/apache/lucene/index/TestIndexReaderReopen.java (revision 778452) +++ src/test/org/apache/lucene/index/TestIndexReaderReopen.java (working copy) @@ -139,12 +139,12 @@ // in each iteration verify the work of previous iteration. // try this once with reopen once recreate, on both RAMDir and FSDir. public void testCommitReopenFS () throws IOException { - Directory dir = FSDirectory.getDirectory(indexDir); + Directory dir = FSDirectory.open(indexDir); doTestReopenWithCommit(dir, true); dir.close(); } public void testCommitRecreateFS () throws IOException { - Directory dir = FSDirectory.getDirectory(indexDir); + Directory dir = FSDirectory.open(indexDir); doTestReopenWithCommit(dir, false); dir.close(); } @@ -1079,7 +1079,7 @@ // LUCENE-1453 public void testFSDirectoryReopen() throws CorruptIndexException, IOException { - Directory dir1 = FSDirectory.getDirectory(indexDir); + Directory dir1 = FSDirectory.open(indexDir, null); createIndex(dir1, false); dir1.close(); @@ -1108,7 +1108,7 @@ throw new IOException("java.io.tmpdir undefined, cannot run test"); File indexDir2 = new File(tempDir, "IndexReaderReopen2"); - Directory dir1 = FSDirectory.getDirectory(indexDir2); + Directory dir1 = FSDirectory.open(indexDir2); createIndex(dir1, false); IndexReader lastReader = IndexReader.open(indexDir2); Index: src/test/org/apache/lucene/index/TestStressIndexing.java =================================================================== --- src/test/org/apache/lucene/index/TestStressIndexing.java (revision 778452) +++ src/test/org/apache/lucene/index/TestStressIndexing.java (working copy) @@ -174,7 +174,7 @@ // FSDir String tempDir = System.getProperty("java.io.tmpdir"); File dirPath = new File(tempDir, "lucene.test.stress"); - directory = FSDirectory.getDirectory(dirPath); + directory = FSDirectory.open(dirPath); runStressTest(directory, true, null); directory.close(); @@ -184,7 +184,7 @@ directory.close(); // With ConcurrentMergeScheduler, in FSDir - directory = FSDirectory.getDirectory(dirPath); + directory = FSDirectory.open(dirPath); runStressTest(directory, true, new ConcurrentMergeScheduler()); directory.close(); @@ -194,7 +194,7 @@ directory.close(); // With ConcurrentMergeScheduler and autoCommit=false, in FSDir - directory = FSDirectory.getDirectory(dirPath); + directory = FSDirectory.open(dirPath); runStressTest(directory, false, new ConcurrentMergeScheduler()); directory.close(); Index: src/test/org/apache/lucene/index/TestPayloads.java =================================================================== --- src/test/org/apache/lucene/index/TestPayloads.java (revision 778452) +++ src/test/org/apache/lucene/index/TestPayloads.java (working copy) @@ -160,7 +160,7 @@ // now use a FSDirectory and repeat same test String dirName = "test_payloads"; - dir = FSDirectory.getDirectory(dirName); + dir = FSDirectory.open(new File(dirName)); performTest(dir); rmDir(dirName); } Index: src/test/org/apache/lucene/index/TestBackwardsCompatibility.java =================================================================== --- src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (revision 778452) +++ src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (working copy) @@ -134,7 +134,7 @@ String dirName = "src/test/org/apache/lucene/index/index." + oldNames[i]; unzip(dirName, oldNames[i]); String fullPath = fullDir(oldNames[i]); - Directory dir = FSDirectory.getDirectory(fullPath); + Directory dir = FSDirectory.open(new File(fullPath)); IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED); w.optimize(); w.close(); @@ -195,7 +195,7 @@ dirName = fullDir(dirName); - Directory dir = FSDirectory.getDirectory(dirName); + Directory dir = FSDirectory.open(new File(dirName)); IndexSearcher searcher = new IndexSearcher(dir); IndexReader reader = searcher.getIndexReader(); @@ -263,7 +263,7 @@ dirName = fullDir(dirName); - Directory dir = FSDirectory.getDirectory(dirName); + Directory dir = FSDirectory.open(new File(dirName)); // open writer IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false); @@ -325,7 +325,7 @@ dirName = fullDir(dirName); - Directory dir = FSDirectory.getDirectory(dirName); + Directory dir = FSDirectory.open(new File(dirName)); // make sure searching sees right # hits IndexSearcher searcher = new IndexSearcher(dir); @@ -375,7 +375,7 @@ dirName = fullDir(dirName); - Directory dir = FSDirectory.getDirectory(dirName); + Directory dir = FSDirectory.open(new File(dirName)); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); writer.setUseCompoundFile(doCFS); writer.setMaxBufferedDocs(10); @@ -407,7 +407,7 @@ rmDir(outputDir); try { - Directory dir = FSDirectory.getDirectory(fullDir(outputDir)); + Directory dir = FSDirectory.open(new File(fullDir(outputDir))); boolean autoCommit = 0 == pass; Index: src/test/org/apache/lucene/index/TestNorms.java =================================================================== --- src/test/org/apache/lucene/index/TestNorms.java (revision 778452) +++ src/test/org/apache/lucene/index/TestNorms.java (working copy) @@ -81,7 +81,7 @@ // test with a single index: index1 File indexDir1 = new File(tempDir, "lucenetestindex1"); - Directory dir1 = FSDirectory.getDirectory(indexDir1); + Directory dir1 = FSDirectory.open(indexDir1); norms = new ArrayList(); modifiedNorms = new ArrayList(); @@ -99,14 +99,14 @@ numDocNorms = 0; File indexDir2 = new File(tempDir, "lucenetestindex2"); - Directory dir2 = FSDirectory.getDirectory(indexDir2); + Directory dir2 = FSDirectory.open(indexDir2); createIndex(dir2); doTestNorms(dir2); // add index1 and index2 to a third index: index3 File indexDir3 = new File(tempDir, "lucenetestindex3"); - Directory dir3 = FSDirectory.getDirectory(indexDir3); + Directory dir3 = FSDirectory.open(indexDir3); createIndex(dir3); IndexWriter iw = new IndexWriter(dir3,anlzr,false, IndexWriter.MaxFieldLength.LIMITED); Index: src/test/org/apache/lucene/store/TestLockFactory.java =================================================================== --- src/test/org/apache/lucene/store/TestLockFactory.java (revision 778452) +++ src/test/org/apache/lucene/store/TestLockFactory.java (working copy) @@ -339,7 +339,7 @@ } public void _testStressLocks(LockFactory lockFactory, String indexDirName) throws Exception { - FSDirectory fs1 = FSDirectory.getDirectory(indexDirName, lockFactory); + FSDirectory fs1 = FSDirectory.open(new File(indexDirName), lockFactory); // First create a 1 doc index: IndexWriter w = new IndexWriter(fs1, new WhitespaceAnalyzer(), true, @@ -389,8 +389,8 @@ public void testNativeFSLockFactoryPrefix() throws IOException { // Make sure we get identical instances: - Directory dir1 = FSDirectory.getDirectory("TestLockFactory.8", new NativeFSLockFactory("TestLockFactory.8")); - Directory dir2 = FSDirectory.getDirectory("TestLockFactory.9", new NativeFSLockFactory("TestLockFactory.9")); + Directory dir1 = FSDirectory.open(new File("TestLockFactory.8"), new NativeFSLockFactory("TestLockFactory.8")); + Directory dir2 = FSDirectory.open(new File("TestLockFactory.9"), new NativeFSLockFactory("TestLockFactory.9")); String prefix1 = dir1.getLockFactory().getLockPrefix(); String prefix2 = dir2.getLockFactory().getLockPrefix(); @@ -406,7 +406,7 @@ public void testDefaultFSLockFactoryPrefix() throws IOException { // Make sure we get null prefix: - Directory dir = FSDirectory.getDirectory("TestLockFactory.10"); + Directory dir = FSDirectory.open(new File("TestLockFactory.10")); String prefix = dir.getLockFactory().getLockPrefix(); Index: src/test/org/apache/lucene/store/TestDirectory.java =================================================================== --- src/test/org/apache/lucene/store/TestDirectory.java (revision 778452) +++ src/test/org/apache/lucene/store/TestDirectory.java (working copy) @@ -35,7 +35,7 @@ } catch (AlreadyClosedException ace) { } - dir = FSDirectory.getDirectory(System.getProperty("tempDir")); + dir = FSDirectory.open(new File(System.getProperty("tempDir"))); dir.close(); try { dir.createOutput("test"); @@ -138,7 +138,7 @@ // LUCENE-1468 public void testFSDirectoryFilter() throws IOException { - checkDirectoryFilter(FSDirectory.getDirectory("test")); + checkDirectoryFilter(FSDirectory.open(new File("test"))); } // LUCENE-1468 Index: src/test/org/apache/lucene/store/TestWindowsMMap.java =================================================================== --- src/test/org/apache/lucene/store/TestWindowsMMap.java (revision 778452) +++ src/test/org/apache/lucene/store/TestWindowsMMap.java (working copy) @@ -37,7 +37,6 @@ public void setUp() throws Exception { super.setUp(); random = newRandom(); - System.setProperty("org.apache.lucene.FSDirectory.class", "org.apache.lucene.store.MMapDirectory"); } private String randomToken() { @@ -65,7 +64,7 @@ public void testMmapIndex() throws Exception { FSDirectory storeDirectory; - storeDirectory = FSDirectory.getDirectory(storePathname); + storeDirectory = new MMapDirectory(new File(storePathname), null); // plan to add a set of useful stopwords, consider changing some of the // interior filters. Index: src/test/org/apache/lucene/store/TestBufferedIndexInput.java =================================================================== --- src/test/org/apache/lucene/store/TestBufferedIndexInput.java (revision 778452) +++ src/test/org/apache/lucene/store/TestBufferedIndexInput.java (working copy) @@ -212,7 +212,7 @@ public MockFSDirectory(File path, Random rand) throws IOException { this.rand = rand; lockFactory = new NoLockFactory(); - dir = FSDirectory.getDirectory(path); + dir = FSDirectory.open(path); } public IndexInput openInput(String name) throws IOException { Index: src/java/org/apache/lucene/store/NIOFSDirectory.java =================================================================== --- src/java/org/apache/lucene/store/NIOFSDirectory.java (revision 778452) +++ src/java/org/apache/lucene/store/NIOFSDirectory.java (working copy) @@ -23,24 +23,21 @@ import java.nio.channels.FileChannel; /** - * NIO version of FSDirectory. Uses FileChannel.read(ByteBuffer dst, long position) method - * which allows multiple threads to read from the file without synchronizing. FSDirectory - * synchronizes in the FSIndexInput.readInternal method which can cause pileups when there - * are many threads accessing the Directory concurrently. + * An {@link FSDirectory} implementation that uses + * java.nio's FileChannel's positional read, which allows + * multiple threads to read from the same file without + * synchronizing. * - * This class only uses FileChannel when reading; writing - * with an IndexOutput is inherited from FSDirectory. + *

This class only uses FileChannel when reading; writing + * is achieved with {@link SimpleFSDirectory.SimpleFSIndexOutput}. * - * Note: NIOFSDirectory is not recommended on Windows because of a bug + *

NOTE: NIOFSDirectory is not recommended on Windows because of a bug * in how FileChannel.read is implemented in Sun's JRE. * Inside of the implementation the position is apparently - * synchronized. See here for details: - - * http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6265734 - * - * @see FSDirectory + * synchronized. See here + * for details. */ - public class NIOFSDirectory extends FSDirectory { /** Create a new NIOFSDirectory for the named location. @@ -54,6 +51,7 @@ } // back compatibility so FSDirectory can instantiate via reflection + /* @deprecated */ protected NIOFSDirectory() throws IOException { } @@ -63,6 +61,11 @@ return new NIOFSIndexInput(new File(getFile(), name), bufferSize); } + public IndexOutput createOutput(String name) throws IOException { + initOutput(name); + return new SimpleFSDirectory.SimpleFSIndexOutput(new File(directory, name)); + } + private static class NIOFSIndexInput extends FSDirectory.FSIndexInput { private ByteBuffer byteBuf; // wraps the buffer for NIO Index: src/java/org/apache/lucene/store/SimpleFSDirectory.java =================================================================== --- src/java/org/apache/lucene/store/SimpleFSDirectory.java (revision 0) +++ src/java/org/apache/lucene/store/SimpleFSDirectory.java (revision 0) @@ -0,0 +1,197 @@ +package org.apache.lucene.store; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; + +/** A straighforward implementation of {@link FSDirectory} + * using java.io.RandomAccessFile. However, this class has + * poor concurrent performance (multiple threads will + * bottleneck) as it synchronizes when multiple threads + * read from the same file. It's usually better to use + * {@link NIOFSDirectory} or {@link MMapDirectory} instead. */ +public class SimpleFSDirectory extends FSDirectory { + + /** Create a new SimpleFSDirectory for the named location. + * + * @param path the path of the directory + * @param lockFactory the lock factory to use, or null for the default. + * @throws IOException + */ + public SimpleFSDirectory(File path, LockFactory lockFactory) throws IOException { + super(path, lockFactory); + } + + // Inherit javadoc + public IndexOutput createOutput(String name) throws IOException { + ensureOpen(); + createDir(); + File file = new File(directory, name); + if (file.exists() && !file.delete()) // delete existing, if any + throw new IOException("Cannot overwrite: " + file); + + return new SimpleFSIndexOutput(file); + } + + // Inherit javadoc + public IndexInput openInput(String name, int bufferSize) throws IOException { + ensureOpen(); + return new SimpleFSIndexInput(new File(directory, name), bufferSize); + } + + protected static class SimpleFSIndexInput extends BufferedIndexInput { + + protected static class Descriptor extends RandomAccessFile { + // remember if the file is open, so that we don't try to close it + // more than once + protected volatile boolean isOpen; + long position; + final long length; + + public Descriptor(File file, String mode) throws IOException { + super(file, mode); + isOpen=true; + length=length(); + } + + public void close() throws IOException { + if (isOpen) { + isOpen=false; + super.close(); + } + } + + protected void finalize() throws Throwable { + try { + close(); + } finally { + super.finalize(); + } + } + } + + protected final Descriptor file; + boolean isClone; + + public SimpleFSIndexInput(File path) throws IOException { + this(path, BufferedIndexInput.BUFFER_SIZE); + } + + public SimpleFSIndexInput(File path, int bufferSize) throws IOException { + super(bufferSize); + file = new Descriptor(path, "r"); + } + + /** IndexInput methods */ + protected void readInternal(byte[] b, int offset, int len) + throws IOException { + synchronized (file) { + long position = getFilePointer(); + if (position != file.position) { + file.seek(position); + file.position = position; + } + int total = 0; + do { + int i = file.read(b, offset+total, len-total); + if (i == -1) + throw new IOException("read past EOF"); + file.position += i; + total += i; + } while (total < len); + } + } + + public void close() throws IOException { + // only close the file if this is not a clone + if (!isClone) file.close(); + } + + protected void seekInternal(long position) { + } + + public long length() { + return file.length; + } + + public Object clone() { + FSIndexInput clone = (FSIndexInput)super.clone(); + clone.isClone = true; + return clone; + } + + /** Method used for testing. Returns true if the underlying + * file descriptor is valid. + */ + boolean isFDValid() throws IOException { + return file.getFD().valid(); + } + } + + protected static class SimpleFSIndexOutput extends BufferedIndexOutput { + RandomAccessFile file = null; + + // remember if the file is open, so that we don't try to close it + // more than once + private volatile boolean isOpen; + + public SimpleFSIndexOutput(File path) throws IOException { + file = new RandomAccessFile(path, "rw"); + isOpen = true; + } + + /** output methods: */ + public void flushBuffer(byte[] b, int offset, int size) throws IOException { + file.write(b, offset, size); + } + public void close() throws IOException { + // only close the file if it has not been closed yet + if (isOpen) { + boolean success = false; + try { + super.close(); + success = true; + } finally { + isOpen = false; + if (!success) { + try { + file.close(); + } catch (Throwable t) { + // Suppress so we don't mask original exception + } + } else + file.close(); + } + } + } + + /** Random-access methods */ + public void seek(long pos) throws IOException { + super.seek(pos); + file.seek(pos); + } + public long length() throws IOException { + return file.length(); + } + public void setLength(long length) throws IOException { + file.setLength(length); + } + } +} Index: src/java/org/apache/lucene/store/MMapDirectory.java =================================================================== --- src/java/org/apache/lucene/store/MMapDirectory.java (revision 778452) +++ src/java/org/apache/lucene/store/MMapDirectory.java (working copy) @@ -24,16 +24,21 @@ import java.nio.channels.FileChannel; import java.nio.channels.FileChannel.MapMode; -/** File-based {@link Directory} implementation that uses mmap for input. +/** File-based {@link Directory} implementation that uses + * mmap for reading, and {@link + * SimpleFSDirectory.SimpleFSIndexOutput} for writing. * - *

To use this, invoke Java with the System property - * org.apache.lucene.FSDirectory.class set to - * org.apache.lucene.store.MMapDirectory. This will cause {@link - * FSDirectory#getDirectory(File,boolean)} to return instances of this class. + *

NOTE: memory mapping uses up a portion of the + * virtual memory address space in your process equal to the + * size of the file being mapped. Before using this class, + * be sure your have plenty of virtual memory, eg by using a + * 64 bit JRE, or a 32 bit JRE with indexes that are + * guaranteed to fit within the address space. */ public class MMapDirectory extends FSDirectory { /** Create a new MMapDirectory for the named location. + * * @param path the path of the directory * @param lockFactory the lock factory to use, or null for the default. * @throws IOException @@ -42,7 +47,9 @@ super(path, lockFactory); } - // back compatibility so FSDirectory can instantiate via reflection + // back compatibility so FSDirectory can instantiate via + // reflection + /* @deprecated */ protected MMapDirectory() throws IOException { } @@ -86,6 +93,9 @@ public void close() throws IOException {} } + // Because Java's ByteBuffer uses an int to address the + // values, it's necessary to access a file > + // Integer.MAX_VALUE in size using multiple byte buffers. private static class MultiMMapIndexInput extends IndexInput { private ByteBuffer[] buffers; @@ -201,6 +211,11 @@ private final int MAX_BBUF = Integer.MAX_VALUE; public IndexInput openInput(String name) throws IOException { + return openInput(name, BufferedIndexInput.BUFFER_SIZE); + } + + public IndexInput openInput(String name, int bufferSize) throws IOException { + ensureOpen(); File f = new File(getFile(), name); RandomAccessFile raf = new RandomAccessFile(f, "r"); try { @@ -212,7 +227,8 @@ } } - public IndexInput openInput(String name, int bufferSize) throws IOException { - return openInput(name); + public IndexOutput createOutput(String name) throws IOException { + initOutput(name); + return new SimpleFSDirectory.SimpleFSIndexOutput(new File(directory, name)); } } Index: src/java/org/apache/lucene/store/FSDirectory.java =================================================================== --- src/java/org/apache/lucene/store/FSDirectory.java (revision 778452) +++ src/java/org/apache/lucene/store/FSDirectory.java (working copy) @@ -29,21 +29,60 @@ import java.util.Map; import org.apache.lucene.index.IndexFileNameFilter; +import org.apache.lucene.util.Constants; // Used only for WRITE_LOCK_NAME in deprecated create=true case: import org.apache.lucene.index.IndexWriter; /** - * Straightforward implementation of {@link Directory} as a directory of files. - * Locking implementation is by default the {@link SimpleFSLockFactory}, but - * can be changed either by passing in a {@link LockFactory} instance to - * getDirectory, or specifying the LockFactory class by setting - * org.apache.lucene.store.FSDirectoryLockFactoryClass Java system - * property, or by calling {@link #setLockFactory} after creating - * the Directory. + * + * Base class for Directory implementations that store index + * files in the file system. There are currently three core + * subclasses: * + *

+ * + * The {@link #create} method will chooses a reasonable + * default FSDirectory implementation given your platform, + * and is the simplest way to get a good FSDirectory + * implementation. + * + *

The locking implementation is by default {@link + * SimpleFSLockFactory}, but can be changed either by + * passing in a custom {@link LockFactory} instance, or + * specifying the LockFactory class by setting + * org.apache.lucene.store.FSDirectoryLockFactoryClass + * Java system property, or by calling {@link + * #setLockFactory} after creating the Directory. + * * @see Directory */ +// TODO: in 3.0 this will become an abstract base class public class FSDirectory extends Directory { /** This cache of directories ensures that there is a unique Directory @@ -53,6 +92,7 @@ * instance for a given canonical path is closed, we remove the * instance from the cache. See LUCENE-776 * for some relevant discussion. + * @deprecated Not used by any non-deprecated methods anymore */ private static final Map DIRECTORIES = new HashMap(); @@ -96,6 +136,7 @@ System.getProperty("java.io.tmpdir")); /** The default class which implements filesystem-based directories. */ + // deprecated private static Class IMPL; static { try { @@ -130,7 +171,7 @@ /** Returns the directory instance for the named location. * - * @deprecated Use {@link #FSDirectory(File, LockFactory)} + * @deprecated Use {@link SimpleFSDirectory#SimpleFSDirectory(File, LockFactory)} * * @param path the path to the directory. * @return the FSDirectory for the named file. */ @@ -141,7 +182,7 @@ /** Returns the directory instance for the named location. * - * @deprecated Use {@link #FSDirectory(File, LockFactory)} + * @deprecated Use {@link SimpleFSDirectory#SimpleFSDirectory(File, LockFactory)} * * @param path the path to the directory. * @param lockFactory instance of {@link LockFactory} providing the @@ -154,7 +195,7 @@ /** Returns the directory instance for the named location. * - * @deprecated Use {@link #FSDirectory(File, LockFactory)} + * @deprecated Use {@link SimpleFSDirectory#SimpleFSDirectory(File, LockFactory)} * * @param file the path to the directory. * @return the FSDirectory for the named file. */ @@ -165,7 +206,7 @@ /** Returns the directory instance for the named location. * - * @deprecated Use {@link #FSDirectory(File, LockFactory)} + * @deprecated Use {@link SimpleFSDirectory#SimpleFSDirectory(File, LockFactory)} * * @param file the path to the directory. * @param lockFactory instance of {@link LockFactory} providing the @@ -232,14 +273,14 @@ // This is now deprecated (creation should only be done // by IndexWriter): if (create) { - dir.create(); + dir.deleteFiles(); } return dir; } /** @deprecated */ - private void create() throws IOException { + private void deleteFiles() throws IOException { if (directory.exists()) { String[] files = directory.list(IndexFileNameFilter.getFilter()); // clear old files if (files == null) @@ -270,13 +311,21 @@ } } - private File directory = null; + final void initOutput(String name) throws IOException { + ensureOpen(); + createDir(); + File file = new File(directory, name); + if (file.exists() && !file.delete()) // delete existing, if any + throw new IOException("Cannot overwrite: " + file); + } + + protected File directory = null; private int refCount; protected FSDirectory() {}; // permit subclassing /** Create a new FSDirectory for the named location. - * + * @deprecated Use {@link SimpleFSDirectory#SimpleFSDirectory}. * @param path the path of the directory * @param lockFactory the lock factory to use, or null for the default. * @throws IOException @@ -287,6 +336,35 @@ refCount = 1; } + /** Creates an FSDirectory instance, trying to pick the + * best implementation given the current platform. + * Currently this returns {@link NIOFSDirectory} on + * non-Windows platforms, and {@link SimpleFSDirectory} + * on Windows platforms. + * + *

NOTE: this method may suddenly change which + * implementation is returned from release to release, in + * the event that higher performance defaults become + * possible; if the precise implementation is important to + * your application, please instantiate it directly, + * instead. + * + *

See above */ + public static FSDirectory open(File path) throws IOException { + return open(path, null); + } + + /** Just like {@link #create(File)}, but allows you to + * also specify a custom {@link LockFactory}. */ + public static FSDirectory open(File path, LockFactory lockFactory) throws IOException { + if (Constants.WINDOWS) { + // TODO: MMap if we're on 64 bit? + return new SimpleFSDirectory(path, lockFactory); + } else { + return new NIOFSDirectory(path, lockFactory); + } + } + private void init(File path, LockFactory lockFactory) throws IOException { // Set up lockFactory with cascaded defaults: if an instance was passed in, @@ -501,8 +579,7 @@ } } - /** Creates a new, empty file in the directory with the given name. - Returns a stream writing this file. */ + /** @deprecated In 3.0 this method will become abstract */ public IndexOutput createOutput(String name) throws IOException { ensureOpen(); createDir(); @@ -556,7 +633,7 @@ return openInput(name, BufferedIndexInput.BUFFER_SIZE); } - // Inherit javadoc + /** @deprecated In 3.0 this method will become abstract */ public IndexInput openInput(String name, int bufferSize) throws IOException { ensureOpen(); return new FSIndexInput(new File(directory, name), bufferSize); @@ -613,6 +690,7 @@ return this.getClass().getName() + "@" + directory; } + /** @deprecated Use SimpleFSDirectory.SimpleFSIndexInput instead */ protected static class FSIndexInput extends BufferedIndexInput { protected static class Descriptor extends RandomAccessFile { @@ -702,6 +780,7 @@ } } + /** @deprecated Use SimpleFSDirectory.SimpleFSIndexOutput instead */ protected static class FSIndexOutput extends BufferedIndexOutput { RandomAccessFile file = null;