Index: CHANGES.txt =================================================================== --- CHANGES.txt (revision 592487) +++ CHANGES.txt (working copy) @@ -155,9 +155,16 @@ obtained or released, throwing an exception if an illegal lock obtain occurred. (Patrick Kimber via Mike McCandless) - 6. LUCENE-1015: Added FieldCache extension (ExtendedFieldCache) to support doubles and longs. - Added support into SortField for sorting on doubles and longs as well. (Grant Ingersoll) + 6. LUCENE-1015: Added FieldCache extension (ExtendedFieldCache) to support doubles and longs. + Added support into SortField for sorting on doubles and longs as well. (Grant Ingersoll) + 7. LUCENE-1044: Added optional doSync boolean to + FSDirectory.getDirectory(...). If true (the default) then we will + always sync() a file before closing it, which improves the + likelihood that the index will remain consistent when the OS or + machine crashes, or power to the machine is cut. (Venkat Rangan + via Mike McCandless) + Optimizations 1. LUCENE-937: CachingTokenFilter now uses an iterator to access the Index: src/test/org/apache/lucene/store/TestLockFactory.java =================================================================== --- src/test/org/apache/lucene/store/TestLockFactory.java (revision 592487) +++ src/test/org/apache/lucene/store/TestLockFactory.java (working copy) @@ -320,7 +320,7 @@ } public void _testStressLocks(LockFactory lockFactory, String indexDirName) throws IOException { - FSDirectory fs1 = FSDirectory.getDirectory(indexDirName, lockFactory); + FSDirectory fs1 = FSDirectory.getDirectory(indexDirName, lockFactory, false); // First create a 1 doc index: IndexWriter w = new IndexWriter(fs1, new WhitespaceAnalyzer(), true); Index: src/test/org/apache/lucene/index/TestThreadedOptimize.java =================================================================== --- src/test/org/apache/lucene/index/TestThreadedOptimize.java (revision 592487) +++ src/test/org/apache/lucene/index/TestThreadedOptimize.java (working copy) @@ -149,7 +149,7 @@ throw new IOException("tempDir undefined, cannot run test"); String dirName = tempDir + "/luceneTestThreadedOptimize"; - directory = FSDirectory.getDirectory(dirName); + directory = FSDirectory.getDirectory(dirName, null, false); runTest(directory, false, null); runTest(directory, true, null); runTest(directory, false, new ConcurrentMergeScheduler()); Index: src/test/org/apache/lucene/index/TestAtomicUpdate.java =================================================================== --- src/test/org/apache/lucene/index/TestAtomicUpdate.java (revision 592487) +++ src/test/org/apache/lucene/index/TestAtomicUpdate.java (working copy) @@ -177,7 +177,7 @@ // Second in an FSDirectory: String tempDir = System.getProperty("java.io.tmpdir"); File dirPath = new File(tempDir, "lucene.test.atomic"); - directory = FSDirectory.getDirectory(dirPath); + directory = FSDirectory.getDirectory(dirPath, null, false); runTest(directory); directory.close(); _TestUtil.rmDir(dirPath); Index: src/test/org/apache/lucene/index/TestCompoundFile.java =================================================================== --- src/test/org/apache/lucene/index/TestCompoundFile.java (revision 592487) +++ src/test/org/apache/lucene/index/TestCompoundFile.java (working copy) @@ -62,7 +62,7 @@ super.setUp(); File file = new File(System.getProperty("tempDir"), "testIndex"); _TestUtil.rmDir(file); - dir = FSDirectory.getDirectory(file); + dir = FSDirectory.getDirectory(file, null, false); } Index: src/test/org/apache/lucene/index/TestStressIndexing.java =================================================================== --- src/test/org/apache/lucene/index/TestStressIndexing.java (revision 592487) +++ src/test/org/apache/lucene/index/TestStressIndexing.java (working copy) @@ -178,7 +178,7 @@ // FSDir String tempDir = System.getProperty("java.io.tmpdir"); File dirPath = new File(tempDir, "lucene.test.stress"); - directory = FSDirectory.getDirectory(dirPath); + directory = FSDirectory.getDirectory(dirPath, null, false); runStressTest(directory, true, null); directory.close(); @@ -188,7 +188,7 @@ directory.close(); // With ConcurrentMergeScheduler, in FSDir - directory = FSDirectory.getDirectory(dirPath); + directory = FSDirectory.getDirectory(dirPath, null, false); runStressTest(directory, true, new ConcurrentMergeScheduler()); directory.close(); @@ -198,7 +198,7 @@ directory.close(); // With ConcurrentMergeScheduler and autoCommit=false, in FSDir - directory = FSDirectory.getDirectory(dirPath); + directory = FSDirectory.getDirectory(dirPath, null, false); runStressTest(directory, false, new ConcurrentMergeScheduler()); directory.close(); Index: src/test/org/apache/lucene/index/TestIndexModifier.java =================================================================== --- src/test/org/apache/lucene/index/TestIndexModifier.java (revision 592487) +++ src/test/org/apache/lucene/index/TestIndexModifier.java (working copy) @@ -147,7 +147,7 @@ if (tempDir == null) throw new IOException("java.io.tmpdir undefined, cannot run test"); File indexDir = new File(tempDir, "lucenetestindex"); - Directory rd = FSDirectory.getDirectory(indexDir); + Directory rd = FSDirectory.getDirectory(indexDir, null, false); IndexThread.id = 0; IndexThread.idStack.clear(); IndexModifier index = new IndexModifier(rd, new StandardAnalyzer(), create); Index: src/test/org/apache/lucene/index/TestBackwardsCompatibility.java =================================================================== --- src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (revision 592487) +++ src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (working copy) @@ -152,7 +152,7 @@ dirName = fullDir(dirName); - Directory dir = FSDirectory.getDirectory(dirName); + Directory dir = FSDirectory.getDirectory(dirName, null, false); IndexSearcher searcher = new IndexSearcher(dir); Hits hits = searcher.search(new TermQuery(new Term("content", "aaa"))); @@ -172,7 +172,7 @@ dirName = fullDir(dirName); - Directory dir = FSDirectory.getDirectory(dirName); + Directory dir = FSDirectory.getDirectory(dirName, null, false); // open writer IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false); @@ -232,7 +232,7 @@ dirName = fullDir(dirName); - Directory dir = FSDirectory.getDirectory(dirName); + Directory dir = FSDirectory.getDirectory(dirName, null, false); // make sure searching sees right # hits IndexSearcher searcher = new IndexSearcher(dir); @@ -280,7 +280,7 @@ dirName = fullDir(dirName); - Directory dir = FSDirectory.getDirectory(dirName); + Directory dir = FSDirectory.getDirectory(dirName, null, false); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true); writer.setUseCompoundFile(doCFS); @@ -311,7 +311,7 @@ rmDir(outputDir); try { - Directory dir = FSDirectory.getDirectory(fullDir(outputDir)); + Directory dir = FSDirectory.getDirectory(fullDir(outputDir), null, false); boolean autoCommit = 0 == pass; Index: src/test/org/apache/lucene/index/TestNorms.java =================================================================== --- src/test/org/apache/lucene/index/TestNorms.java (revision 592487) +++ src/test/org/apache/lucene/index/TestNorms.java (working copy) @@ -81,7 +81,7 @@ // test with a single index: index1 File indexDir1 = new File(tempDir, "lucenetestindex1"); - Directory dir1 = FSDirectory.getDirectory(indexDir1); + Directory dir1 = FSDirectory.getDirectory(indexDir1, null, false); norms = new ArrayList(); modifiedNorms = new ArrayList(); @@ -99,14 +99,14 @@ numDocNorms = 0; File indexDir2 = new File(tempDir, "lucenetestindex2"); - Directory dir2 = FSDirectory.getDirectory(indexDir2); + Directory dir2 = FSDirectory.getDirectory(indexDir2, null, false); createIndex(dir2); doTestNorms(dir2); // add index1 and index2 to a third index: index3 File indexDir3 = new File(tempDir, "lucenetestindex3"); - Directory dir3 = FSDirectory.getDirectory(indexDir3); + Directory dir3 = FSDirectory.getDirectory(indexDir3, null, false); createIndex(dir3); IndexWriter iw = new IndexWriter(dir3,anlzr,false); Index: src/java/org/apache/lucene/store/FSDirectory.java =================================================================== --- src/java/org/apache/lucene/store/FSDirectory.java (revision 592487) +++ src/java/org/apache/lucene/store/FSDirectory.java (working copy) @@ -62,6 +62,14 @@ private static boolean disableLocks = false; + private static boolean DEFAULT_DO_SYNC = true; + + // True if we should call sync() before closing a file. + // This improves chances that index will still be + // consistent if the machine or OS abruptly crashes. See + // LUCENE-1044. + private boolean doSync = DEFAULT_DO_SYNC; + // TODO: should this move up to the Directory base class? Also: should we // make a per-instance (in addition to the static "default") version? @@ -136,33 +144,67 @@ * @return the FSDirectory for the named file. */ public static FSDirectory getDirectory(String path) throws IOException { - return getDirectory(new File(path), null); + return getDirectory(new File(path), null, DEFAULT_DO_SYNC); } /** Returns the directory instance for the named location. * @param path the path to the directory. * @param lockFactory instance of {@link LockFactory} providing the - * locking implementation. + * locking implementation. If null, the default + * {@link SimpleFSLockFactory} is used. * @return the FSDirectory for the named file. */ public static FSDirectory getDirectory(String path, LockFactory lockFactory) throws IOException { - return getDirectory(new File(path), lockFactory); + return getDirectory(new File(path), lockFactory, DEFAULT_DO_SYNC); } /** Returns the directory instance for the named location. + * @param path the path to the directory. + * @param lockFactory instance of {@link LockFactory} providing the + * locking implementation. If null, the default + * {@link SimpleFSLockFactory} is used. + * @param doSync if true (the default), sync() is called + * on all file descriptors before close(). This + * improves the likelihood that the index will + * remain consistent when the OS or machine crashes + * or the power cord is pulled. + * @return the FSDirectory for the named file. */ + public static FSDirectory getDirectory(String path, LockFactory lockFactory, boolean doSync) + throws IOException { + return getDirectory(new File(path), lockFactory, doSync); + } + + /** Returns the directory instance for the named location. * @param file the path to the directory. * @return the FSDirectory for the named file. */ public static FSDirectory getDirectory(File file) throws IOException { - return getDirectory(file, null); + return getDirectory(file, null, DEFAULT_DO_SYNC); } /** Returns the directory instance for the named location. * @param file the path to the directory. * @param lockFactory instance of {@link LockFactory} providing the - * locking implementation. + * locking implementation. If null, the default + * {@link SimpleFSLockFactory} is used. * @return the FSDirectory for the named file. */ public static FSDirectory getDirectory(File file, LockFactory lockFactory) + throws IOException { + return getDirectory(file, lockFactory, DEFAULT_DO_SYNC); + } + + /** Returns the directory instance for the named location. + * @param file the path to the directory. + * @param lockFactory instance of {@link LockFactory} providing the + * locking implementation. If null, the default + * {@link SimpleFSLockFactory} is used. + * @param doSync if true (the default), sync() is called + * on all file descriptors before close(). This + * improves the likelihood that the index will + * remain consistent when the OS or machine crashes + * or the power cord is pulled. + * @return the FSDirectory for the named file. */ + public static FSDirectory getDirectory(File file, LockFactory lockFactory, boolean doSync) throws IOException { file = new File(file.getCanonicalPath()); @@ -183,7 +225,7 @@ } catch (Exception e) { throw new RuntimeException("cannot load FSDirectory class: " + e.toString(), e); } - dir.init(file, lockFactory); + dir.init(file, lockFactory, doSync); DIRECTORIES.put(file, dir); } else { // Catch the case where a Directory is pulled from the cache, but has a @@ -254,7 +296,7 @@ protected FSDirectory() {}; // permit subclassing - private void init(File path, LockFactory lockFactory) throws IOException { + private void init(File path, LockFactory lockFactory, boolean doSync) throws IOException { // Set up lockFactory with cascaded defaults: if an instance was passed in, // use that; else if locks are disabled, use NoLockFactory; else if the @@ -262,6 +304,7 @@ // instantiate that; else, use SimpleFSLockFactory: directory = path; + this.doSync = doSync; boolean doClearLockID = false; @@ -432,7 +475,7 @@ if (file.exists() && !file.delete()) // delete existing, if any throw new IOException("Cannot overwrite: " + file); - return new FSIndexOutput(file); + return new FSIndexOutput(file, doSync); } // Inherit javadoc @@ -588,10 +631,13 @@ // remember if the file is open, so that we don't try to close it // more than once private boolean isOpen; - - public FSIndexOutput(File path) throws IOException { + private boolean doSync; + + public FSIndexOutput(File path, boolean doSync) throws IOException { file = new RandomAccessFile(path, "rw"); + isOpen = true; + this.doSync = doSync; } /** output methods: */ @@ -601,9 +647,14 @@ public void close() throws IOException { // only close the file if it has not been closed yet if (isOpen) { - super.close(); - file.close(); - isOpen = false; + try { + super.close(); + if (doSync) + file.getFD().sync(); + } finally { + file.close(); + isOpen = false; + } } } Index: contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java =================================================================== --- contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java (revision 592487) +++ contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java (working copy) @@ -126,7 +126,8 @@ FileUtils.fullyDelete(indexDir); } indexDir.mkdirs(); - directory = FSDirectory.getDirectory(indexDir); + final boolean doSync = config.get("fsdirectory.dosync", false); + directory = FSDirectory.getDirectory(indexDir, null, doSync); } else { directory = new RAMDirectory(); } Index: contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html =================================================================== --- contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html (revision 592487) +++ contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html (working copy) @@ -479,6 +479,12 @@
  • + fsdirectory.dosync - true/false + If "true", the FSDirectory will call sync() before close() on each + file. +
    • + +
  • Index work parameters: Multi int/boolean values would be iterated with calls to NewRound. There would be also added as columns in the reports, first string in the @@ -543,6 +549,7 @@
  • merge.factor
  • max.buffered
  • directory +
  • fsdirectory.dosync
  • ram.flush.mb
  • autocommit
    • @@ -676,4 +683,4 @@
     
    - \ No newline at end of file +