Index: CHANGES.txt =================================================================== --- CHANGES.txt (revision 428324) +++ CHANGES.txt (working copy) @@ -47,6 +47,14 @@ a RAMDirectory from a file system directory that contains files not related to Lucene. (Simon Willnauer via Daniel Naber) + + 8. LUCENE-635: Decoupling locking implementation from Directory + implementation. Added set/getLockFactory to Directory and moved + all locking code into subclasses of abstract class LockFactory. + FSDirectory and RAMDirectory still default to their prior locking + implementations, but now you can mix & match, for example using + SingleInstanceLockFactory (ie, in memory locking) locking with an + FSDirectory. (Michael McCandless) Bug fixes Index: src/test/org/apache/lucene/store/TestLockFactory.java =================================================================== --- src/test/org/apache/lucene/store/TestLockFactory.java (revision 0) +++ src/test/org/apache/lucene/store/TestLockFactory.java (revision 0) @@ -0,0 +1,443 @@ +package org.apache.lucene.store; + +/** + * Copyright 2004 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import junit.framework.TestCase; + +import java.util.Hashtable; +import java.util.Enumeration; + +import java.io.IOException; +import java.io.File; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.Hits; +import org.apache.lucene.analysis.WhitespaceAnalyzer; + +public class TestLockFactory extends TestCase { + + // Verify: we can provide our own LockFactory implementation, the right + // methods are called at the right time, locks are created, etc. + + public void testCustomLockFactory() throws IOException { + Directory dir = new RAMDirectory(); + MockLockFactory lf = new MockLockFactory(); + dir.setLockFactory(lf); + + // Lock prefix should have been set: + assertTrue("lock prefix was not set by the RAMDirectory", lf.lockPrefixSet); + + IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true); + + // add 100 documents (so that commit lock is used) + for (int i = 0; i < 100; i++) { + addDoc(writer); + } + + // Both write lock and commit lock should have been created: + assertEquals("# of unique locks created (after instantiating IndexWriter)", + 2, lf.locksCreated.size()); + assertTrue("# calls to makeLock <= 2 (after instantiating IndexWriter)", + lf.makeLockCount > 2); + + for(Enumeration e = lf.locksCreated.keys(); e.hasMoreElements();) { + String lockName = (String) e.nextElement(); + MockLockFactory.MockLock lock = (MockLockFactory.MockLock) lf.locksCreated.get(lockName); + assertTrue("# calls to Lock.obtain is 0 (after instantiating IndexWriter)", + lock.lockAttempts > 0); + } + + writer.close(); + } + + // Verify: we can use the NoLockFactory with RAMDirectory w/ no + // exceptions raised: + // Verify: NoLockFactory allows two IndexWriters + public void testRAMDirectoryNoLocking() throws IOException { + Directory dir = new RAMDirectory(); + dir.setLockFactory(NoLockFactory.getNoLockFactory()); + + assertTrue("RAMDirectory.setLockFactory did not take", + NoLockFactory.class.isInstance(dir.getLockFactory())); + + IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true); + + // Create a 2nd IndexWriter. This is normally not allowed but it should run through since we're not + // using any locks: + IndexWriter writer2 = null; + try { + writer2 = new IndexWriter(dir, new WhitespaceAnalyzer(), false); + } catch (Exception e) { + fail("Should not have hit an IOException with no locking"); + } + + writer.close(); + if (writer2 != null) { + writer2.close(); + } + } + + // Verify: SingleInstanceLockFactory is the default lock for RAMDirectory + // Verify: RAMDirectory does basic locking correctly (can't create two IndexWriters) + public void testDefaultRAMDirectory() throws IOException { + Directory dir = new RAMDirectory(); + + assertTrue("RAMDirectory did not use correct LockFactory: got " + dir.getLockFactory(), + SingleInstanceLockFactory.class.isInstance(dir.getLockFactory())); + + IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true); + + // Create a 2nd IndexWriter. This should fail: + IndexWriter writer2 = null; + try { + writer2 = new IndexWriter(dir, new WhitespaceAnalyzer(), false); + fail("Should have hit an IOException with two IndexWriters on default SingleInstanceLockFactory"); + } catch (IOException e) { + } + + writer.close(); + if (writer2 != null) { + writer2.close(); + } + } + + // Verify: SimpleFSLockFactory is the default for FSDirectory + // Verify: FSDirectory does basic locking correctly + public void testDefaultFSDirectory() throws IOException { + String indexDirName = "index.TestLockFactory1"; + + IndexWriter writer = new IndexWriter(indexDirName, new WhitespaceAnalyzer(), true); + + assertTrue("FSDirectory did not use correct LockFactory: got " + writer.getDirectory().getLockFactory(), + SimpleFSLockFactory.class.isInstance(writer.getDirectory().getLockFactory())); + + IndexWriter writer2 = null; + + // Create a 2nd IndexWriter. This should fail: + try { + writer2 = new IndexWriter(indexDirName, new WhitespaceAnalyzer(), false); + fail("Should have hit an IOException with two IndexWriters on default SimpleFSLockFactory"); + } catch (IOException e) { + } + + writer.close(); + if (writer2 != null) { + writer2.close(); + } + + // Cleanup + rmDir(indexDirName); + } + + // Verify: FSDirectory's default lockFactory clears all locks correctly + public void testFSDirectoryTwoCreates() throws IOException { + String indexDirName = "index.TestLockFactory2"; + + IndexWriter writer = new IndexWriter(indexDirName, new WhitespaceAnalyzer(), true); + + assertTrue("FSDirectory did not use correct LockFactory: got " + writer.getDirectory().getLockFactory(), + SimpleFSLockFactory.class.isInstance(writer.getDirectory().getLockFactory())); + + // Create a 2nd IndexWriter. This should not fail: + IndexWriter writer2 = null; + try { + writer2 = new IndexWriter(indexDirName, new WhitespaceAnalyzer(), true); + } catch (IOException e) { + fail("Should not have hit an IOException with two IndexWriters with create=true, on default SimpleFSLockFactory"); + } + + writer.close(); + if (writer2 != null) { + writer2.close(); + } + // Cleanup + rmDir(indexDirName); + } + + + // Verify: setting custom lock factory class (as system property) works: + // Verify: FSDirectory does basic locking correctly + public void testLockClassProperty() throws IOException { + String indexDirName = "index.TestLockFactory3"; + + System.setProperty("org.apache.lucene.store.FSDirectoryLockFactoryClass", + "org.apache.lucene.store.NoLockFactory"); + + IndexWriter writer = new IndexWriter(indexDirName, new WhitespaceAnalyzer(), true); + + assertTrue("FSDirectory did not use correct LockFactory: got " + writer.getDirectory().getLockFactory(), + NoLockFactory.class.isInstance(writer.getDirectory().getLockFactory())); + + // Put back to the correct default for subsequent tests: + System.setProperty("org.apache.lucene.store.FSDirectoryLockFactoryClass", + "org.apache.lucene.store.SimpleFSLockFactory"); + + writer.close(); + // Cleanup + rmDir(indexDirName); + } + + // Verify: setDisableLocks works + public void testDisableLocks() throws IOException { + String indexDirName = "index.TestLockFactory4"; + + assertTrue("Locks are already disabled", !FSDirectory.getDisableLocks()); + FSDirectory.setDisableLocks(true); + + IndexWriter writer = new IndexWriter(indexDirName, new WhitespaceAnalyzer(), true); + + assertTrue("FSDirectory did not use correct default LockFactory: got " + writer.getDirectory().getLockFactory(), + NoLockFactory.class.isInstance(writer.getDirectory().getLockFactory())); + + // Should be no error since locking is disabled: + IndexWriter writer2 = null; + try { + writer2 = new IndexWriter(indexDirName, new WhitespaceAnalyzer(), false); + } catch (IOException e) { + fail("Should not have hit an IOException with locking disabled"); + } + + // Put back to the correct default for subsequent tests: + System.setProperty("org.apache.lucene.store.FSDirectoryLockFactoryClass", + "org.apache.lucene.store.SimpleFSLockFactory"); + + FSDirectory.setDisableLocks(false); + writer.close(); + if (writer2 != null) { + writer2.close(); + } + // Cleanup + rmDir(indexDirName); + } + + // Verify: if I try to getDirectory() with two different locking implementations, I get an IOException + public void testFSDirectoryDifferentLockFactory() throws IOException { + String indexDirName = "index.TestLockFactory5"; + + LockFactory lf = new SingleInstanceLockFactory(); + FSDirectory fs1 = FSDirectory.getDirectory(indexDirName, true, lf); + + // Different lock factory instance should hit IOException: + try { + FSDirectory fs2 = FSDirectory.getDirectory(indexDirName, true, new SingleInstanceLockFactory()); + fail("Should have hit an IOException because LockFactory instances differ"); + } catch (IOException e) { + } + + FSDirectory fs2 = null; + + // Same lock factory instance should not: + try { + fs2 = FSDirectory.getDirectory(indexDirName, true, lf); + } catch (IOException e) { + fail("Should not have hit an IOException because LockFactory instances are the same"); + } + + fs1.close(); + if (fs2 != null) { + fs2.close(); + } + // Cleanup + rmDir(indexDirName); + } + + // Verify: do stress test, by opening IndexReaders and + // IndexWriters over & over in 2 threads and making sure + // no unexpected exceptions are raised: + public void testStressLocks() throws IOException { + + String indexDirName = "index.TestLockFactory6"; + FSDirectory fs1 = FSDirectory.getDirectory(indexDirName, true); + // fs1.setLockFactory(NoLockFactory.getNoLockFactory()); + + // First create a 1 doc index: + IndexWriter w = new IndexWriter(fs1, new WhitespaceAnalyzer(), true); + addDoc(w); + w.close(); + + WriterThread writer = new WriterThread(100, fs1); + SearcherThread searcher = new SearcherThread(100, fs1); + writer.start(); + searcher.start(); + + while(writer.isAlive() || searcher.isAlive()) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + } + } + + assertTrue("IndexWriter hit unexpected exceptions", !writer.hitException); + assertTrue("IndexSearcher hit unexpected exceptions", !searcher.hitException); + + // Cleanup + rmDir(indexDirName); + } + + private class WriterThread extends Thread { + private Directory dir; + private int numIteration; + public boolean hitException = false; + public WriterThread(int numIteration, Directory dir) { + this.numIteration = numIteration; + this.dir = dir; + } + public void run() { + WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer(); + IndexWriter writer = null; + for(int i=0;i implementation of an index as a single file; * * + * Directory locking is implemented by an instance of {@link + * LockFactory}, and can be changed for each Directory + * instance using {@link #setLockFactory}. + * * @author Doug Cutting */ public abstract class Directory { + + /** Holds the LockFactory instance (implements locking for + * this Directory instance). */ + protected LockFactory lockFactory; + /** Returns an array of strings, one for each file in the directory. */ public abstract String[] list() throws IOException; @@ -75,9 +84,43 @@ /** Construct a {@link Lock}. * @param name the name of the lock file */ - public abstract Lock makeLock(String name); + public Lock makeLock(String name) { + return lockFactory.makeLock(name); + } /** Closes the store. */ public abstract void close() throws IOException; + + /** + * Set the LockFactory that this Directory instance should + * use for its locking implementation. Each * instance of + * LockFactory should only be used for one directory (ie, + * do not share a single instance across multiple + * Directories). + * + * @param lockFactory instance of {@link LockFactory}. + */ + public void setLockFactory(LockFactory lockFactory) { + this.lockFactory = lockFactory; + lockFactory.setLockPrefix(this.getLockID()); + } + /** + * Get the LockFactory that this Directory instance is using for its locking implementation. + */ + public LockFactory getLockFactory() { + return this.lockFactory; + } + + /** + * Return a string identifier that uniquely differentiates + * this Directory instance from other Directory instances. + * This ID should be the same if two Directory instances + * (even in different JVMs and/or on different machines) + * are considered "the same index". This is how locking + * "scopes" to the right index. + */ + public String getLockID() { + return this.toString(); + } } Index: src/java/org/apache/lucene/store/RAMDirectory.java =================================================================== --- src/java/org/apache/lucene/store/RAMDirectory.java (revision 428324) +++ src/java/org/apache/lucene/store/RAMDirectory.java (working copy) @@ -27,7 +27,9 @@ import org.apache.lucene.store.IndexOutput; /** - * A memory-resident {@link Directory} implementation. + * A memory-resident {@link Directory} implementation. Locking + * implementation is by default the {@link SingleInstanceLockFactory} + * but can be changed with {@link #setLockFactory}. * * @version $Id$ */ @@ -39,6 +41,7 @@ /** Constructs an empty {@link Directory}. */ public RAMDirectory() { + setLockFactory(new SingleInstanceLockFactory()); } /** @@ -56,6 +59,7 @@ } private RAMDirectory(Directory dir, boolean closeDir) throws IOException { + this(); final String[] files = dir.list(); byte[] buf = new byte[BufferedIndexOutput.BUFFER_SIZE]; for (int i = 0; i < files.length; i++) { @@ -175,29 +179,6 @@ return new RAMInputStream(file); } - /** Construct a {@link Lock}. - * @param name the name of the lock file - */ - public final Lock makeLock(final String name) { - return new Lock() { - public boolean obtain() throws IOException { - synchronized (files) { - if (!fileExists(name)) { - createOutput(name).close(); - return true; - } - return false; - } - } - public void release() { - deleteFile(name); - } - public boolean isLocked() { - return fileExists(name); - } - }; - } - /** Closes the store to future operations, releasing associated memory. */ public final void close() { files = null; Index: src/java/org/apache/lucene/store/SingleInstanceLockFactory.java =================================================================== --- src/java/org/apache/lucene/store/SingleInstanceLockFactory.java (revision 0) +++ src/java/org/apache/lucene/store/SingleInstanceLockFactory.java (revision 0) @@ -0,0 +1,86 @@ +package org.apache.lucene.store; + +/** + * Copyright 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.HashSet; +import java.util.Enumeration; + +/** + * Implements {@link LockFactory} for a single in-process instance, + * meaning all locking will take place through this one instance. + * Only use this {@link LockFactory} when you are certain all + * IndexReaders and IndexWriters for a given index are running + * against a single shared in-process Directory instance. This is + * currently the default locking for RAMDirectory. + * + * @see LockFactory + */ + +public class SingleInstanceLockFactory extends LockFactory { + + private HashSet locks = new HashSet(); + + public Lock makeLock(String lockName) { + // We do not use the LockPrefix at all, because the private + // HashSet instance effectively scopes the locking to this + // single Directory instance. + return new SingleInstanceLock(locks, lockName); + } + + public void clearAllLocks() throws IOException { + locks = new HashSet(); + } +}; + +class SingleInstanceLock extends Lock { + + String lockName; + private HashSet locks; + + public SingleInstanceLock(HashSet locks, String lockName) { + this.locks = locks; + this.lockName = lockName; + } + + public boolean obtain() throws IOException { + synchronized(locks) { + if (!locks.contains(lockName)) { + locks.add(lockName); + return true; + } else { + return false; + } + } + } + + public void release() { + synchronized(locks) { + locks.remove(lockName); + } + } + + public boolean isLocked() { + synchronized(locks) { + return locks.contains(lockName); + } + } + + public String toString() { + return "SingleInstanceLock: " + lockName; + } +} Index: src/java/org/apache/lucene/store/NoLockFactory.java =================================================================== --- src/java/org/apache/lucene/store/NoLockFactory.java (revision 0) +++ src/java/org/apache/lucene/store/NoLockFactory.java (revision 0) @@ -0,0 +1,62 @@ +package org.apache.lucene.store; + +/** + * Copyright 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +/** + * Use this {@link LockFactory} to disable locking entirely. + * This LockFactory is used when you call {@link FSDirectory#setDisableLocks}. + * Only one instance of this lock is created. You should call {@link + * #getNoLockFactory()} to get the instance. + * + * @see LockFactory + */ + +public class NoLockFactory extends LockFactory { + + // Single instance returned whenever makeLock is called. + private static NoLock singletonLock = new NoLock(); + private static NoLockFactory singleton = new NoLockFactory(); + + public static NoLockFactory getNoLockFactory() { + return singleton; + } + + public Lock makeLock(String lockName) { + return singletonLock; + } + + public void clearAllLocks() {}; +}; + +class NoLock extends Lock { + public boolean obtain() throws IOException { + return true; + } + + public void release() { + } + + public boolean isLocked() { + return false; + } + + public String toString() { + return "NoLock"; + } +} Index: src/java/org/apache/lucene/store/LockFactory.java =================================================================== --- src/java/org/apache/lucene/store/LockFactory.java (revision 0) +++ src/java/org/apache/lucene/store/LockFactory.java (revision 0) @@ -0,0 +1,56 @@ +package org.apache.lucene.store; + +/** + * Copyright 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +/** + * Base class for Locking implementation. {@link Directory} uses + * instances of this class to implement locking. + */ + +public abstract class LockFactory { + + protected String lockPrefix = ""; + + /** + * Set the prefix in use for all locks created in this LockFactory. + */ + public void setLockPrefix(String lockPrefix) { + this.lockPrefix = lockPrefix; + } + + /** + * Get the prefix in use for all locks created in this LockFactory. + */ + public String getLockPrefix() { + return this.lockPrefix; + } + + /** + * Return a new Lock instance identified by lockName. + * @param lockName name of the lock to be created. + */ + public abstract Lock makeLock(String lockName); + + /** + * Clear any existing locks. Only call this at a time when you + * are certain the lock files are not in use. {@link FSDirectory} + * calls this when creating a new index. + */ + public abstract void clearAllLocks() throws IOException; +} Index: src/java/org/apache/lucene/store/SimpleFSLockFactory.java =================================================================== --- src/java/org/apache/lucene/store/SimpleFSLockFactory.java (revision 0) +++ src/java/org/apache/lucene/store/SimpleFSLockFactory.java (revision 0) @@ -0,0 +1,130 @@ +package org.apache.lucene.store; + +/** + * Copyright 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.File; +import java.io.IOException; + +/** + * Implements {@link LockFactory} using {@link File#createNewFile()}. This is + * currently the default LockFactory used for {@link FSDirectory} if no + * LockFactory instance is otherwise provided. + * + * Note that there are known problems with this locking implementation on NFS. + * + * @see LockFactory + */ + +public class SimpleFSLockFactory extends LockFactory { + + /** + * Directory specified by org.apache.lucene.lockDir + * system property. If that is not set, then java.io.tmpdir + * system property is used. + */ + + public static final String LOCK_DIR = + System.getProperty("org.apache.lucene.lockDir", + System.getProperty("java.io.tmpdir")); + + private File lockDir; + + /** + * Instantiate using default LOCK_DIR: org.apache.lucene.lockDir + * system property, or (if that is null) then java.io.tmpdir. + */ + public SimpleFSLockFactory() throws IOException { + lockDir = new File(LOCK_DIR); + init(lockDir); + } + + /** + * Instantiate using the provided directory (as a File instance). + * @param lockDir where lock files should be created. + */ + public SimpleFSLockFactory(File lockDir) throws IOException { + init(lockDir); + } + + /** + * Instantiate using the provided directory name (String). + * @param lockDirName where lock files should be created. + */ + public SimpleFSLockFactory(String lockDirName) throws IOException { + lockDir = new File(lockDirName); + init(lockDir); + } + + protected void init(File lockDir) throws IOException { + + this.lockDir = lockDir; + + // Ensure that lockDir exists and is a directory. + if (!lockDir.exists()) { + if (!lockDir.mkdirs()) + throw new IOException("Cannot create directory: " + + lockDir.getAbsolutePath()); + } else if (!lockDir.isDirectory()) { + throw new IOException("Found regular file where directory expected: " + + lockDir.getAbsolutePath()); + } + } + + public Lock makeLock(String lockName) { + return new SimpleFSLock(lockDir, lockPrefix + "-" + lockName); + } + + public void clearAllLocks() throws IOException { + String[] files = lockDir.list(); + if (files == null) + throw new IOException("Cannot read lock directory " + + lockDir.getAbsolutePath()); + String prefix = lockPrefix + "-"; + for (int i = 0; i < files.length; i++) { + if (!files[i].startsWith(prefix)) + continue; + File lockFile = new File(lockDir, files[i]); + if (!lockFile.delete()) + throw new IOException("Cannot delete " + lockFile); + } + } +}; + +class SimpleFSLock extends Lock { + + File lockFile; + + public SimpleFSLock(File lockDir, String lockFileName) { + lockFile = new File(lockDir, lockFileName); + } + + public boolean obtain() throws IOException { + return lockFile.createNewFile(); + } + + public void release() { + lockFile.delete(); + } + + public boolean isLocked() { + return lockFile.exists(); + } + + public String toString() { + return "SimpleFSLock@" + lockFile; + } +} Index: src/java/org/apache/lucene/store/FSDirectory.java =================================================================== --- src/java/org/apache/lucene/store/FSDirectory.java (revision 428324) +++ src/java/org/apache/lucene/store/FSDirectory.java (working copy) @@ -29,6 +29,12 @@ /** * Straightforward implementation of {@link Directory} as a directory of files. + * Locking implementation is by default the {@link SimpleFSLockFactory}, but + * can be changed either by passing in a {@link LockFactory} instance to + * getDirectory, or specifying the LockFactory class by setting + * org.apache.lucene.store.FSDirectoryLockFactoryClass Java system + * property, or by calling {@link #setLockFactory} after creating + * the Directory. * * @see Directory * @author Doug Cutting @@ -46,6 +52,9 @@ private static boolean disableLocks = false; + // TODO: should this move up to the Directory base class? Also: should we + // make a per-instance (in addition to the static "default") version? + /** * Set whether Lucene's use of lock files is disabled. By default, * lock files are enabled. They should only be disabled if the index @@ -63,13 +72,17 @@ return FSDirectory.disableLocks; } + // TODO: LOCK_DIR really should only appear in the SimpleFSLockFactory + // (and any other file-system based locking implementations). When we + // can next break backwards compatibility we should deprecate it and then + // move it. + /** * Directory specified by org.apache.lucene.lockDir - * or java.io.tmpdir system property + * or java.io.tmpdir system property. This may be deprecated in the future. Please use + * {@link SimpleFSLockFactory#LOCK_DIR} instead. */ - public static final String LOCK_DIR = - System.getProperty("org.apache.lucene.lockDir", - System.getProperty("java.io.tmpdir")); + public static final String LOCK_DIR = SimpleFSLockFactory.LOCK_DIR; /** The default class which implements filesystem-based directories. */ private static Class IMPL; @@ -114,9 +127,27 @@ * @return the FSDirectory for the named file. */ public static FSDirectory getDirectory(String path, boolean create) throws IOException { - return getDirectory(new File(path), create); + return getDirectory(path, create, null); } + /** Returns the directory instance for the named location, using the + * provided LockFactory implementation. + * + *

Directories are cached, so that, for a given canonical path, the same + * FSDirectory instance will always be returned. This permits + * synchronization on directories. + * + * @param path the path to the directory. + * @param create if true, create, or erase any existing contents. + * @param lockFactory instance of {@link LockFactory} providing the + * locking implementation. + * @return the FSDirectory for the named file. */ + public static FSDirectory getDirectory(String path, boolean create, + LockFactory lockFactory) + throws IOException { + return getDirectory(new File(path), create, lockFactory); + } + /** Returns the directory instance for the named location. * *

Directories are cached, so that, for a given canonical path, the same @@ -128,6 +159,24 @@ * @return the FSDirectory for the named file. */ public static FSDirectory getDirectory(File file, boolean create) throws IOException { + return getDirectory(file, create, null); + } + + /** Returns the directory instance for the named location, using the + * provided LockFactory implementation. + * + *

Directories are cached, so that, for a given canonical path, the same + * FSDirectory instance will always be returned. This permits + * synchronization on directories. + * + * @param file the path to the directory. + * @param create if true, create, or erase any existing contents. + * @param lockFactory instance of {@link LockFactory} providing the + * locking implementation. + * @return the FSDirectory for the named file. */ + public static FSDirectory getDirectory(File file, boolean create, + LockFactory lockFactory) + throws IOException { file = new File(file.getCanonicalPath()); FSDirectory dir; synchronized (DIRECTORIES) { @@ -138,10 +187,19 @@ } catch (Exception e) { throw new RuntimeException("cannot load FSDirectory class: " + e.toString(), e); } - dir.init(file, create); + dir.init(file, create, lockFactory); DIRECTORIES.put(file, dir); - } else if (create) { - dir.create(); + } else { + + // Catch the case where a Directory is pulled from the cache, but has a + // different LockFactory instance. + if (lockFactory != null && lockFactory != dir.getLockFactory()) { + throw new IOException("Directory was previously created with a different LockFactory instance; please pass null as the lockFactory instance and use setLockFactory to change it"); + } + + if (create) { + dir.create(); + } } } synchronized (dir) { @@ -152,27 +210,12 @@ private File directory = null; private int refCount; - private File lockDir; protected FSDirectory() {}; // permit subclassing private void init(File path, boolean create) throws IOException { directory = path; - if (LOCK_DIR == null) { - lockDir = directory; - } - else { - lockDir = new File(LOCK_DIR); - } - // Ensure that lockDir exists and is a directory. - if (!lockDir.exists()) { - if (!lockDir.mkdirs()) - throw new IOException("Cannot create directory: " + lockDir.getAbsolutePath()); - } else if (!lockDir.isDirectory()) { - throw new IOException("Found regular file where directory expected: " + - lockDir.getAbsolutePath()); - } if (create) { create(); } @@ -181,6 +224,64 @@ throw new IOException(path + " not a directory"); } + private void init(File path, boolean create, LockFactory lockFactory) throws IOException { + + // Set up lockFactory with cascaded defaults: if an instance was passed in, + // use that; else if locks are disabled, use NoLockFactory; else if the + // system property org.apache.lucene.lockClass is set, instantiate that; + // else, use SimpleFSLockFactory: + + if (lockFactory == null) { + + if (disableLocks) { + // Locks are disabled: + lockFactory = NoLockFactory.getNoLockFactory(); + } else { + String lockClassName = System.getProperty("org.apache.lucene.store.FSDirectoryLockFactoryClass"); + + if (lockClassName != null) { + Class c; + + try { + c = Class.forName(lockClassName); + } catch (ClassNotFoundException e) { + throw new IOException("unable to find LockClass " + lockClassName); + } + + try { + lockFactory = (LockFactory) c.newInstance(); + } catch (IllegalAccessException e) { + throw new IOException("IllegalAccessException when instantiating LockClass " + lockClassName); + } catch (InstantiationException e) { + throw new IOException("InstantiationException when instantiating LockClass " + lockClassName); + } catch (ClassCastException e) { + throw new IOException("unable to cast LockClass " + lockClassName + " instance to a LockFactory"); + } + } else { + // Our default lock is SimpleFSLockFactory: + File lockDir; + if (LOCK_DIR == null) { + lockDir = directory; + } else { + lockDir = new File(LOCK_DIR); + } + lockFactory = new SimpleFSLockFactory(lockDir); + } + } + } + + // Must initialize directory here because setLockFactory uses it + // (when the LockFactory calls getLockID). But we want to create + // the lockFactory before calling init() because init() needs to + // use the lockFactory to clear old locks. So this breaks + // chicken/egg: + directory = path; + + setLockFactory(lockFactory); + + init(path, create); + } + private synchronized void create() throws IOException { if (!directory.exists()) if (!directory.mkdirs()) @@ -198,17 +299,7 @@ throw new IOException("Cannot delete " + file); } - String lockPrefix = getLockPrefix().toString(); // clear old locks - files = lockDir.list(); - if (files == null) - throw new IOException("Cannot read lock directory " + lockDir.getAbsolutePath()); - for (int i = 0; i < files.length; i++) { - if (!files[i].startsWith(lockPrefix)) - continue; - File lockFile = new File(lockDir, files[i]); - if (!lockFile.delete()) - throw new IOException("Cannot delete " + lockFile); - } + lockFactory.clearAllLocks(); } /** Returns an array of strings, one for each Lucene index file in the directory. */ @@ -338,51 +429,8 @@ private static final char[] HEX_DIGITS = {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'}; - /** Constructs a {@link Lock} with the specified name. Locks are implemented - * with {@link File#createNewFile()}. - * - * @param name the name of the lock file - * @return an instance of Lock holding the lock - */ - public Lock makeLock(String name) { - StringBuffer buf = getLockPrefix(); - buf.append("-"); - buf.append(name); - - // create a lock file - final File lockFile = new File(lockDir, buf.toString()); - - return new Lock() { - public boolean obtain() throws IOException { - if (disableLocks) - return true; - - if (!lockDir.exists()) { - if (!lockDir.mkdirs()) { - throw new IOException("Cannot create lock directory: " + lockDir); - } - } - - return lockFile.createNewFile(); - } - public void release() { - if (disableLocks) - return; - lockFile.delete(); - } - public boolean isLocked() { - if (disableLocks) - return false; - return lockFile.exists(); - } - - public String toString() { - return "Lock@" + lockFile; - } - }; - } - - private StringBuffer getLockPrefix() { + + public String getLockID() { String dirName; // name to be hashed try { dirName = directory.getCanonicalPath(); @@ -402,7 +450,7 @@ buf.append(HEX_DIGITS[b & 0xf]); } - return buf; + return buf.toString(); } /** Closes the store to future operations. */