Index: src/test/org/apache/lucene/index/store/TestRAMDirectory.java =================================================================== --- src/test/org/apache/lucene/index/store/TestRAMDirectory.java (revision 475873) +++ src/test/org/apache/lucene/index/store/TestRAMDirectory.java (working copy) @@ -44,6 +44,7 @@ public class TestRAMDirectory extends TestCase { private File indexDir = null; + private long indexDirSizeInBytes = 0; // add enough document so that the index will be larger than RAMDirectory.READ_BUFFER_SIZE private final int docsToAdd = 500; @@ -64,8 +65,11 @@ writer.addDocument(doc); } assertEquals(docsToAdd, writer.docCount()); - writer.optimize(); writer.close(); + + String[] files = indexDir.list(); + for (int i=0; i 0) { mergeSegments(ramSegmentInfos, 0, ramSegmentInfos.size()); maybeMergeSegments(minMergeDocs); } } + /** Expert: Return the total size of all index files currently cached in memory. + * Useful for size management with flushRamDocs() + */ + public final long ramSizeInBytes() { + return ramDirectory.sizeInBytes(); + } + + /** Expert: Return the number of documents whose segments are currently cached in memory. + * Useful when calling flushRamSegments() + */ + public final synchronized int numRamDocs() { + return ramSegmentInfos.size(); + } + /** Incremental segment merger. */ private final void maybeMergeSegments(int startUpperBound) throws IOException { long lowerBound = -1; Index: src/java/org/apache/lucene/store/RAMDirectory.java =================================================================== --- src/java/org/apache/lucene/store/RAMDirectory.java (revision 475873) +++ src/java/org/apache/lucene/store/RAMDirectory.java (working copy) @@ -20,12 +20,11 @@ import java.io.IOException; import java.io.File; import java.io.Serializable; -import java.util.Hashtable; +import java.util.Collection; import java.util.Enumeration; - -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Set; /** * A memory-resident {@link Directory} implementation. Locking @@ -38,7 +37,14 @@ private static final long serialVersionUID = 1l; - Hashtable files = new Hashtable(); + private HashMap fileMap = new HashMap(); + private Set fileNames = fileMap.keySet(); + private Collection files = fileMap.values(); + private long sizeInBytes = 0; + + // ***** + // Lock acquisition sequence: RAMDirectory, then RAMFile + // ***** /** Constructs an empty {@link Directory}. */ public RAMDirectory() { @@ -105,78 +111,119 @@ } /** Returns an array of strings, one for each file in the directory. */ - public final String[] list() { - String[] result = new String[files.size()]; - int i = 0; - Enumeration names = files.keys(); - while (names.hasMoreElements()) - result[i++] = (String)names.nextElement(); - return result; + public synchronized final String[] list() { + String[] result = new String[fileNames.size()]; + int i = 0; + Iterator it = fileNames.iterator(); + while (it.hasNext()) + result[i++] = (String)it.next(); + return result; } /** Returns true iff the named file exists in this directory. */ public final boolean fileExists(String name) { - RAMFile file = (RAMFile)files.get(name); + RAMFile file; + synchronized (this) { + file = (RAMFile)fileMap.get(name); + } return file != null; } /** Returns the time the named file was last modified. */ public final long fileModified(String name) { - RAMFile file = (RAMFile)files.get(name); - return file.lastModified; + RAMFile file; + synchronized (this) { + file = (RAMFile)fileMap.get(name); + } + return file.getLastModified(); } /** Set the modified time of an existing file to now. */ public void touchFile(String name) { -// final boolean MONITOR = false; + RAMFile file; + synchronized (this) { + file = (RAMFile)fileMap.get(name); + } - RAMFile file = (RAMFile)files.get(name); long ts2, ts1 = System.currentTimeMillis(); do { try { Thread.sleep(0, 1); } catch (InterruptedException e) {} ts2 = System.currentTimeMillis(); -// if (MONITOR) { -// count++; -// } } while(ts1 == ts2); - - file.lastModified = ts2; - -// if (MONITOR) -// System.out.println("SLEEP COUNT: " + count); + + file.setLastModified(ts2); } /** Returns the length in bytes of a file in the directory. */ public final long fileLength(String name) { - RAMFile file = (RAMFile)files.get(name); - return file.length; + RAMFile file; + synchronized (this) { + file = (RAMFile)fileMap.get(name); + } + return file.getLength(); + } + + /** Return total size in bytes of all files in this directory */ + public synchronized final long sizeInBytes() { + return sizeInBytes; + } + + synchronized final void updateSizeInBytes(long delta) { + sizeInBytes += delta; + } + + /** Provided for testing purposes. Use sizeInBytes() instead. */ + public synchronized final long getRecomputedSizeInBytes() { + long size = 0; + Iterator it = files.iterator(); + while (it.hasNext()) + size += ((RAMFile) it.next()).getLength(); + return size; } /** Removes an existing file in the directory. */ - public final void deleteFile(String name) { - files.remove(name); + public synchronized final void deleteFile(String name) { + RAMFile file = (RAMFile)fileMap.get(name); + if (file!=null) + synchronized (file) { + long length = file.length; + fileMap.remove(name); + file.directory = null; + sizeInBytes -= length; + } } /** Removes an existing file in the directory. */ - public final void renameFile(String from, String to) { - RAMFile file = (RAMFile)files.get(from); - files.remove(from); - files.put(to, file); + public synchronized final void renameFile(String from, String to) { + RAMFile fromFile = (RAMFile)fileMap.get(from); + RAMFile toFile = (RAMFile)fileMap.get(to); + if (toFile!=null) + synchronized (toFile) { + sizeInBytes -= toFile.length; + toFile.directory = null; + } + fileMap.remove(from); + fileMap.put(to, fromFile); } /** Creates a new, empty file in the directory with the given name. Returns a stream writing this file. */ public final IndexOutput createOutput(String name) { - RAMFile file = new RAMFile(); - files.put(name, file); + RAMFile file = new RAMFile(this); + synchronized (this) { + fileMap.put(name, file); + } return new RAMOutputStream(file); } /** Returns a stream reading an existing file. */ public final IndexInput openInput(String name) { - RAMFile file = (RAMFile)files.get(name); + RAMFile file; + synchronized (this) { + file = (RAMFile)fileMap.get(name); + } return new RAMInputStream(file); } Index: src/java/org/apache/lucene/store/RAMOutputStream.java =================================================================== --- src/java/org/apache/lucene/store/RAMOutputStream.java (revision 475873) +++ src/java/org/apache/lucene/store/RAMOutputStream.java (working copy) @@ -63,7 +63,7 @@ throw new RuntimeException(e.toString()); } - file.length = 0; + file.setLength(0); } public void flushBuffer(byte[] src, int len) { @@ -89,9 +89,9 @@ } if (pointer > file.length) - file.length = pointer; + file.setLength(pointer); - file.lastModified = System.currentTimeMillis(); + file.setLastModified(System.currentTimeMillis()); } public void close() throws IOException { Index: src/java/org/apache/lucene/store/RAMFile.java =================================================================== --- src/java/org/apache/lucene/store/RAMFile.java (revision 475873) +++ src/java/org/apache/lucene/store/RAMFile.java (working copy) @@ -24,7 +24,48 @@ private static final long serialVersionUID = 1l; + // Direct read-only access to state supported for streams since a writing stream implies no other concurrent streams Vector buffers = new Vector(); long length; - long lastModified = System.currentTimeMillis(); + RAMDirectory directory; + + // This is publicly modifiable via Directory.touchFile(), so direct access not supported + private long lastModified = System.currentTimeMillis(); + + // File used as buffer, in no RAMDirectory + RAMFile() {} + + RAMFile(RAMDirectory directory) { + this.directory = directory; + } + + // For non-stream access from thread that might be concurrent with writing + synchronized long getLength() { + return length; + } + + void setLength(long length) { + if (directory!=null) + // Ensure correct locking order; see RAMDirectory + synchronized (directory) { + synchronized (this) { + directory.updateSizeInBytes(length-this.length); + this.length = length; + } + } + else + synchronized (this) { + this.length = length; + } + } + + // For non-stream access from thread that might be concurrent with writing + synchronized long getLastModified() { + return lastModified; + } + + synchronized void setLastModified(long lastModified) { + this.lastModified = lastModified; + } + }