Index: lucene/contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java (revision 1160142) +++ lucene/contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java (working copy) @@ -228,22 +228,21 @@ } } - // final due to LUCENE-3382: currently CFS backdoors the directory to create CFE - // by using the basic implementation and not delegating, we ensure that all - // openInput/createOutput requests come thru NRTCachingDirectory. - @Override - public final CompoundFileDirectory openCompoundInput(String name, IOContext context) throws IOException { - return super.openCompoundInput(name, context); + protected IndexInputHandle createHandle(final String name, final IOContext context) throws IOException { + ensureOpen(); + if (VERBOSE) { + System.out.println("nrtdir.openInput name=" + name); + } + if (cache.fileExists(name)) { + if (VERBOSE) { + System.out.println(" from cache"); + } + return cache.createHandle(name, context); + } else { + return delegate.createHandle(name, context); + } } - // final due to LUCENE-3382: currently CFS backdoors the directory to create CFE - // by using the basic implementation and not delegating, we ensure that all - // openInput/createOutput requests come thru NRTCachingDirectory. - @Override - public final CompoundFileDirectory createCompoundOutput(String name, IOContext context) throws IOException { - return super.createCompoundOutput(name, context); - } - /** Close this directory, which flushes any cached files * to the delegate and then closes the delegate. */ @Override Index: lucene/contrib/misc/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java =================================================================== --- lucene/contrib/misc/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java (revision 1160142) +++ lucene/contrib/misc/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java (working copy) @@ -148,7 +148,7 @@ // LUCENE-3382 test that delegate compound files correctly. public void testCompoundFileAppendTwice() throws IOException { Directory newDir = new NRTCachingDirectory(newDirectory(), 2.0, 25.0); - CompoundFileDirectory csw = newDir.createCompoundOutput("d.cfs", newIOContext(random)); + CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random), true); createSequenceFile(newDir, "d1", (byte) 0, 15); IndexOutput out = csw.createOutput("d.xyz", newIOContext(random)); out.writeInt(0); @@ -164,7 +164,7 @@ csw.close(); - CompoundFileDirectory cfr = newDir.openCompoundInput("d.cfs", newIOContext(random)); + CompoundFileDirectory cfr = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random), false); assertEquals(1, cfr.listAll().length); assertEquals("d.xyz", cfr.listAll()[0]); cfr.close(); Index: lucene/src/java/org/apache/lucene/index/IndexReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexReader.java (revision 1160142) +++ lucene/src/java/org/apache/lucene/index/IndexReader.java (working copy) @@ -1435,7 +1435,7 @@ String dirname = file.getAbsoluteFile().getParent(); filename = file.getName(); dir = FSDirectory.open(new File(dirname)); - cfr = dir.openCompoundInput(filename, IOContext.DEFAULT); + cfr = new CompoundFileDirectory(dir, filename, IOContext.DEFAULT, false); String [] files = cfr.listAll(); ArrayUtil.mergeSort(files); // sort the array of filename so that the output is more readable Index: lucene/src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexWriter.java (revision 1160142) +++ lucene/src/java/org/apache/lucene/index/IndexWriter.java (working copy) @@ -2262,7 +2262,7 @@ String compoundFileName = IndexFileNames.segmentFileName(newSegment.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION); message("creating compound file " + compoundFileName); // Now build compound file - final Directory cfsDir = directory.createCompoundOutput(compoundFileName, context); + final Directory cfsDir = new CompoundFileDirectory(directory, compoundFileName, context, true); IOException prior = null; try { for(String fileName : newSegment.files()) { @@ -2594,7 +2594,7 @@ private void copySegmentIntoCFS(SegmentInfo info, String segName, IOContext context) throws IOException { String segFileName = IndexFileNames.segmentFileName(segName, "", IndexFileNames.COMPOUND_FILE_EXTENSION); Collection files = info.files(); - final CompoundFileDirectory cfsdir = directory.createCompoundOutput(segFileName, context); + final CompoundFileDirectory cfsdir = new CompoundFileDirectory(directory, segFileName, context, true); try { for (String file : files) { String newFileName = segName + IndexFileNames.stripSegmentName(file); Index: lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java (revision 1160142) +++ lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java (working copy) @@ -76,7 +76,7 @@ try { Directory dir0 = dir; if (si.getUseCompoundFile()) { - cfsReader = dir.openCompoundInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context); + cfsReader = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false); dir0 = cfsReader; } cfsDir = dir0; @@ -140,9 +140,9 @@ if (si.getDocStoreOffset() != -1) { if (si.getDocStoreIsCompoundFile()) { assert storeCFSReader == null; - storeCFSReader = dir.openCompoundInput( + storeCFSReader = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(si.getDocStoreSegment(), "", IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), - context); + context, false); storeDir = storeCFSReader; assert storeDir != null; } else { @@ -154,7 +154,7 @@ // was not used, but then we are asked to open doc // stores after the segment has switched to CFS if (cfsReader == null) { - cfsReader = dir.openCompoundInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context); + cfsReader = new CompoundFileDirectory(dir,IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false); } storeDir = cfsReader; assert storeDir != null; Index: lucene/src/java/org/apache/lucene/index/SegmentInfo.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentInfo.java (revision 1160142) +++ lucene/src/java/org/apache/lucene/index/SegmentInfo.java (working copy) @@ -30,6 +30,7 @@ import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; +import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -247,7 +248,7 @@ } final Directory dirToTest; if (isCompoundFile) { - dirToTest = dir.openCompoundInput(IndexFileNames.segmentFileName(storesSegment, "", ext), IOContext.READONCE); + dirToTest = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(storesSegment, "", ext), IOContext.READONCE, false); } else { dirToTest = dir; } @@ -265,8 +266,8 @@ if (fieldInfos == null) { Directory dir0 = dir; if (isCompoundFile && checkCompoundFile) { - dir0 = dir.openCompoundInput(IndexFileNames.segmentFileName(name, - "", IndexFileNames.COMPOUND_FILE_EXTENSION), IOContext.READONCE); + dir0 = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(name, + "", IndexFileNames.COMPOUND_FILE_EXTENSION), IOContext.READONCE, false); } try { fieldInfos = new FieldInfos(dir0, IndexFileNames.segmentFileName(name, Index: lucene/src/java/org/apache/lucene/index/SegmentMerger.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentMerger.java (revision 1160142) +++ lucene/src/java/org/apache/lucene/index/SegmentMerger.java (working copy) @@ -153,7 +153,7 @@ // Now merge all added files Collection files = info.files(); - CompoundFileDirectory cfsDir = directory.createCompoundOutput(fileName, context); + CompoundFileDirectory cfsDir = new CompoundFileDirectory(directory, fileName, context, true); try { for (String file : files) { assert !IndexFileNames.matchesExtension(file, IndexFileNames.DELETES_EXTENSION) Index: lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java (revision 1160142) +++ lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java (working copy) @@ -28,6 +28,7 @@ import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.values.Writer; +import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.util.BytesRef; @@ -51,9 +52,9 @@ this.bytesUsed = state.bytesUsed; this.context = state.context; //TODO maybe we should enable a global CFS that all codecs can pull on demand to further reduce the number of files? - this.directory = useCompoundFile ? state.directory.createCompoundOutput( + this.directory = useCompoundFile ? new CompoundFileDirectory(state.directory, IndexFileNames.segmentFileName(segmentName, codecId, - IndexFileNames.COMPOUND_FILE_EXTENSION), context) : state.directory; + IndexFileNames.COMPOUND_FILE_EXTENSION), context, true) : state.directory; this.comparator = comparator; this.useCompoundFile = useCompoundFile; } Index: lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java (revision 1160142) +++ lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java (working copy) @@ -32,6 +32,7 @@ import org.apache.lucene.index.values.Floats; import org.apache.lucene.index.values.Ints; import org.apache.lucene.index.values.ValueType; +import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.util.BytesRef; @@ -78,7 +79,7 @@ this.sortComparator = sortComparator; final Directory directory; if (useCompoundFile) { - cfs = directory = dir.openCompoundInput(IndexFileNames.segmentFileName(si.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION), context); + cfs = directory = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(si.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION), context, false); } else { cfs = null; directory = dir; Index: lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java (revision 1160142) +++ lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java (working copy) @@ -27,6 +27,7 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.ChecksumIndexInput; +import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -68,13 +69,13 @@ Directory dir = directory; if (si.getDocStoreOffset() != -1) { if (si.getDocStoreIsCompoundFile()) { - dir = dir.openCompoundInput(IndexFileNames.segmentFileName( + dir = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName( si.getDocStoreSegment(), "", - IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), context); + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), context, false); } } else if (si.getUseCompoundFile()) { - dir = dir.openCompoundInput(IndexFileNames.segmentFileName( - si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context); + dir = new CompoundFileDirectory(dir,IndexFileNames.segmentFileName( + si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false); } try { Index: lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java (revision 1160142) +++ lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java (working copy) @@ -34,7 +34,7 @@ * Directory methods that would normally modify data throw an exception. * @lucene.experimental */ -public abstract class CompoundFileDirectory extends Directory { +public final class CompoundFileDirectory extends Directory { /** Offset/Length for a slice inside of a compound file */ public static final class FileEntry { @@ -45,41 +45,46 @@ private final Directory directory; private final String fileName; protected final int readBufferSize; - private Map entries; - private boolean openForWrite; + private final Map entries; + private final boolean openForWrite; private static final Map SENTINEL = Collections.emptyMap(); - private CompoundFileWriter writer; + private final CompoundFileWriter writer; + private final IndexInputHandle handle; /** * Create a new CompoundFileDirectory. *

* NOTE: subclasses must call {@link #initForRead(Map)} before the directory can be used. */ - public CompoundFileDirectory(Directory directory, String fileName, IOContext context) throws IOException { - + public CompoundFileDirectory(Directory directory, String fileName, IOContext context, boolean openForWrite) throws IOException { this.directory = directory; this.fileName = fileName; this.readBufferSize = BufferedIndexInput.bufferSize(context); this.isOpen = false; + this.openForWrite = openForWrite; + if (!openForWrite) { + IndexInput openInput = directory.openInput(fileName, IOContext.READONCE); + boolean success = false; + try { + this.entries = readEntries(openInput, directory, fileName); + success = true; + } finally { + IOUtils.closeSafely(!success, openInput); + } + handle = directory.createHandle(fileName, context); + this.isOpen = true; + writer = null; + } else { + assert !(directory instanceof CompoundFileDirectory) : "compound file inside of compound file: " + fileName; + this.entries = SENTINEL; + this.isOpen = true; + writer = new CompoundFileWriter(directory, fileName); + handle = null; + } } - - /** Initialize with a map of filename->slices */ - protected final void initForRead(Map entries) { - this.entries = entries; - this.isOpen = true; - this.openForWrite = false; - } - - protected final void initForWrite() throws IOException { - assert !(directory instanceof CompoundFileDirectory) : "compound file inside of compound file: " + fileName; - this.entries = SENTINEL; - this.openForWrite = true; - this.isOpen = true; - writer = new CompoundFileWriter(directory, fileName); - } - + /** Helper method that reads CFS entries from an input stream */ - public static final Map readEntries(IndexInput stream, Directory dir, String name) throws IOException { + private static final Map readEntries(IndexInput stream, Directory dir, String name) throws IOException { // read the first VInt. If it is negative, it's the version number // otherwise it's the count (pre-3.1 indexes) final int firstInt = stream.readVInt(); @@ -173,32 +178,29 @@ public synchronized void close() throws IOException { if (!isOpen) { // allow double close - usually to be consistent with other closeables - assert entries == null; return; // already closed } - entries = null; isOpen = false; if (writer != null) { assert openForWrite; writer.close(); + } else { + IOUtils.closeSafely(false, handle); } } @Override - public synchronized IndexInput openInput(String fileName, IOContext context) throws IOException { + public synchronized IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); assert !openForWrite; - final String id = IndexFileNames.stripSegmentName(fileName); + final String id = IndexFileNames.stripSegmentName(name); final FileEntry entry = entries.get(id); if (entry == null) { - throw new IOException("No sub-file with id " + id + " found (fileName=" + fileName + " files: " + entries.keySet() + ")"); + throw new IOException("No sub-file with id " + id + " found (fileName=" + name + " files: " + entries.keySet() + ")"); } - return openInputSlice(id, entry.offset, entry.length, readBufferSize); + return handle.openSlice(entry.offset, entry.length); } - /** Return an IndexInput that represents a "slice" or portion of the CFS file. */ - public abstract IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException; - /** Returns an array of strings, one for each file in the directory. */ @Override public String[] listAll() { @@ -279,51 +281,24 @@ public Lock makeLock(String name) { throw new UnsupportedOperationException(); } - + @Override - public CompoundFileDirectory openCompoundInput(String name, IOContext context) throws IOException { - FileEntry fileEntry = this.entries.get(IndexFileNames.stripSegmentName(name)); - if (fileEntry == null) { - throw new FileNotFoundException("file " + name + " does not exists in this CFS"); - } - return new NestedCompoundFileDirectory(name, context, fileEntry.offset, fileEntry.length); - } - - /** Not implemented - * @throws UnsupportedOperationException */ - @Override - public CompoundFileDirectory createCompoundOutput(String name, IOContext context) + protected IndexInputHandle createHandle(final String name, IOContext context) throws IOException { - throw new UnsupportedOperationException("can not create nested CFS, create seperately and use Directory.copy instead"); - } - - private class NestedCompoundFileDirectory extends CompoundFileDirectory { - - private final long cfsOffset; - private final long cfsLength; - - public NestedCompoundFileDirectory(String fileName, IOContext context, long offset, long length) - throws IOException { - super(directory, fileName, context); - this.cfsOffset = offset; - this.cfsLength = length; - IndexInput input = null; - try { - input = CompoundFileDirectory.this.openInput(fileName, IOContext.READONCE); - initForRead(CompoundFileDirectory.readEntries(input, - CompoundFileDirectory.this, fileName)); - } finally { - IOUtils.closeSafely(false, input); - } + final String id = IndexFileNames.stripSegmentName(name); + final FileEntry entry = entries.get(id); + if (entry == null) { + throw new IOException("No sub-file with id " + id + " found (fileName=" + name + " files: " + entries.keySet() + ")"); } - - @Override - public IndexInput openInputSlice(String id, long offset, long length, - int readBufferSize) throws IOException { - assert offset + length <= cfsLength; - return CompoundFileDirectory.this.openInputSlice(id, cfsOffset + offset, length, readBufferSize); - } - + return new IndexInputHandle() { + @Override + public void close() throws IOException { + } + + @Override + protected IndexInput openSlice(long offset, long length) throws IOException { + return handle.openSlice(entry.offset + offset, length); + } + }; } - } Index: lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java (revision 1160142) +++ lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java (working copy) @@ -190,6 +190,7 @@ private final long copyFileEntry(IndexOutput dataOut, FileEntry fileEntry) throws IOException, MergeAbortedException { final IndexInput is = fileEntry.dir.openInput(fileEntry.file, IOContext.READONCE); + boolean success = false; try { final long startPtr = dataOut.getFilePointer(); final long length = fileEntry.length; @@ -201,11 +202,14 @@ throw new IOException("Difference in the output file offsets " + diff + " does not match the original file length " + length); fileEntry.offset = startPtr; - // copy successful - delete file - fileEntry.dir.deleteFile(fileEntry.file); + success = true; return length; } finally { - is.close(); + IOUtils.closeSafely(!success, is); + if (success) { + // copy successful - delete file + fileEntry.dir.deleteFile(fileEntry.file); + } } } Index: lucene/src/java/org/apache/lucene/store/DefaultCompoundFileDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/DefaultCompoundFileDirectory.java (revision 1160142) +++ lucene/src/java/org/apache/lucene/store/DefaultCompoundFileDirectory.java (working copy) @@ -1,140 +0,0 @@ -package org.apache.lucene.store; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.util.IOUtils; - -/** - * Default implementation of {@link CompoundFileDirectory}. - *

- * This implementation returns a BufferedIndexInput that wraps the underlying - * Directory's IndexInput for the compound file (using unbuffered reads). - * @lucene.experimental - */ -public class DefaultCompoundFileDirectory extends CompoundFileDirectory { - protected IndexInput stream; - - public DefaultCompoundFileDirectory(Directory directory, String fileName, IOContext context, boolean writeable) throws IOException { - super(directory, fileName, context); - if (!writeable) { - try { - stream = directory.openInput(fileName, context); - initForRead(CompoundFileDirectory.readEntries(stream, directory, fileName)); - } catch (IOException e) { - IOUtils.closeSafely(e, stream); - } - } else { - initForWrite(); - } - } - - @Override - public IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException { - return new CSIndexInput(stream, offset, length, readBufferSize); - } - - @Override - public synchronized void close() throws IOException { - try { - IOUtils.closeSafely(false, stream); - } finally { - super.close(); - } - } - - /** Implementation of an IndexInput that reads from a portion of the - * compound file. - */ - static final class CSIndexInput extends BufferedIndexInput { - IndexInput base; - long fileOffset; - long length; - - CSIndexInput(final IndexInput base, final long fileOffset, final long length) { - this(base, fileOffset, length, BufferedIndexInput.BUFFER_SIZE); - } - - CSIndexInput(final IndexInput base, final long fileOffset, final long length, int readBufferSize) { - super(readBufferSize); - this.base = (IndexInput)base.clone(); - this.fileOffset = fileOffset; - this.length = length; - } - - @Override - public Object clone() { - CSIndexInput clone = (CSIndexInput)super.clone(); - clone.base = (IndexInput)base.clone(); - clone.fileOffset = fileOffset; - clone.length = length; - return clone; - } - - /** Expert: implements buffer refill. Reads bytes from the current - * position in the input. - * @param b the array to read bytes into - * @param offset the offset in the array to start storing bytes - * @param len the number of bytes to read - */ - @Override - protected void readInternal(byte[] b, int offset, int len) throws IOException { - long start = getFilePointer(); - if(start + len > length) - throw new IOException("read past EOF"); - base.seek(fileOffset + start); - base.readBytes(b, offset, len, false); - } - - /** Expert: implements seek. Sets current position in this file, where - * the next {@link #readInternal(byte[],int,int)} will occur. - * @see #readInternal(byte[],int,int) - */ - @Override - protected void seekInternal(long pos) {} - - /** Closes the stream to further operations. */ - @Override - public void close() throws IOException { - base.close(); - } - - @Override - public long length() { - return length; - } - - @Override - public void copyBytes(IndexOutput out, long numBytes) throws IOException { - // Copy first whatever is in the buffer - numBytes -= flushBuffer(out, numBytes); - - // If there are more bytes left to copy, delegate the copy task to the - // base IndexInput, in case it can do an optimized copy. - if (numBytes > 0) { - long start = getFilePointer(); - if (start + numBytes > length) { - throw new IOException("read past EOF"); - } - base.seek(fileOffset + start); - base.copyBytes(out, numBytes); - } - } - } -} Index: lucene/src/java/org/apache/lucene/store/Directory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/Directory.java (revision 1160142) +++ lucene/src/java/org/apache/lucene/store/Directory.java (working copy) @@ -112,30 +112,6 @@ */ public abstract IndexInput openInput(String name, IOContext context) throws IOException; - /** - * Returns a {@link CompoundFileDirectory} capable of - * reading the Lucene compound file format. - *

- * The default implementation returns - * {@link DefaultCompoundFileDirectory}. - * @lucene.experimental - */ - public CompoundFileDirectory openCompoundInput(String name, IOContext context) throws IOException { - return new DefaultCompoundFileDirectory(this, name, context, false); - } - - /** - * Returns a {@link CompoundFileDirectory} capable of - * writing the Lucene compound file format. - *

- * The default implementation returns - * {@link DefaultCompoundFileDirectory}. - * @lucene.experimental - */ - public CompoundFileDirectory createCompoundOutput(String name, IOContext context) throws IOException { - return new DefaultCompoundFileDirectory(this, name, context, true); - } - /** Construct a {@link Lock}. * @param name the name of the lock file */ @@ -231,12 +207,121 @@ IOUtils.closeSafely(priorException, os, is); } } + /** + * nocommit - javadoc + * @throws IOException + * @lucene.internal + * @lucene.experimental + */ + //nocommit find a better name? + protected IndexInputHandle createHandle(final String name, final IOContext context) throws IOException { + return new IndexInputHandle() { + private final IndexInput base = Directory.this.openInput(name, context); + @Override + protected IndexInput openSlice(long offset, long length) { + return new CSIndexInput(base, offset, length); + } + @Override + public void close() throws IOException { + base.close(); + } + }; + } + + /** * @throws AlreadyClosedException if this Directory is closed */ protected final void ensureOpen() throws AlreadyClosedException { if (!isOpen) throw new AlreadyClosedException("this Directory is closed"); } + + /** + * @lucene.internal + * @lucene.experimental + */ + protected abstract class IndexInputHandle implements Closeable { + protected abstract IndexInput openSlice(long offset, long length) throws IOException; + } + + /** Implementation of an IndexInput that reads from a portion of + * a file. + */ + static final class CSIndexInput extends BufferedIndexInput { + IndexInput base; + long fileOffset; + long length; + + CSIndexInput(final IndexInput base, final long fileOffset, final long length) { + this(base, fileOffset, length, BufferedIndexInput.BUFFER_SIZE); + } + + CSIndexInput(final IndexInput base, final long fileOffset, final long length, int readBufferSize) { + super(readBufferSize); + this.base = (IndexInput) base.clone(); + this.fileOffset = fileOffset; + this.length = length; + } + + @Override + public Object clone() { + CSIndexInput clone = (CSIndexInput)super.clone(); + clone.base = (IndexInput)base.clone(); + clone.fileOffset = fileOffset; + clone.length = length; + return clone; + } + + /** Expert: implements buffer refill. Reads bytes from the current + * position in the input. + * @param b the array to read bytes into + * @param offset the offset in the array to start storing bytes + * @param len the number of bytes to read + */ + @Override + protected void readInternal(byte[] b, int offset, int len) throws IOException { + long start = getFilePointer(); + if(start + len > length) + throw new IOException("read past EOF"); + base.seek(fileOffset + start); + base.readBytes(b, offset, len, false); + } + + /** Expert: implements seek. Sets current position in this file, where + * the next {@link #readInternal(byte[],int,int)} will occur. + * @see #readInternal(byte[],int,int) + */ + @Override + protected void seekInternal(long pos) {} + + /** Closes the stream to further operations. */ + @Override + public void close() throws IOException { + base.close(); + } + + @Override + public long length() { + return length; + } + + @Override + public void copyBytes(IndexOutput out, long numBytes) throws IOException { + // Copy first whatever is in the buffer + numBytes -= flushBuffer(out, numBytes); + + // If there are more bytes left to copy, delegate the copy task to the + // base IndexInput, in case it can do an optimized copy. + if (numBytes > 0) { + long start = getFilePointer(); + if (start + numBytes > length) { + throw new IOException("read past EOF"); + } + base.seek(fileOffset + start); + base.copyBytes(out, numBytes); + } + } + } } Index: lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java (revision 1160142) +++ lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java (working copy) @@ -177,19 +177,9 @@ return getDirectory(name).openInput(name, context); } - // final due to LUCENE-3380: currently CFS backdoors the directory to create CFE - // by using the basic implementation and not delegating, we ensure that all - // openInput/createOutput requests come thru NRTCachingDirectory. @Override - public final CompoundFileDirectory openCompoundInput(String name, IOContext context) throws IOException { - return super.openCompoundInput(name, context); + protected IndexInputHandle createHandle(String name, IOContext context) + throws IOException { + return getDirectory(name).createHandle(name, context); } - - // final due to LUCENE-3380: currently CFS backdoors the directory to create CFE - // by using the basic implementation and not delegating, we ensure that all - // openInput/createOutput requests come thru NRTCachingDirectory. - @Override - public final CompoundFileDirectory createCompoundOutput(String name, IOContext context) throws IOException { - return super.createCompoundOutput(name, context); - } } Index: lucene/src/java/org/apache/lucene/store/MMapDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/MMapDirectory.java (revision 1160142) +++ lucene/src/java/org/apache/lucene/store/MMapDirectory.java (working copy) @@ -32,7 +32,6 @@ import java.lang.reflect.Method; import org.apache.lucene.util.Constants; -import org.apache.lucene.util.IOUtils; /** File-based {@link Directory} implementation that uses * mmap for reading, and {@link @@ -220,42 +219,22 @@ } } - @Override - public CompoundFileDirectory openCompoundInput(String name, IOContext context) throws IOException { - return new MMapCompoundFileDirectory(name, context); - } - - private final class MMapCompoundFileDirectory extends CompoundFileDirectory { - private RandomAccessFile raf = null; + protected IndexInputHandle createHandle(final String name, final IOContext context) throws IOException { + ensureOpen(); + File f = new File(getDirectory(), name); + final RandomAccessFile raf = new RandomAccessFile(f, "r"); + return new IndexInputHandle() { - public MMapCompoundFileDirectory(String fileName, IOContext context) throws IOException { - super(MMapDirectory.this, fileName, context); - IndexInput stream = null; - try { - File f = new File(MMapDirectory.this.getDirectory(), fileName); - raf = new RandomAccessFile(f, "r"); - stream = new MMapIndexInput(raf, 0, raf.length(), chunkSizePower); - initForRead(CompoundFileDirectory.readEntries(stream, MMapDirectory.this, fileName)); - stream.close(); - } catch (IOException e) { - // throw our original exception - IOUtils.closeSafely(e, raf, stream); + @Override + public void close() throws IOException { + raf.close(); } - } - @Override - public IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException { - return new MMapIndexInput(raf, offset, length, chunkSizePower); - } - - @Override - public synchronized void close() throws IOException { - try { - raf.close(); - } finally { - super.close(); + @Override + protected IndexInput openSlice(long offset, long length) throws IOException { + return new MMapIndexInput(raf, offset, length, chunkSizePower); } - } + }; } // Because Java's ByteBuffer uses an int to address the @@ -273,12 +252,15 @@ private ByteBuffer curBuf; // redundant for speed: buffers[curBufIndex] private boolean isClone = false; + + private final RandomAccessFile raf; MMapIndexInput(RandomAccessFile raf, long offset, long length, int chunkSizePower) throws IOException { this.length = length; this.chunkSizePower = chunkSizePower; this.chunkSize = 1L << chunkSizePower; this.chunkSizeMask = chunkSize - 1L; + this.raf = raf; if (chunkSizePower < 0 || chunkSizePower > 30) throw new IllegalArgumentException("Invalid chunkSizePower used for ByteBuffer size: " + chunkSizePower); Index: lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java (revision 1160142) +++ lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java (working copy) @@ -24,8 +24,7 @@ import java.nio.channels.FileChannel; import java.util.concurrent.Future; // javadoc -import org.apache.lucene.store.SimpleFSDirectory.SimpleFSIndexInput; -import org.apache.lucene.util.IOUtils; +import org.apache.lucene.store.SimpleFSDirectory.SimpleFSIndexInput.Descriptor; /** * An {@link FSDirectory} implementation that uses java.nio's FileChannel's @@ -81,45 +80,24 @@ return new NIOFSIndexInput(new File(getDirectory(), name), context, getReadChunkSize()); } - @Override - public CompoundFileDirectory openCompoundInput(String name, IOContext context) throws IOException { - return new NIOFSCompoundFileDirectory(name, context); - } + protected IndexInputHandle createHandle(final String name, + final IOContext context) throws IOException { + ensureOpen(); + final File file = new File(getDirectory(), name); + final Descriptor descriptor = new Descriptor(file, "r"); + return new IndexInputHandle() { - private final class NIOFSCompoundFileDirectory extends CompoundFileDirectory { - private SimpleFSIndexInput.Descriptor fd; - private FileChannel fc; - - public NIOFSCompoundFileDirectory(String fileName, IOContext context) throws IOException { - super(NIOFSDirectory.this, fileName, context); - IndexInput stream = null; - try { - File f = new File(NIOFSDirectory.this.getDirectory(), fileName); - fd = new SimpleFSIndexInput.Descriptor(f, "r"); - fc = fd.getChannel(); - stream = new NIOFSIndexInput(fd, fc, 0, fd.length, readBufferSize, - getReadChunkSize()); - initForRead(CompoundFileDirectory.readEntries(stream, NIOFSDirectory.this, fileName)); - stream.close(); - } catch (IOException e) { - // throw our original exception - IOUtils.closeSafely(e, fc, fd, stream); + @Override + public void close() throws IOException { + descriptor.close(); } - } - - @Override - public IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException { - return new NIOFSIndexInput(fd, fc, offset, length, readBufferSize, getReadChunkSize()); - } - @Override - public synchronized void close() throws IOException { - try { - IOUtils.closeSafely(false, fc, fd); - } finally { - super.close(); + @Override + protected IndexInput openSlice(long offset, long length) throws IOException { + return new NIOFSIndexInput(descriptor, descriptor.getChannel(), offset, + length, BufferedIndexInput.bufferSize(context), getReadChunkSize()); } - } + }; } protected static class NIOFSIndexInput extends SimpleFSDirectory.SimpleFSIndexInput { Index: lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java (revision 1160142) +++ lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java (working copy) @@ -21,9 +21,8 @@ import java.io.IOException; import java.io.RandomAccessFile; -import org.apache.lucene.util.IOUtils; +import org.apache.lucene.store.SimpleFSDirectory.SimpleFSIndexInput.Descriptor; - /** A straightforward implementation of {@link FSDirectory} * using java.io.RandomAccessFile. However, this class has * poor concurrent performance (multiple threads will @@ -59,45 +58,29 @@ return new SimpleFSIndexInput(new File(directory, name), context, getReadChunkSize()); } - @Override - public CompoundFileDirectory openCompoundInput(String name, IOContext context) throws IOException { - return new SimpleFSCompoundFileDirectory(name, context); - } + - private final class SimpleFSCompoundFileDirectory extends CompoundFileDirectory { - private SimpleFSIndexInput.Descriptor fd; + protected IndexInputHandle createHandle(final String name, + final IOContext context) throws IOException { + ensureOpen(); + final File file = new File(getDirectory(), name); + final Descriptor descriptor = new Descriptor(file, "r"); + return new IndexInputHandle() { - public SimpleFSCompoundFileDirectory(String fileName, IOContext context) throws IOException { - super(SimpleFSDirectory.this, fileName, context); - IndexInput stream = null; - try { - final File f = new File(SimpleFSDirectory.this.getDirectory(), fileName); - fd = new SimpleFSIndexInput.Descriptor(f, "r"); - stream = new SimpleFSIndexInput(fd, 0, fd.length, readBufferSize, - getReadChunkSize()); - initForRead(CompoundFileDirectory.readEntries(stream, SimpleFSDirectory.this, fileName)); - stream.close(); - } catch (IOException e) { - // throw our original exception - IOUtils.closeSafely(e, fd, stream); + @Override + public void close() throws IOException { + descriptor.close(); } - } - @Override - public IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException { - return new SimpleFSIndexInput(fd, offset, length, readBufferSize, getReadChunkSize()); - } - - @Override - public synchronized void close() throws IOException { - try { - fd.close(); - } finally { - super.close(); + @Override + public IndexInput openSlice(long offset, long length) throws IOException { + return new SimpleFSIndexInput(descriptor, offset, + length, BufferedIndexInput.bufferSize(context), getReadChunkSize()); } - } + }; } + protected static class SimpleFSIndexInput extends BufferedIndexInput { protected static class Descriptor extends RandomAccessFile { Index: lucene/src/test-framework/org/apache/lucene/store/MockCompoundFileDirectoryWrapper.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/store/MockCompoundFileDirectoryWrapper.java (revision 1160142) +++ lucene/src/test-framework/org/apache/lucene/store/MockCompoundFileDirectoryWrapper.java (working copy) @@ -1,149 +0,0 @@ -package org.apache.lucene.store; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Collection; - -public class MockCompoundFileDirectoryWrapper extends CompoundFileDirectory { - private final MockDirectoryWrapper parent; - private final CompoundFileDirectory delegate; - private final String name; - - public MockCompoundFileDirectoryWrapper(String name, MockDirectoryWrapper parent, CompoundFileDirectory delegate, boolean forWrite) throws IOException { - super(parent, name, IOContext.DEFAULT); - this.name = name; - this.parent = parent; - this.delegate = delegate; - // don't initialize here since we delegate everything - if not initialized a direct call will cause an assert to fail! - parent.addFileHandle(this, name, !forWrite); - } - - @Override - public Directory getDirectory() { - return delegate.getDirectory(); - } - - @Override - public String getName() { - return delegate.getName(); - } - - @Override - public synchronized void close() throws IOException { - delegate.close(); - parent.removeOpenFile(this, name); - } - - @Override - public synchronized IndexInput openInput(String id, IOContext context) throws IOException { - return delegate.openInput(id, context); - } - - @Override - public String[] listAll() { - return delegate.listAll(); - } - - @Override - public boolean fileExists(String name) { - return delegate.fileExists(name); - } - - @Override - public long fileModified(String name) throws IOException { - return delegate.fileModified(name); - } - - @Override - public void deleteFile(String name) { - delegate.deleteFile(name); - } - - @Override - public void renameFile(String from, String to) { - delegate.renameFile(from, to); - } - - @Override - public long fileLength(String name) throws IOException { - return delegate.fileLength(name); - } - - @Override - public IndexOutput createOutput(String name, IOContext context) throws IOException { - return delegate.createOutput(name, context); - } - - @Override - public void sync(Collection names) throws IOException { - delegate.sync(names); - } - - @Override - public Lock makeLock(String name) { - return delegate.makeLock(name); - } - - @Override - public void clearLock(String name) throws IOException { - delegate.clearLock(name); - } - - @Override - public void setLockFactory(LockFactory lockFactory) throws IOException { - delegate.setLockFactory(lockFactory); - } - - @Override - public LockFactory getLockFactory() { - return delegate.getLockFactory(); - } - - @Override - public String getLockID() { - return delegate.getLockID(); - } - - @Override - public String toString() { - return "MockCompoundFileDirectoryWrapper(" + super.toString() + ")"; - } - - @Override - public void copy(Directory to, String src, String dest, IOContext context) throws IOException { - delegate.copy(to, src, dest, context); - } - - @Override - public IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException { - return delegate.openInputSlice(id, offset, length, readBufferSize); - } - - @Override - public CompoundFileDirectory createCompoundOutput(String name, IOContext context) throws IOException { - return delegate.createCompoundOutput(name, context); - } - - @Override - public CompoundFileDirectory openCompoundInput(String name, IOContext context) - throws IOException { - return delegate.openCompoundInput(name, context); - } - -} Index: lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java (revision 1160142) +++ lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java (working copy) @@ -421,18 +421,6 @@ return ii; } - @Override - public synchronized CompoundFileDirectory openCompoundInput(String name, IOContext context) throws IOException { - maybeYield(); - return new MockCompoundFileDirectoryWrapper(name, this, delegate.openCompoundInput(name, context), false); - } - - @Override - public CompoundFileDirectory createCompoundOutput(String name, IOContext context) throws IOException { - maybeYield(); - return new MockCompoundFileDirectoryWrapper(name, this, delegate.createCompoundOutput(name, context), true); - } - /** Provided for testing purposes. Use sizeInBytes() instead. */ public synchronized final long getRecomputedSizeInBytes() throws IOException { if (!(delegate instanceof RAMDirectory)) @@ -658,5 +646,43 @@ // randomize the IOContext here? delegate.copy(to, src, dest, context); } + + @Override + protected IndexInputHandle createHandle(final String name, IOContext context) + throws IOException { + maybeYield(); + if (!delegate.fileExists(name)) + throw new FileNotFoundException(name); + // cannot open a file for input if it's still open for + // output, except for segments.gen and segments_N + if (openFilesForWrite.contains(name) && !name.startsWith("segments")) { + throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false); + } + final IndexInputHandle handle = delegate.createHandle(name, context); + return new IndexInputHandle() { + + @Override + public void close() throws IOException { + handle.close(); + } + + @Override + public IndexInput openSlice(long offset, long length) throws IOException { + maybeYield(); + // cannot open a file for input if it's still open for + // output, except for segments.gen and segments_N + if (openFilesForWrite.contains(name) && !name.startsWith("segments")) { + throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false); + } + + IndexInput ii = new MockIndexInputWrapper(MockDirectoryWrapper.this, name, handle.openSlice(offset, length)); + addFileHandle(ii, name, true); + return ii; + } + + }; + } + + } Index: lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (revision 1160142) +++ lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (working copy) @@ -548,7 +548,7 @@ // figure out which field number corresponds to // "content", and then set our expected file names below // accordingly: - CompoundFileDirectory cfsReader = dir.openCompoundInput("_0.cfs", newIOContext(random)); + CompoundFileDirectory cfsReader = new CompoundFileDirectory(dir, "_0.cfs", newIOContext(random), false); FieldInfos fieldInfos = new FieldInfos(cfsReader, "_0.fnm"); int contentFieldIndex = -1; for (FieldInfo fi : fieldInfos) { Index: lucene/src/test/org/apache/lucene/index/TestCompoundFile.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestCompoundFile.java (revision 1160142) +++ lucene/src/test/org/apache/lucene/index/TestCompoundFile.java (working copy) @@ -182,11 +182,11 @@ for (int i=0; i