Index: lucene/contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java (revision 1135779) +++ lucene/contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java (working copy) @@ -234,6 +234,15 @@ } @Override + public synchronized CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException { + if (cache.fileExists(name)) { + return cache.openCompoundInput(name, bufferSize); + } else { + return delegate.openCompoundInput(name, bufferSize); + } + } + + @Override public synchronized IndexInput openInput(String name, int bufferSize) throws IOException { if (cache.fileExists(name)) { return cache.openInput(name, bufferSize); Index: lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (revision 1135779) +++ lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (working copy) @@ -20,6 +20,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.search.DefaultSimilarity; import org.apache.lucene.search.Similarity; +import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; @@ -91,7 +92,7 @@ // figure out which field number corresponds to // "content", and then set our expected file names below // accordingly: - CompoundFileReader cfsReader = new CompoundFileReader(dir, "_2.cfs"); + CompoundFileDirectory cfsReader = dir.openCompoundInput("_2.cfs", 1024); FieldInfos fieldInfos = new FieldInfos(cfsReader, "_2.fnm"); int contentFieldIndex = -1; for (FieldInfo fi : fieldInfos) { Index: lucene/src/test/org/apache/lucene/index/TestCompoundFile.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestCompoundFile.java (revision 1135779) +++ lucene/src/test/org/apache/lucene/index/TestCompoundFile.java (working copy) @@ -23,6 +23,8 @@ import org.apache.lucene.util.LuceneTestCase; import junit.framework.TestSuite; import junit.textui.TestRunner; + +import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; @@ -204,7 +206,7 @@ csw.addFile(name); csw.close(); - CompoundFileReader csr = new CompoundFileReader(dir, name + ".cfs"); + CompoundFileDirectory csr = dir.openCompoundInput(name + ".cfs", 1024); IndexInput expected = dir.openInput(name); IndexInput actual = csr.openInput(name); assertSameStreams(name, expected, actual); @@ -228,7 +230,7 @@ csw.addFile("d2"); csw.close(); - CompoundFileReader csr = new CompoundFileReader(dir, "d.csf"); + CompoundFileDirectory csr = dir.openCompoundInput("d.csf", 1024); IndexInput expected = dir.openInput("d1"); IndexInput actual = csr.openInput("d1"); assertSameStreams("d1", expected, actual); @@ -283,7 +285,7 @@ } csw.close(); - CompoundFileReader csr = new CompoundFileReader(dir, "test.cfs"); + CompoundFileDirectory csr = dir.openCompoundInput("test.cfs", 1024); for (int i=0; i entries = new HashMap(); - - public CompoundFileReader(Directory dir, String name) throws IOException { - this(dir, name, BufferedIndexInput.BUFFER_SIZE); - } - - public CompoundFileReader(Directory dir, String name, int readBufferSize) throws IOException { - assert !(dir instanceof CompoundFileReader) : "compound file inside of compound file: " + name; - directory = dir; - fileName = name; - this.readBufferSize = readBufferSize; - - boolean success = false; - - try { - stream = dir.openInput(name, readBufferSize); - - // read the first VInt. If it is negative, it's the version number - // otherwise it's the count (pre-3.1 indexes) - int firstInt = stream.readVInt(); - - final int count; - final boolean stripSegmentName; - if (firstInt < CompoundFileWriter.FORMAT_PRE_VERSION) { - if (firstInt < CompoundFileWriter.FORMAT_CURRENT) { - throw new CorruptIndexException("Incompatible format version: " - + firstInt + " expected " + CompoundFileWriter.FORMAT_CURRENT); - } - // It's a post-3.1 index, read the count. - count = stream.readVInt(); - stripSegmentName = false; - } else { - count = firstInt; - stripSegmentName = true; - } - - // read the directory and init files - FileEntry entry = null; - for (int i=0; i names) throws IOException { - } - - /** Not implemented - * @throws UnsupportedOperationException */ - @Override - public Lock makeLock(String name) { - throw new UnsupportedOperationException(); - } - - /** Implementation of an IndexInput that reads from a portion of the - * compound file. The visibility is left as "package" *only* because - * this helps with testing since JUnit test cases in a different class - * can then access package fields of this class. - */ - static final class CSIndexInput extends BufferedIndexInput { - IndexInput base; - long fileOffset; - long length; - - CSIndexInput(final IndexInput base, final long fileOffset, final long length) { - this(base, fileOffset, length, BufferedIndexInput.BUFFER_SIZE); - } - - CSIndexInput(final IndexInput base, final long fileOffset, final long length, int readBufferSize) { - super(readBufferSize); - this.base = (IndexInput)base.clone(); - this.fileOffset = fileOffset; - this.length = length; - } - - @Override - public Object clone() { - CSIndexInput clone = (CSIndexInput)super.clone(); - clone.base = (IndexInput)base.clone(); - clone.fileOffset = fileOffset; - clone.length = length; - return clone; - } - - /** Expert: implements buffer refill. Reads bytes from the current - * position in the input. - * @param b the array to read bytes into - * @param offset the offset in the array to start storing bytes - * @param len the number of bytes to read - */ - @Override - protected void readInternal(byte[] b, int offset, int len) throws IOException { - long start = getFilePointer(); - if(start + len > length) - throw new IOException("read past EOF"); - base.seek(fileOffset + start); - base.readBytes(b, offset, len, false); - } - - /** Expert: implements seek. Sets current position in this file, where - * the next {@link #readInternal(byte[],int,int)} will occur. - * @see #readInternal(byte[],int,int) - */ - @Override - protected void seekInternal(long pos) {} - - /** Closes the stream to further operations. */ - @Override - public void close() throws IOException { - base.close(); - } - - @Override - public long length() { - return length; - } - - @Override - public void copyBytes(IndexOutput out, long numBytes) throws IOException { - // Copy first whatever is in the buffer - numBytes -= flushBuffer(out, numBytes); - - // If there are more bytes left to copy, delegate the copy task to the - // base IndexInput, in case it can do an optimized copy. - if (numBytes > 0) { - long start = getFilePointer(); - if (start + numBytes > length) { - throw new IOException("read past EOF"); - } - base.seek(fileOffset + start); - base.copyBytes(out, numBytes); - } - } - } -} Index: lucene/src/java/org/apache/lucene/index/CompoundFileWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/CompoundFileWriter.java (revision 1135779) +++ lucene/src/java/org/apache/lucene/index/CompoundFileWriter.java (working copy) @@ -66,7 +66,8 @@ } // Before versioning started. - static final int FORMAT_PRE_VERSION = 0; + /** @lucene.internal */ + public static final int FORMAT_PRE_VERSION = 0; // Segment name is not written in the file names. static final int FORMAT_NO_SEGMENT_PREFIX = -1; @@ -74,7 +75,8 @@ // NOTE: if you introduce a new format, make it 1 lower // than the current one, and always change this if you // switch to a new format! - static final int FORMAT_CURRENT = FORMAT_NO_SEGMENT_PREFIX; + /** @lucene.internal */ + public static final int FORMAT_CURRENT = FORMAT_NO_SEGMENT_PREFIX; private Directory directory; private String fileName; Index: lucene/src/java/org/apache/lucene/index/IndexReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexReader.java (revision 1135779) +++ lucene/src/java/org/apache/lucene/index/IndexReader.java (working copy) @@ -1432,14 +1432,14 @@ } Directory dir = null; - CompoundFileReader cfr = null; + CompoundFileDirectory cfr = null; try { File file = new File(filename); String dirname = file.getAbsoluteFile().getParent(); filename = file.getName(); dir = FSDirectory.open(new File(dirname)); - cfr = new CompoundFileReader(dir, filename); + cfr = dir.openCompoundInput(filename, BufferedIndexInput.BUFFER_SIZE); String [] files = cfr.listAll(); ArrayUtil.mergeSort(files); // sort the array of filename so that the output is more readable Index: lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (revision 1135779) +++ lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (working copy) @@ -25,7 +25,6 @@ import java.util.Map; import java.util.TreeMap; -import org.apache.lucene.index.CompoundFileReader; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; @@ -37,6 +36,7 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.Bits; @@ -177,8 +177,8 @@ // terms reader with index, the segment has switched // to CFS - if (!(dir instanceof CompoundFileReader)) { - dir0 = cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize); + if (!(dir instanceof CompoundFileDirectory)) { + dir0 = cfsReader = dir.openCompoundInput(IndexFileNames.segmentFileName(si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize); } else { dir0 = dir; } Index: lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java (revision 1135779) +++ lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java (working copy) @@ -19,7 +19,6 @@ import java.io.IOException; -import org.apache.lucene.index.CompoundFileReader; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.FieldsReader; import org.apache.lucene.index.IndexFileNames; @@ -68,12 +67,12 @@ Directory dir = directory; if (si.getDocStoreOffset() != -1) { if (si.getDocStoreIsCompoundFile()) { - dir = new CompoundFileReader(dir, IndexFileNames.segmentFileName( + dir = dir.openCompoundInput(IndexFileNames.segmentFileName( si.getDocStoreSegment(), "", IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), 1024); } } else if (si.getUseCompoundFile()) { - dir = new CompoundFileReader(dir, IndexFileNames.segmentFileName( + dir = dir.openCompoundInput(IndexFileNames.segmentFileName( si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), 1024); } Index: lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java (revision 1135779) +++ lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java (working copy) @@ -23,6 +23,7 @@ import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.PerDocValues; +import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; /** Holds core readers that are shared (unchanged) when @@ -52,8 +53,8 @@ FieldsReader fieldsReaderOrig; TermVectorsReader termVectorsReaderOrig; - CompoundFileReader cfsReader; - CompoundFileReader storeCFSReader; + CompoundFileDirectory cfsReader; + CompoundFileDirectory storeCFSReader; @@ -73,7 +74,7 @@ try { Directory dir0 = dir; if (si.getUseCompoundFile()) { - cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize); + cfsReader = dir.openCompoundInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize); dir0 = cfsReader; } cfsDir = dir0; @@ -161,7 +162,7 @@ if (si.getDocStoreOffset() != -1) { if (si.getDocStoreIsCompoundFile()) { assert storeCFSReader == null; - storeCFSReader = new CompoundFileReader(dir, + storeCFSReader = dir.openCompoundInput( IndexFileNames.segmentFileName(si.getDocStoreSegment(), "", IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), readBufferSize); storeDir = storeCFSReader; @@ -175,7 +176,7 @@ // was not used, but then we are asked to open doc // stores after the segment has switched to CFS if (cfsReader == null) { - cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize); + cfsReader = dir.openCompoundInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize); } storeDir = cfsReader; assert storeDir != null; Index: lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java (revision 1135779) +++ lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java (working copy) @@ -24,6 +24,9 @@ import java.nio.channels.FileChannel; import java.util.concurrent.Future; // javadoc +import org.apache.lucene.store.SimpleFSDirectory.SimpleFSIndexInput; +import org.apache.lucene.util.IOUtils; + /** * An {@link FSDirectory} implementation that uses java.nio's FileChannel's * positional read, which allows multiple threads to read from the same file @@ -77,7 +80,47 @@ ensureOpen(); return new NIOFSIndexInput(new File(getDirectory(), name), bufferSize, getReadChunkSize()); } + + @Override + public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException { + return new NIOFSCompoundFileDirectory(this, name, bufferSize); + } + private final class NIOFSCompoundFileDirectory extends CompoundFileDirectory { + private SimpleFSIndexInput.Descriptor fd; + private FileChannel fc; + + public NIOFSCompoundFileDirectory(Directory directory, String fileName, int readBufferSize) throws IOException { + super(directory, fileName, readBufferSize); + IndexInput stream = null; + try { + File f = new File(NIOFSDirectory.this.getDirectory(), fileName); + fd = new SimpleFSIndexInput.Descriptor(f, "r"); + fc = fd.getChannel(); + stream = new NIOFSIndexInput(fd, fc, 0, fd.length, readBufferSize, getReadChunkSize()); + init(CompoundFileDirectory.readEntries(stream)); + stream.close(); + } catch (IOException e) { + // throw our original exception + IOUtils.closeSafely(e, fc, fd, stream); + } + } + + @Override + public IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException { + return new NIOFSIndexInput(fd, fc, offset, length, readBufferSize, getReadChunkSize()); + } + + @Override + public synchronized void close() throws IOException { + try { + IOUtils.closeSafely(false, fc, fd); + } finally { + super.close(); + } + } + } + protected static class NIOFSIndexInput extends SimpleFSDirectory.SimpleFSIndexInput { private ByteBuffer byteBuf; // wraps the buffer for NIO @@ -91,6 +134,12 @@ super(path, bufferSize, chunkSize); channel = file.getChannel(); } + + public NIOFSIndexInput(Descriptor file, FileChannel fc, long off, long length, int bufferSize, int chunkSize) throws IOException { + super(file, off, length, bufferSize, chunkSize); + channel = fc; + isClone = true; + } @Override protected void newBuffer(byte[] newBuffer) { @@ -145,7 +194,11 @@ int readLength = bb.limit() - readOffset; assert readLength == len; - long pos = getFilePointer(); + long pos = getFilePointer() + off; + + if (pos + len > end) { + throw new IOException("read past EOF"); + } try { while (readLength > 0) { @@ -159,9 +212,6 @@ } bb.limit(limit); int i = channel.read(bb, pos); - if (i == -1) { - throw new IOException("read past EOF"); - } pos += i; readOffset += i; readLength -= i; Index: lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java (revision 0) +++ lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java (revision 0) @@ -0,0 +1,239 @@ +package org.apache.lucene.store; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.CompoundFileWriter; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.Lock; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.io.FileNotFoundException; +import java.io.IOException; + +/** + * Class for accessing a compound stream. + * This class implements a directory, but is limited to only read operations. + * Directory methods that would normally modify data throw an exception. + * @lucene.experimental + */ +public abstract class CompoundFileDirectory extends Directory { + /** Offset/Length for a slice inside of a compound file */ + public static final class FileEntry { + long offset; + long length; + } + + // Base info + private final Directory directory; + private final String fileName; + private final int readBufferSize; + private Map entries; + + /** + * Create a new CompoundFileDirectory. + *

+ * NOTE: subclasses must call {@link #init(HashMap)} before the directory can be used. + */ + public CompoundFileDirectory(Directory directory, String fileName, int readBufferSize) throws IOException { + assert !(directory instanceof CompoundFileDirectory) : "compound file inside of compound file: " + fileName; + this.directory = directory; + this.fileName = fileName; + this.readBufferSize = readBufferSize; + this.isOpen = false; + } + + /** Initialize with a map of filename->slices */ + protected final void init(Map entries) { + this.entries = entries; + this.isOpen = true; + } + + /** Helper method that reads CFS entries from an input stream */ + public static final Map readEntries(IndexInput stream) throws IOException { + Map entries = new HashMap(); + + // read the first VInt. If it is negative, it's the version number + // otherwise it's the count (pre-3.1 indexes) + int firstInt = stream.readVInt(); + + final int count; + final boolean stripSegmentName; + if (firstInt < CompoundFileWriter.FORMAT_PRE_VERSION) { + if (firstInt < CompoundFileWriter.FORMAT_CURRENT) { + throw new CorruptIndexException("Incompatible format version: " + + firstInt + " expected " + CompoundFileWriter.FORMAT_CURRENT); + } + // It's a post-3.1 index, read the count. + count = stream.readVInt(); + stripSegmentName = false; + } else { + count = firstInt; + stripSegmentName = true; + } + + // read the directory and init files + long streamLength = stream.length(); + FileEntry entry = null; + for (int i=0; i streamLength) { + throw new CorruptIndexException("Invalid CFS entry offset: " + offset); + } + String id = stream.readString(); + + if (stripSegmentName) { + // Fix the id to not include the segment names. This is relevant for + // pre-3.1 indexes. + id = IndexFileNames.stripSegmentName(id); + } + + if (entry != null) { + // set length of the previous entry + entry.length = offset - entry.offset; + } + + entry = new FileEntry(); + entry.offset = offset; + entries.put(id, entry); + } + + // set the length of the final entry + if (entry != null) { + entry.length = streamLength - entry.offset; + } + + return entries; + } + + public Directory getDirectory() { + return directory; + } + + public String getName() { + return fileName; + } + + @Override + public synchronized void close() throws IOException { + ensureOpen(); + entries = null; + isOpen = false; + } + + @Override + public synchronized IndexInput openInput(String id) throws IOException { + // Default to readBufferSize passed in when we were opened + return openInput(id, readBufferSize); + } + + @Override + public synchronized IndexInput openInput(String id, int readBufferSize) throws IOException { + ensureOpen(); + id = IndexFileNames.stripSegmentName(id); + final FileEntry entry = entries.get(id); + if (entry == null) + throw new IOException("No sub-file with id " + id + " found (files: " + entries.keySet() + ")"); + + return openInputSlice(id, entry.offset, entry.length, readBufferSize); + } + + /** Return an IndexInput that represents a "slice" or portion of the CFS file. */ + public abstract IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException; + + /** Returns an array of strings, one for each file in the directory. */ + @Override + public String[] listAll() { + ensureOpen(); + String[] res = entries.keySet().toArray(new String[entries.size()]); + // Add the segment name + String seg = fileName.substring(0, fileName.indexOf('.')); + for (int i = 0; i < res.length; i++) { + res[i] = seg + res[i]; + } + return res; + } + + /** Returns true iff a file with the given name exists. */ + @Override + public boolean fileExists(String name) { + ensureOpen(); + return entries.containsKey(IndexFileNames.stripSegmentName(name)); + } + + /** Returns the time the compound file was last modified. */ + @Override + public long fileModified(String name) throws IOException { + return directory.fileModified(fileName); + } + + /** Not implemented + * @throws UnsupportedOperationException */ + @Override + public void deleteFile(String name) { + throw new UnsupportedOperationException(); + } + + /** Not implemented + * @throws UnsupportedOperationException */ + public void renameFile(String from, String to) { + throw new UnsupportedOperationException(); + } + + /** Returns the length of a file in the directory. + * @throws IOException if the file does not exist */ + @Override + public long fileLength(String name) throws IOException { + ensureOpen(); + FileEntry e = entries.get(IndexFileNames.stripSegmentName(name)); + if (e == null) + throw new FileNotFoundException(name); + return e.length; + } + + /** Not implemented + * @throws UnsupportedOperationException */ + @Override + public IndexOutput createOutput(String name) { + throw new UnsupportedOperationException(); + } + + @Override + public void sync(Collection names) throws IOException { + } + + /** Not implemented + * @throws UnsupportedOperationException */ + @Override + public Lock makeLock(String name) { + throw new UnsupportedOperationException(); + } + + /** Not implemented + * @throws UnsupportedOperationException */ + @Override + public final CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException { + // NOTE: final to make nested compounding impossible. + throw new UnsupportedOperationException(); + } +} Property changes on: lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java ___________________________________________________________________ Added: svn:eol-style + native Index: lucene/src/java/org/apache/lucene/store/DefaultCompoundFileDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/DefaultCompoundFileDirectory.java (revision 0) +++ lucene/src/java/org/apache/lucene/store/DefaultCompoundFileDirectory.java (revision 0) @@ -0,0 +1,136 @@ +package org.apache.lucene.store; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.util.IOUtils; + +/** + * Default implementation of {@link CompoundFileDirectory}. + *

+ * This implementation returns a BufferedIndexInput that wraps the underlying + * Directory's IndexInput for the compound file (using unbuffered reads). + * @lucene.experimental + */ +public class DefaultCompoundFileDirectory extends CompoundFileDirectory { + protected IndexInput stream; + + public DefaultCompoundFileDirectory(Directory directory, String fileName, int readBufferSize) throws IOException { + super(directory, fileName, readBufferSize); + try { + stream = directory.openInput(fileName, readBufferSize); + init(CompoundFileDirectory.readEntries(stream)); + } catch (IOException e) { + IOUtils.closeSafely(e, stream); + } + } + + @Override + public IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException { + return new CSIndexInput(stream, offset, length, readBufferSize); + } + + @Override + public synchronized void close() throws IOException { + try { + stream.close(); + } finally { + super.close(); + } + } + + /** Implementation of an IndexInput that reads from a portion of the + * compound file. + */ + static final class CSIndexInput extends BufferedIndexInput { + IndexInput base; + long fileOffset; + long length; + + CSIndexInput(final IndexInput base, final long fileOffset, final long length) { + this(base, fileOffset, length, BufferedIndexInput.BUFFER_SIZE); + } + + CSIndexInput(final IndexInput base, final long fileOffset, final long length, int readBufferSize) { + super(readBufferSize); + this.base = (IndexInput)base.clone(); + this.fileOffset = fileOffset; + this.length = length; + } + + @Override + public Object clone() { + CSIndexInput clone = (CSIndexInput)super.clone(); + clone.base = (IndexInput)base.clone(); + clone.fileOffset = fileOffset; + clone.length = length; + return clone; + } + + /** Expert: implements buffer refill. Reads bytes from the current + * position in the input. + * @param b the array to read bytes into + * @param offset the offset in the array to start storing bytes + * @param len the number of bytes to read + */ + @Override + protected void readInternal(byte[] b, int offset, int len) throws IOException { + long start = getFilePointer(); + if(start + len > length) + throw new IOException("read past EOF"); + base.seek(fileOffset + start); + base.readBytes(b, offset, len, false); + } + + /** Expert: implements seek. Sets current position in this file, where + * the next {@link #readInternal(byte[],int,int)} will occur. + * @see #readInternal(byte[],int,int) + */ + @Override + protected void seekInternal(long pos) {} + + /** Closes the stream to further operations. */ + @Override + public void close() throws IOException { + base.close(); + } + + @Override + public long length() { + return length; + } + + @Override + public void copyBytes(IndexOutput out, long numBytes) throws IOException { + // Copy first whatever is in the buffer + numBytes -= flushBuffer(out, numBytes); + + // If there are more bytes left to copy, delegate the copy task to the + // base IndexInput, in case it can do an optimized copy. + if (numBytes > 0) { + long start = getFilePointer(); + if (start + numBytes > length) { + throw new IOException("read past EOF"); + } + base.seek(fileOffset + start); + base.copyBytes(out, numBytes); + } + } + } +} Property changes on: lucene/src/java/org/apache/lucene/store/DefaultCompoundFileDirectory.java ___________________________________________________________________ Added: svn:eol-style + native Index: lucene/src/java/org/apache/lucene/store/MMapDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/MMapDirectory.java (revision 1135779) +++ lucene/src/java/org/apache/lucene/store/MMapDirectory.java (working copy) @@ -32,6 +32,7 @@ import java.lang.reflect.Method; import org.apache.lucene.util.Constants; +import org.apache.lucene.util.IOUtils; /** File-based {@link Directory} implementation that uses * mmap for reading, and {@link @@ -213,12 +214,50 @@ File f = new File(getDirectory(), name); RandomAccessFile raf = new RandomAccessFile(f, "r"); try { - return new MMapIndexInput(raf, chunkSizePower); + return new MMapIndexInput(raf, 0, raf.length(), chunkSizePower); } finally { raf.close(); } } + + @Override + public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException { + return new MMapCompoundFileDirectory(this, name, bufferSize); + } + + private final class MMapCompoundFileDirectory extends CompoundFileDirectory { + private RandomAccessFile raf = null; + public MMapCompoundFileDirectory(Directory directory, String fileName, int readBufferSize) throws IOException { + super(directory, fileName, readBufferSize); + IndexInput stream = null; + try { + File f = new File(MMapDirectory.this.getDirectory(), fileName); + raf = new RandomAccessFile(f, "r"); + stream = new MMapIndexInput(raf, 0, raf.length(), chunkSizePower); + init(CompoundFileDirectory.readEntries(stream)); + stream.close(); + } catch (IOException e) { + // throw our original exception + IOUtils.closeSafely(e, raf, stream); + } + } + + @Override + public IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException { + return new MMapIndexInput(raf, offset, length, chunkSizePower); + } + + @Override + public synchronized void close() throws IOException { + try { + raf.close(); + } finally { + super.close(); + } + } + } + // Because Java's ByteBuffer uses an int to address the // values, it's necessary to access a file > // Integer.MAX_VALUE in size using multiple byte buffers. @@ -235,8 +274,8 @@ private boolean isClone = false; - MMapIndexInput(RandomAccessFile raf, int chunkSizePower) throws IOException { - this.length = raf.length(); + MMapIndexInput(RandomAccessFile raf, long offset, long length, int chunkSizePower) throws IOException { + this.length = length; this.chunkSizePower = chunkSizePower; this.chunkSize = 1L << chunkSizePower; this.chunkSizeMask = chunkSize - 1L; @@ -261,7 +300,7 @@ ? chunkSize : (length - bufferStart) ); - this.buffers[bufNr] = rafc.map(MapMode.READ_ONLY, bufferStart, bufSize); + this.buffers[bufNr] = rafc.map(MapMode.READ_ONLY, offset + bufferStart, bufSize); bufferStart += bufSize; } seek(0L); Index: lucene/src/java/org/apache/lucene/store/Directory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/Directory.java (revision 1135779) +++ lucene/src/java/org/apache/lucene/store/Directory.java (working copy) @@ -117,6 +117,18 @@ public IndexInput openInput(String name, int bufferSize) throws IOException { return openInput(name); } + + /** + * Returns a {@link CompoundFileDirectory} capable of + * reading the Lucene compound file format. + *

+ * The default implementation returns + * {@link DefaultCompoundFileDirectory}. + * @lucene.experimental + */ + public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException { + return new DefaultCompoundFileDirectory(this, name, bufferSize); + } /** Construct a {@link Lock}. * @param name the name of the lock file Index: lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java (revision 1135779) +++ lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java (working copy) @@ -21,6 +21,8 @@ import java.io.IOException; import java.io.RandomAccessFile; +import org.apache.lucene.util.IOUtils; + /** A straightforward implementation of {@link FSDirectory} * using java.io.RandomAccessFile. However, this class has * poor concurrent performance (multiple threads will @@ -55,7 +57,45 @@ ensureOpen(); return new SimpleFSIndexInput(new File(directory, name), bufferSize, getReadChunkSize()); } + + @Override + public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException { + return new SimpleFSCompoundFileDirectory(this, name, bufferSize); + } + private final class SimpleFSCompoundFileDirectory extends CompoundFileDirectory { + private SimpleFSIndexInput.Descriptor fd; + + public SimpleFSCompoundFileDirectory(Directory directory, String fileName, int readBufferSize) throws IOException { + super(directory, fileName, readBufferSize); + IndexInput stream = null; + try { + File f = new File(SimpleFSDirectory.this.getDirectory(), fileName); + fd = new SimpleFSIndexInput.Descriptor(f, "r"); + stream = new SimpleFSIndexInput(fd, 0, fd.length, readBufferSize, getReadChunkSize()); + init(CompoundFileDirectory.readEntries(stream)); + stream.close(); + } catch (IOException e) { + // throw our original exception + IOUtils.closeSafely(e, fd, stream); + } + } + + @Override + public IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException { + return new SimpleFSIndexInput(fd, offset, length, readBufferSize, getReadChunkSize()); + } + + @Override + public synchronized void close() throws IOException { + try { + fd.close(); + } finally { + super.close(); + } + } + } + protected static class SimpleFSIndexInput extends BufferedIndexInput { protected static class Descriptor extends RandomAccessFile { @@ -84,25 +124,42 @@ boolean isClone; // LUCENE-1566 - maximum read length on a 32bit JVM to prevent incorrect OOM protected final int chunkSize; + protected final long off; + protected final long end; public SimpleFSIndexInput(File path, int bufferSize, int chunkSize) throws IOException { super(bufferSize); - file = new Descriptor(path, "r"); + this.file = new Descriptor(path, "r"); this.chunkSize = chunkSize; + this.off = 0L; + this.end = file.length; } + + public SimpleFSIndexInput(Descriptor file, long off, long length, int bufferSize, int chunkSize) throws IOException { + super(bufferSize); + this.file = file; + this.chunkSize = chunkSize; + this.off = off; + this.end = off + length; + this.isClone = true; // well, we are sorta? + } /** IndexInput methods */ @Override protected void readInternal(byte[] b, int offset, int len) throws IOException { synchronized (file) { - long position = getFilePointer(); + long position = off + getFilePointer(); if (position != file.position) { file.seek(position); file.position = position; } int total = 0; + if (position + len > end) { + throw new IOException("read past EOF"); + } + try { do { final int readLength; @@ -113,9 +170,6 @@ readLength = chunkSize; } final int i = file.read(b, offset + total, readLength); - if (i == -1) { - throw new IOException("read past EOF"); - } file.position += i; total += i; } while (total < len); @@ -144,7 +198,7 @@ @Override public long length() { - return file.length; + return end - off; } @Override Index: lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java (revision 1135779) +++ lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java (working copy) @@ -148,4 +148,9 @@ public IndexInput openInput(String name) throws IOException { return getDirectory(name).openInput(name); } + + @Override + public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException { + return getDirectory(name).openCompoundInput(name, bufferSize); + } } Index: lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java (revision 1135779) +++ lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java (working copy) @@ -388,7 +388,7 @@ } } - private void addFileHandle(Closeable c, String name, boolean input) { + void addFileHandle(Closeable c, String name, boolean input) { Integer v = openFiles.get(name); if (v != null) { v = Integer.valueOf(v.intValue()+1); @@ -416,7 +416,13 @@ addFileHandle(ii, name, true); return ii; } - + + @Override + public synchronized CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException { + maybeYield(); + return new MockCompoundFileDirectoryWrapper(name, this, delegate.openCompoundInput(name, bufferSize)); + } + /** Provided for testing purposes. Use sizeInBytes() instead. */ public synchronized final long getRecomputedSizeInBytes() throws IOException { if (!(delegate instanceof RAMDirectory)) @@ -481,7 +487,7 @@ delegate.close(); } - private synchronized void removeOpenFile(Closeable c, String name) { + synchronized void removeOpenFile(Closeable c, String name) { Integer v = openFiles.get(name); // Could be null when crash() was called if (v != null) { Index: lucene/src/test-framework/org/apache/lucene/store/MockCompoundFileDirectoryWrapper.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/store/MockCompoundFileDirectoryWrapper.java (revision 0) +++ lucene/src/test-framework/org/apache/lucene/store/MockCompoundFileDirectoryWrapper.java (revision 0) @@ -0,0 +1,143 @@ +package org.apache.lucene.store; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; + +public class MockCompoundFileDirectoryWrapper extends CompoundFileDirectory { + private final MockDirectoryWrapper parent; + private final CompoundFileDirectory delegate; + private final String name; + + public MockCompoundFileDirectoryWrapper(String name, MockDirectoryWrapper parent, CompoundFileDirectory delegate) throws IOException { + super(parent, name, 1024); + this.name = name; + this.parent = parent; + this.delegate = delegate; + super.init(Collections.emptyMap()); + parent.addFileHandle(this, name, true); + } + + @Override + public Directory getDirectory() { + return delegate.getDirectory(); + } + + @Override + public String getName() { + return delegate.getName(); + } + + @Override + public synchronized void close() throws IOException { + try { + delegate.close(); + parent.removeOpenFile(this, name); + } finally { + super.close(); + } + } + + @Override + public synchronized IndexInput openInput(String id, int readBufferSize) throws IOException { + return delegate.openInput(id, readBufferSize); + } + + @Override + public String[] listAll() { + return delegate.listAll(); + } + + @Override + public boolean fileExists(String name) { + return delegate.fileExists(name); + } + + @Override + public long fileModified(String name) throws IOException { + return delegate.fileModified(name); + } + + @Override + public void deleteFile(String name) { + delegate.deleteFile(name); + } + + @Override + public void renameFile(String from, String to) { + delegate.renameFile(from, to); + } + + @Override + public long fileLength(String name) throws IOException { + return delegate.fileLength(name); + } + + @Override + public IndexOutput createOutput(String name) { + return delegate.createOutput(name); + } + + @Override + public void sync(Collection names) throws IOException { + delegate.sync(names); + } + + @Override + public Lock makeLock(String name) { + return delegate.makeLock(name); + } + + @Override + public void clearLock(String name) throws IOException { + delegate.clearLock(name); + } + + @Override + public void setLockFactory(LockFactory lockFactory) throws IOException { + delegate.setLockFactory(lockFactory); + } + + @Override + public LockFactory getLockFactory() { + return delegate.getLockFactory(); + } + + @Override + public String getLockID() { + return delegate.getLockID(); + } + + @Override + public String toString() { + return "MockCompoundFileDirectoryWrapper(" + super.toString() + ")"; + } + + @Override + public void copy(Directory to, String src, String dest) throws IOException { + delegate.copy(to, src, dest); + } + + @Override + public IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException { + return delegate.openInputSlice(id, offset, length, readBufferSize); + } + +} Property changes on: lucene/src/test-framework/org/apache/lucene/store/MockCompoundFileDirectoryWrapper.java ___________________________________________________________________ Added: svn:eol-style + native