Index: CHANGES.txt =================================================================== --- CHANGES.txt (revision 722429) +++ CHANGES.txt (working copy) @@ -38,6 +38,14 @@ These methods can be used to avoid additional calls to doc(). (Michael Busch) +6. LUCENE-1468: Deprecate Directory.list(), which sometimes filters + out files that don't look like index files, in favor of new + Directory.listAll(), which does no filtering. Also, listAll() will + never return null; instead, it throws an IOException (or + subclass). Specifically, FSDirectory.listAll() will throw + the newly added NoSuchDirectoryException if the directory does not + exist. (Marcel Reutegger, Mike McCandless) + Bug fixes 1. LUCENE-1415: MultiPhraseQuery has incorrect hashCode() and equals() Index: src/test/org/apache/lucene/store/TestDirectory.java =================================================================== --- src/test/org/apache/lucene/store/TestDirectory.java (revision 722429) +++ src/test/org/apache/lucene/store/TestDirectory.java (working copy) @@ -21,6 +21,8 @@ import org.apache.lucene.util._TestUtil; import java.io.File; +import java.io.IOException; +import java.util.Arrays; public class TestDirectory extends LuceneTestCase { @@ -128,5 +130,60 @@ _TestUtil.rmDir(path); } } + + // LUCENE-1468 + public void testRAMDirectoryFilter() throws IOException { + checkDirectoryFilter(new RAMDirectory()); + } + + // LUCENE-1468 + public void testFSDirectoryFilter() throws IOException { + checkDirectoryFilter(FSDirectory.getDirectory("test")); + } + + // LUCENE-1468 + private void checkDirectoryFilter(Directory dir) throws IOException { + String name = "file"; + try { + dir.createOutput(name).close(); + assertTrue(dir.fileExists(name)); + assertTrue(Arrays.asList(dir.listAll()).contains(name)); + } finally { + dir.close(); + } + } + + // LUCENE-1468 + public void testCopySubdir() throws Throwable { + File path = new File(System.getProperty("tempDir"), "testsubdir"); + try { + path.mkdirs(); + new File(path, "subdir").mkdirs(); + Directory fsDir = new FSDirectory(path, null); + assertEquals(0, new RAMDirectory(fsDir).listAll().length); + } finally { + _TestUtil.rmDir(path); + } + } + + // LUCENE-1468 + public void testNotDirectory() throws Throwable { + File path = new File(System.getProperty("tempDir"), "testnotdir"); + Directory fsDir = new FSDirectory(path, null); + try { + IndexOutput out = fsDir.createOutput("afile"); + out.close(); + assertTrue(fsDir.fileExists("afile")); + try { + new FSDirectory(new File(path, "afile"), null); + fail("did not hit expected exception"); + } catch (NoSuchDirectoryException nsde) { + // Expected + } + } finally { + fsDir.close(); + _TestUtil.rmDir(path); + } + } } Index: src/test/org/apache/lucene/store/TestBufferedIndexInput.java =================================================================== --- src/test/org/apache/lucene/store/TestBufferedIndexInput.java (revision 722429) +++ src/test/org/apache/lucene/store/TestBufferedIndexInput.java (working copy) @@ -271,6 +271,11 @@ { return dir.list(); } + public String[] listAll() + throws IOException + { + return dir.listAll(); + } public long fileLength(String name) throws IOException { return dir.fileLength(name); Index: src/test/org/apache/lucene/index/TestIndexReader.java =================================================================== --- src/test/org/apache/lucene/index/TestIndexReader.java (revision 722429) +++ src/test/org/apache/lucene/index/TestIndexReader.java (working copy) @@ -48,6 +48,7 @@ import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.store.MockRAMDirectory; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.store.NoSuchDirectoryException; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; @@ -962,11 +963,11 @@ // new IndexFileDeleter, have it delete // unreferenced files, then verify that in fact // no files were deleted: - String[] startFiles = dir.list(); + String[] startFiles = dir.listAll(); SegmentInfos infos = new SegmentInfos(); infos.read(dir); new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null); - String[] endFiles = dir.list(); + String[] endFiles = dir.listAll(); Arrays.sort(startFiles); Arrays.sort(endFiles); @@ -1519,4 +1520,18 @@ _TestUtil.rmDir(indexDir); } } + + // LUCENE-1468 -- make sure on attempting to open an + // IndexReader on a non-existent directory, you get a + // good exception + public void testNoDir() throws Throwable { + String tempDir = System.getProperty("java.io.tmpdir"); + Directory dir = FSDirectory.getDirectory(new File(tempDir, "doesnotexist"), null); + try { + IndexReader.open(dir); + fail("did not hit expected exception"); + } catch (NoSuchDirectoryException nsde) { + // expected + } + } } Index: src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- src/test/org/apache/lucene/index/TestIndexWriter.java (revision 722429) +++ src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -170,7 +170,7 @@ addDocWithIndex(writer, 25*i+j); } writer.close(); - String[] files = dirs[i].list(); + String[] files = dirs[i].listAll(); for(int j=0;j 1 but got " + gen, gen > 1); - String[] files = dir.list(); + String[] files = dir.listAll(); for(int i=0;i lastNumFile); @@ -4254,4 +4254,31 @@ r.close(); dir.close(); } + + // LUCENE-1468 -- make sure opening an IndexWriter with + // create=true does not remove non-index files + + public void testOtherFiles() throws Throwable { + File indexDir = new File(System.getProperty("tempDir"), "otherfiles"); + Directory dir = new FSDirectory(indexDir, null); + try { + // Create my own random file: + + IndexOutput out = dir.createOutput("myrandomfile"); + out.writeByte((byte) 42); + out.close(); + + new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED).close(); + + assertTrue(dir.fileExists("myrandomfile")); + + // Make sure this does not copy myrandomfile: + Directory dir2 = new RAMDirectory(dir); + assertTrue(!dir2.fileExists("myrandomfile")); + + } finally { + dir.close(); + _TestUtil.rmDir(indexDir); + } + } } Index: src/test/org/apache/lucene/index/TestIndexFileDeleter.java =================================================================== --- src/test/org/apache/lucene/index/TestIndexFileDeleter.java (revision 722429) +++ src/test/org/apache/lucene/index/TestIndexFileDeleter.java (working copy) @@ -65,7 +65,7 @@ // Now, artificially create an extra .del file & extra // .s0 file: - String[] files = dir.list(); + String[] files = dir.listAll(); /* for(int j=0;j prefixLength && Character.isDigit(fileName.charAt(prefixLength)) && fileName.startsWith(prefix)) { + if (filter.accept(null, fileName) && fileName.length() > prefixLength && Character.isDigit(fileName.charAt(prefixLength)) && fileName.startsWith(prefix)) { files.add(fileName); } } Index: src/java/org/apache/lucene/index/SegmentInfos.java =================================================================== --- src/java/org/apache/lucene/index/SegmentInfos.java (revision 722429) +++ src/java/org/apache/lucene/index/SegmentInfos.java (working copy) @@ -17,11 +17,13 @@ * limitations under the License. */ +import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.ChecksumIndexOutput; import org.apache.lucene.store.ChecksumIndexInput; +import org.apache.lucene.store.NoSuchDirectoryException; import java.io.File; import java.io.FileNotFoundException; @@ -129,8 +131,11 @@ * @param directory -- directory to search for the latest segments_N file */ public static long getCurrentSegmentGeneration(Directory directory) throws IOException { - String[] files = directory.list(); - return getCurrentSegmentGeneration(files); + try { + return getCurrentSegmentGeneration(directory.listAll()); + } catch (NoSuchDirectoryException nsde) { + return -1; + } } /** @@ -558,9 +563,9 @@ long genA = -1; if (directory != null) - files = directory.list(); + files = directory.listAll(); else - files = fileDirectory.list(); + files = FSDirectory.listAll(fileDirectory); if (files != null) genA = getCurrentSegmentGeneration(files); Index: src/java/org/apache/lucene/index/IndexFileDeleter.java =================================================================== --- src/java/org/apache/lucene/index/IndexFileDeleter.java (revision 722429) +++ src/java/org/apache/lucene/index/IndexFileDeleter.java (working copy) @@ -146,9 +146,7 @@ long currentGen = segmentInfos.getGeneration(); IndexFileNameFilter filter = IndexFileNameFilter.getFilter(); - String[] files = directory.list(); - if (files == null) - throw new IOException("cannot read directory " + directory + ": list() returned null"); + String[] files = directory.listAll(); CommitPoint currentCommitPoint = null; @@ -306,9 +304,7 @@ * that segment. */ public void refresh(String segmentName) throws IOException { - String[] files = directory.list(); - if (files == null) - throw new IOException("cannot read directory " + directory + ": list() returned null"); + String[] files = directory.listAll(); IndexFileNameFilter filter = IndexFileNameFilter.getFilter(); String segmentPrefix1; String segmentPrefix2; Index: src/java/org/apache/lucene/store/NoSuchDirectoryException.java =================================================================== --- src/java/org/apache/lucene/store/NoSuchDirectoryException.java (revision 0) +++ src/java/org/apache/lucene/store/NoSuchDirectoryException.java (revision 0) @@ -0,0 +1,31 @@ +package org.apache.lucene.store; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.FileNotFoundException; + +/** + * This exception is thrown when you try to list a + * non-existent directory. + */ + +public class NoSuchDirectoryException extends FileNotFoundException { + public NoSuchDirectoryException(String message) { + super(message); + } +} Property changes on: src/java/org/apache/lucene/store/NoSuchDirectoryException.java ___________________________________________________________________ Name: svn:eol-style + native Index: src/java/org/apache/lucene/store/Directory.java =================================================================== --- src/java/org/apache/lucene/store/Directory.java (revision 722429) +++ src/java/org/apache/lucene/store/Directory.java (working copy) @@ -19,6 +19,8 @@ import java.io.IOException; +import org.apache.lucene.index.IndexFileNameFilter; + /** A Directory is a flat list of files. Files may be written once, when they * are created. Once a file is created it may only be opened for read, or * deleted. Random access is permitted both when reading and writing. @@ -43,14 +45,29 @@ * this Directory instance). */ protected LockFactory lockFactory; - /** Returns an array of strings, one for each file in the - * directory. This method may return null (for example for - * {@link FSDirectory} if the underlying directory doesn't - * exist in the filesystem or there are permissions - * problems).*/ + /** @deprecated For some Directory implementations ({@link + * FSDirectory}, and its subclasses), this method + * silently filters its results to include only index + * files. Please use {@link #listAll} instead, which + * does no filtering. */ public abstract String[] list() throws IOException; + /** Returns an array of strings, one for each file in the + * directory. Unlike {@link #list} this method does no + * filtering of the contents in a directory, and it will + * never return null (throws IOException instead). + * + * Currently this method simply fallsback to {@link + * #list} for Directory impls outside of Lucene's core & + * contrib, but in 3.0 that method will be removed and + * this method will become abstract. */ + public String[] listAll() + throws IOException + { + return list(); + } + /** Returns true iff a file with the given name exists. */ public abstract boolean fileExists(String name) throws IOException; @@ -173,48 +190,55 @@ * are undefined and you could easily hit a * FileNotFoundException. * + *

NOTE: this method only copies files that look + * like index files (ie, have extensions matching the + * known extensions of index files). + * * @param src source directory * @param dest destination directory * @param closeDirSrc if true, call {@link #close()} method on source directory * @throws IOException */ public static void copy(Directory src, Directory dest, boolean closeDirSrc) throws IOException { - final String[] files = src.list(); + final String[] files = src.listAll(); - if (files == null) - throw new IOException("cannot read directory " + src + ": list() returned null"); + IndexFileNameFilter filter = IndexFileNameFilter.getFilter(); - byte[] buf = new byte[BufferedIndexOutput.BUFFER_SIZE]; - for (int i = 0; i < files.length; i++) { - IndexOutput os = null; - IndexInput is = null; + byte[] buf = new byte[BufferedIndexOutput.BUFFER_SIZE]; + for (int i = 0; i < files.length; i++) { + + if (!filter.accept(null, files[i])) + continue; + + IndexOutput os = null; + IndexInput is = null; + try { + // create file in dest directory + os = dest.createOutput(files[i]); + // read current file + is = src.openInput(files[i]); + // and copy to dest directory + long len = is.length(); + long readCount = 0; + while (readCount < len) { + int toRead = readCount + BufferedIndexOutput.BUFFER_SIZE > len ? (int)(len - readCount) : BufferedIndexOutput.BUFFER_SIZE; + is.readBytes(buf, 0, toRead); + os.writeBytes(buf, toRead); + readCount += toRead; + } + } finally { + // graceful cleanup try { - // create file in dest directory - os = dest.createOutput(files[i]); - // read current file - is = src.openInput(files[i]); - // and copy to dest directory - long len = is.length(); - long readCount = 0; - while (readCount < len) { - int toRead = readCount + BufferedIndexOutput.BUFFER_SIZE > len ? (int)(len - readCount) : BufferedIndexOutput.BUFFER_SIZE; - is.readBytes(buf, 0, toRead); - os.writeBytes(buf, toRead); - readCount += toRead; - } + if (os != null) + os.close(); } finally { - // graceful cleanup - try { - if (os != null) - os.close(); - } finally { - if (is != null) - is.close(); - } + if (is != null) + is.close(); } } - if(closeDirSrc) - src.close(); + } + if(closeDirSrc) + src.close(); } /** Index: src/java/org/apache/lucene/store/RAMDirectory.java =================================================================== --- src/java/org/apache/lucene/store/RAMDirectory.java (revision 722429) +++ src/java/org/apache/lucene/store/RAMDirectory.java (working copy) @@ -95,8 +95,11 @@ this(FSDirectory.getDirectory(dir), true); } - /** Returns an array of strings, one for each file in the directory. */ public synchronized final String[] list() { + return listAll(); + } + + public synchronized final String[] listAll() { ensureOpen(); Set fileNames = fileMap.keySet(); String[] result = new String[fileNames.size()]; Index: src/java/org/apache/lucene/store/FSDirectory.java =================================================================== --- src/java/org/apache/lucene/store/FSDirectory.java (revision 722429) +++ src/java/org/apache/lucene/store/FSDirectory.java (working copy) @@ -18,6 +18,7 @@ */ import java.io.File; +import java.io.FilenameFilter; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; @@ -242,6 +243,7 @@ return dir; } + /** @deprecated */ private void create() throws IOException { if (directory.exists()) { String[] files = directory.list(IndexFileNameFilter.getFilter()); // clear old files @@ -265,9 +267,6 @@ final void createDir() throws IOException { if (!checked) { - if (directory.exists() && !directory.isDirectory()) - throw new IOException(directory + " not a directory"); - if (!directory.exists()) if (!directory.mkdirs()) throw new IOException("Cannot create directory: " + directory); @@ -304,6 +303,9 @@ directory = path; + if (directory.exists() && !directory.isDirectory()) + throw new NoSuchDirectoryException("file '" + directory + "' exists but is not a directory"); + boolean doClearLockID = false; if (lockFactory == null) { @@ -356,12 +358,46 @@ } } - /** Returns an array of strings, one for each Lucene index file in the directory. */ + /** Lists all files (not subdirectories) in the + * directory. This method never returns null (throws + * {@link IOException} instead). + * + * @throws NoSuchDirectoryException if the directory + * does not exist, or does exist but is not a + * directory. + * @throws IOException if list() returns null */ + public static String[] listAll(File dir) throws IOException { + if (!dir.exists()) + throw new NoSuchDirectoryException("directory '" + dir + "' does not exist"); + else if (!dir.isDirectory()) + throw new NoSuchDirectoryException("file '" + dir + "' exists but is not a directory"); + + // Exclude subdirs + String[] result = dir.list(new FilenameFilter() { + public boolean accept(File dir, String file) { + return !new File(dir, file).isDirectory(); + } + }); + + if (result == null) + throw new IOException("directory '" + dir + "' exists and is a directory, but cannot be listed: list() returned null"); + + return result; + } + public String[] list() { ensureOpen(); return directory.list(IndexFileNameFilter.getFilter()); } + /** Lists all files (not subdirectories) in the + * directory. + * @see #listAll(File) */ + public String[] listAll() throws IOException { + ensureOpen(); + return listAll(directory); + } + /** Returns true iff a file with the given name exists. */ public boolean fileExists(String name) { ensureOpen(); Index: contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java =================================================================== --- contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (revision 722429) +++ contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (working copy) @@ -24,7 +24,6 @@ import java.util.List; import java.util.Iterator; -import org.apache.lucene.benchmark.byTask.Benchmark; import org.apache.lucene.benchmark.byTask.feeds.DocData; import org.apache.lucene.benchmark.byTask.feeds.NoMoreDataException; import org.apache.lucene.benchmark.byTask.feeds.ReutersDocMaker; @@ -712,7 +711,7 @@ ir.close(); // Make sure we have 3 segments: - final String[] files = benchmark.getRunData().getDirectory().list(); + final String[] files = benchmark.getRunData().getDirectory().listAll(); int cfsCount = 0; for(int i=0;i