files = toSync.files(directory, false);
for(final String fileName: files) {
+ if (!directory.fileExists(fileName)) {
+ System.out.println("can't find:"+fileName);
+ System.out.println("toSync:"+toSync);
+ printDir();
+ }
assert directory.fileExists(fileName): "file " + fileName + " does not exist";
}
Index: src/java/org/apache/lucene/store/PrefixSwitchDirectory.java
===================================================================
--- src/java/org/apache/lucene/store/PrefixSwitchDirectory.java (revision 0)
+++ src/java/org/apache/lucene/store/PrefixSwitchDirectory.java (revision 0)
@@ -0,0 +1,199 @@
+package org.apache.lucene.store;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
+
+/**
+ * Expert: A Directory instance that switches files between two other Directory
+ * instances.
+ *
+ *
+ * Files with the specified extensions or prefixes are placed in the primary
+ * directory; others are placed in the secondary directory. The provided
+ * extension Set must not change once passed to this class, and must allow
+ * multiple threads to call contains at once. The prefix set utilizes copy on
+ * write to allow fast concurrent access.
+ *
+ *
+ *
+ * NOTE: this API is new and experimental and is subject to suddenly
+ * change in the next release.
+ */
+
+public class PrefixSwitchDirectory extends Directory {
+ private final Directory secondaryDir;
+
+ private final Directory primaryDir;
+
+ private final CopyOnWriteArraySet prefixes;
+
+ private boolean doClose;
+
+ public PrefixSwitchDirectory(Directory primaryDir, Directory secondaryDir,
+ boolean doClose) {
+ this.prefixes = new CopyOnWriteArraySet();
+ this.primaryDir = primaryDir;
+ this.secondaryDir = secondaryDir;
+ this.doClose = doClose;
+ this.lockFactory = primaryDir.getLockFactory();
+ }
+
+ public Set getPrefixes() {
+ return prefixes;
+ }
+
+ /** Return the primary directory */
+ public Directory getPrimaryDir() {
+ return primaryDir;
+ }
+
+ /** Return the secondary directory */
+ public Directory getSecondaryDir() {
+ return secondaryDir;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (doClose) {
+ try {
+ secondaryDir.close();
+ } finally {
+ primaryDir.close();
+ }
+ doClose = false;
+ }
+ }
+
+ @Override
+ public String[] listAll() throws IOException {
+ String[] primaryFiles = primaryDir.listAll();
+ String[] secondaryFiles = secondaryDir.listAll();
+ String[] files = new String[primaryFiles.length + secondaryFiles.length];
+ System.arraycopy(primaryFiles, 0, files, 0, primaryFiles.length);
+ System.arraycopy(secondaryFiles, 0, files, primaryFiles.length,
+ secondaryFiles.length);
+ return files;
+ }
+
+ /** Utility method to return a file's extension. */
+ public static String getExtension(String name) {
+ int i = name.lastIndexOf('.');
+ if (i == -1) {
+ return "";
+ }
+ return name.substring(i + 1, name.length());
+ }
+
+ public void addPrefix(String prefix) {
+ assert prefix != null;
+ prefixes.add(prefix);
+ }
+
+ private Directory getDirectory(String name) {
+ // try matching prefixes first
+ if (prefixes != null) {
+ // TODO: extract prefix, and find in set
+ Iterator it = prefixes.iterator();
+ while (it.hasNext()) {
+ String prefix = (String) it.next();
+ if (name.startsWith(prefix)) {
+ return secondaryDir;
+ }
+ }
+ }
+ return primaryDir;
+ }
+
+ @Override
+ public boolean fileExists(String name) throws IOException {
+ return getDirectory(name).fileExists(name);
+ }
+
+ @Override
+ public long fileModified(String name) throws IOException {
+ return getDirectory(name).fileModified(name);
+ }
+
+ @Override
+ public void touchFile(String name) throws IOException {
+ getDirectory(name).touchFile(name);
+ }
+
+ private boolean containsPrefix(String prefix) throws IOException {
+ String[] files = listAll();
+ for (String name : files) {
+ if (name.startsWith(prefix))
+ return true;
+ }
+ return false;
+ }
+
+ public String getDirName(String name) {
+ if (getDirectory(name) == primaryDir) return "primary";
+ else return "ram";
+ }
+
+ /**
+ * If the deleted file is the last of a prefix, remove the prefix from the
+ * set.
+ */
+ @Override
+ public void deleteFile(String name) throws IOException {
+ //System.out.println("deleteFile:"+name+" "+getDirName(name));
+ getDirectory(name).deleteFile(name);
+ int idx = name.indexOf('.');
+ if (idx != -1) {
+ String prefix = name.substring(0, idx);
+ if (!containsPrefix(prefix)) {
+ prefixes.remove(prefix);
+ }
+ }
+ }
+
+ @Override
+ public long fileLength(String name) throws IOException {
+ Directory dir = getDirectory(name);
+ //if (dir == primaryDir) {
+ // System.out.println("primaryDir fileLength:"+name);
+ //} else if (dir == secondaryDir) {
+ // System.out.println("secondaryDir fileLength:"+name);
+ //}
+ return dir.fileLength(name);
+ }
+
+ @Override
+ public IndexOutput createOutput(String name) throws IOException {
+ //System.out.println("createOutput:"+name+" "+getDirName(name));
+ return getDirectory(name).createOutput(name);
+ }
+
+ @Override
+ public void sync(String name) throws IOException {
+ getDirectory(name).sync(name);
+ }
+
+ @Override
+ public IndexInput openInput(String name) throws IOException {
+ return getDirectory(name).openInput(name);
+ }
+}
Index: contrib/benchmark/conf/basicNRT.alg
===================================================================
--- contrib/benchmark/conf/basicNRT.alg (revision 884977)
+++ contrib/benchmark/conf/basicNRT.alg (working copy)
@@ -25,7 +25,7 @@
analyzer=org.apache.lucene.analysis.standard.StandardAnalyzer
directory=FSDirectory
-work.dir = /x/lucene/wiki.5M
+# work.dir = /x/lucene/wiki.5M
doc.stored=true
doc.body.stored=false
@@ -36,13 +36,20 @@
log.step.Search = 10000
compound = false
-content.source=org.apache.lucene.benchmark.byTask.feeds.LineDocSource
+# content.source=org.apache.lucene.benchmark.byTask.feeds.LineDocSource
+
content.source.forever = false
file.query.maker.file = queries.txt
-query.maker=org.apache.lucene.benchmark.byTask.feeds.FileBasedQueryMaker
-docs.file = /x/lucene/enwiki-20090306-lines-1k-fixed.txt
+# query.maker=org.apache.lucene.benchmark.byTask.feeds.FileBasedQueryMaker
+# docs.file = /x/lucene/enwiki-20090306-lines-1k-fixed.txt
+# doc.maker=org.apache.lucene.benchmark.byTask.feeds.EnwikiDocMaker
+docs.file = /home/jason/src/LUCENE-1577/contrib/benchmark/temp/enwiki-20070527-pages-articles.xml
+content.source=org.apache.lucene.benchmark.byTask.feeds.EnwikiContentSource
+
+query.maker=org.apache.lucene.benchmark.byTask.feeds.ReutersQueryMaker
+
# task at this depth or less would print when they start
task.max.depth.log=2
@@ -66,7 +73,7 @@
[ "Searching" { Search > : * ] : 4 &
# Wait 60 sec, then wrap up
- Wait(5.0)
+ Wait(60.0)
}
CloseReader
Index: contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenIndexTask.java
===================================================================
--- contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenIndexTask.java (revision 884977)
+++ contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenIndexTask.java (working copy)
@@ -19,6 +19,7 @@
import org.apache.lucene.benchmark.byTask.PerfRunData;
import org.apache.lucene.benchmark.byTask.utils.Config;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.LogMergePolicy;
@@ -54,17 +55,22 @@
PerfRunData runData = getRunData();
Config config = runData.getConfig();
final IndexCommit ic;
+ IndexWriter writer;
if (commitUserData != null) {
ic = OpenReaderTask.findIndexCommit(runData.getDirectory(), commitUserData);
+ writer = new IndexWriter(runData.getDirectory(),
+ runData.getAnalyzer(),
+ CreateIndexTask.getIndexDeletionPolicy(config),
+ IndexWriter.MaxFieldLength.UNLIMITED,
+ ic);
} else {
- ic = null;
+ writer = new IndexWriter(runData.getDirectory(),
+ runData.getAnalyzer(),
+ true,
+ CreateIndexTask.getIndexDeletionPolicy(config),
+ IndexWriter.MaxFieldLength.UNLIMITED);
}
-
- IndexWriter writer = new IndexWriter(runData.getDirectory(),
- runData.getAnalyzer(),
- CreateIndexTask.getIndexDeletionPolicy(config),
- IndexWriter.MaxFieldLength.UNLIMITED,
- ic);
+ //boolean create = !IndexReader.indexExists(directory);
CreateIndexTask.setIndexWriterConfig(writer, config);
runData.setIndexWriter(writer);
return 1;
Index: contrib/benchmark/build.xml
===================================================================
--- contrib/benchmark/build.xml (revision 884977)
+++ contrib/benchmark/build.xml (working copy)
@@ -120,7 +120,9 @@
- -->
+
Working Directory: ${working.dir}