Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java (revision 1774165) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java (working copy) @@ -67,6 +67,7 @@ import static org.apache.jackrabbit.oak.api.Type.BINARIES; import static org.apache.jackrabbit.oak.api.Type.STRINGS; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INDEX_DATA_CHILD_NAME; +import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; import static org.apache.jackrabbit.oak.plugins.memory.PropertyStates.createProperty; import static org.apache.jackrabbit.oak.spi.blob.BlobOptions.UploadType.SYNCHRONOUS; @@ -252,6 +253,38 @@ return "Directory for " + definition.getIndexName(); } + /** + * Copies the file with the given {@code name} to the {@code dest} + * directory. The file is copied 'by reference'. That is, the file in the + * destination directory will reference the same blob values as the source + * file. + *

+ * This method is a no-op if the file does not exist in this directory. + * + * @param dest the destination directory. + * @param name the name of the file to copy. + * @throws IOException if an error occurs while copying the file. + * @throws IllegalArgumentException if the destination directory does not + * use the same {@link BlobFactory} as {@code this} directory. + */ + public void copy(OakDirectory dest, String name) + throws IOException { + if (blobFactory != dest.blobFactory) { + throw new IllegalArgumentException("Source and destination " + + "directory must reference the same BlobFactory"); + } + NodeBuilder file = directoryBuilder.getChildNode(name); + if (file.exists()) { + // overwrite potentially already existing child + NodeBuilder destFile = dest.directoryBuilder.setChildNode(name, EMPTY_NODE); + for (PropertyState p : file.getProperties()) { + destFile.setProperty(p); + } + dest.fileNames.add(name); + dest.markDirty(); + } + } + public boolean isDirty() { return dirty; } Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/BufferedOakDirectory.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/BufferedOakDirectory.java (nonexistent) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/BufferedOakDirectory.java (working copy) @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.lucene.directory; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Set; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import com.google.common.collect.Sets; + +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.lucene.OakDirectory; +import org.apache.jackrabbit.oak.plugins.index.lucene.OakDirectory.BlobFactory; +import org.apache.jackrabbit.oak.spi.blob.BlobStore; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.Lock; +import org.apache.lucene.store.LockFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.google.common.base.Preconditions.checkNotNull; +import static java.util.Arrays.asList; +import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; +import static org.apache.jackrabbit.oak.plugins.memory.ModifiedNodeState.squeeze; + +/** + * A directory implementation that buffers changes until {@link #close()}, + * except for blob values. Those are written immediately to the store. + */ +public final class BufferedOakDirectory extends Directory { + + static final int DELETE_THRESHOLD_UNTIL_REOPEN = 100; + + private static final Logger LOG = LoggerFactory.getLogger(BufferedOakDirectory.class); + + private final BlobFactory blobFactory; + + private final String dataNodeName; + + private final IndexDefinition definition; + + private final OakDirectory base; + + private final Set bufferedForDelete = Sets.newConcurrentHashSet(); + + private NodeBuilder bufferedBuilder = EMPTY_NODE.builder(); + + private OakDirectory buffered; + + private int deleteCount; + + public BufferedOakDirectory(@Nonnull NodeBuilder builder, + @Nonnull String dataNodeName, + @Nonnull IndexDefinition definition, + @Nullable BlobStore blobStore) { + this.blobFactory = blobStore != null ? + new OakDirectory.BlobStoreBlobFactory(blobStore) : + new OakDirectory.NodeBuilderBlobFactory(builder); + this.dataNodeName = checkNotNull(dataNodeName); + this.definition = checkNotNull(definition); + this.base = new OakDirectory(checkNotNull(builder), dataNodeName, + definition, false, blobFactory); + reopenBuffered(); + } + + @Override + public String[] listAll() throws IOException { + LOG.debug("[{}]listAll()", definition.getIndexPath()); + Set all = Sets.newTreeSet(); + all.addAll(asList(base.listAll())); + all.addAll(asList(buffered.listAll())); + all.removeAll(bufferedForDelete); + return all.toArray(new String[all.size()]); + } + + @Override + public boolean fileExists(String name) throws IOException { + LOG.debug("[{}]fileExists({})", definition.getIndexPath(), name); + if (bufferedForDelete.contains(name)) { + return false; + } + return buffered.fileExists(name) || base.fileExists(name); + } + + @Override + public void deleteFile(String name) throws IOException { + LOG.debug("[{}]deleteFile({})", definition.getIndexPath(), name); + if (base.fileExists(name)) { + bufferedForDelete.add(name); + } + if (buffered.fileExists(name)) { + buffered.deleteFile(name); + fileDeleted(); + } + } + + @Override + public long fileLength(String name) throws IOException { + LOG.debug("[{}]fileLength({})", definition.getIndexPath(), name); + if (bufferedForDelete.contains(name)) { + String msg = String.format("already deleted: [%s] %s", + definition.getIndexPath(), name); + throw new FileNotFoundException(msg); + } + Directory dir = base; + if (buffered.fileExists(name)) { + dir = buffered; + } + return dir.fileLength(name); + } + + @Override + public IndexOutput createOutput(String name, IOContext context) + throws IOException { + LOG.debug("[{}]createOutput({})", definition.getIndexPath(), name); + bufferedForDelete.remove(name); + return buffered.createOutput(name, context); + } + + @Override + public void sync(Collection names) throws IOException { + LOG.debug("[{}]sync({})", definition.getIndexPath(), names); + buffered.sync(names); + base.sync(names); + } + + @Override + public IndexInput openInput(String name, IOContext context) + throws IOException { + LOG.debug("[{}]openInput({})", definition.getIndexPath(), name); + if (bufferedForDelete.contains(name)) { + String msg = String.format("already deleted: [%s] %s", + definition.getIndexPath(), name); + throw new FileNotFoundException(msg); + } + Directory dir = base; + if (buffered.fileExists(name)) { + dir = buffered; + } + return dir.openInput(name, context); + } + + @Override + public Lock makeLock(String name) { + return base.makeLock(name); + } + + @Override + public void clearLock(String name) throws IOException { + base.clearLock(name); + } + + @Override + public void close() throws IOException { + LOG.debug("[{}]close()", definition.getIndexPath()); + buffered.close(); + // copy buffered files to base + for (String name : buffered.listAll()) { + buffered.copy(base, name); + } + // remove files marked as deleted + for (String name : bufferedForDelete) { + base.deleteFile(name); + } + base.close(); + } + + @Override + public void setLockFactory(LockFactory lockFactory) throws IOException { + base.setLockFactory(lockFactory); + } + + @Override + public LockFactory getLockFactory() { + return base.getLockFactory(); + } + + private void fileDeleted() throws IOException { + // get rid of non existing files once in a while + if (++deleteCount >= DELETE_THRESHOLD_UNTIL_REOPEN) { + if (LOG.isDebugEnabled()) { + LOG.debug("Reopen buffered OakDirectory. Current list of files: {}", + Arrays.asList(buffered.listAll())); + } + buffered.close(); + reopenBuffered(); + } + } + + private void reopenBuffered() { + // squeeze out child nodes marked as non existing + // those are files that were created and later deleted again + bufferedBuilder = squeeze(bufferedBuilder.getNodeState()).builder(); + buffered = new OakDirectory(bufferedBuilder, dataNodeName, + definition, false, blobFactory); + } +} Property changes on: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/BufferedOakDirectory.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java (revision 1774165) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java (working copy) @@ -153,7 +153,7 @@ private IndexWriter getWriter() throws IOException { if (writer == null) { final long start = PERF_LOGGER.start(); - directory = newIndexDirectory(definition, definitionBuilder, dirName, blobStore); + directory = newIndexDirectory(definition, definitionBuilder, dirName, indexCopier != null, blobStore); IndexWriterConfig config; if (indexCopier != null){ directory = indexCopier.wrapForWrite(definition, directory, reindex, dirName); Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterUtils.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterUtils.java (revision 1774165) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterUtils.java (working copy) @@ -26,6 +26,7 @@ import javax.annotation.Nullable; +import org.apache.jackrabbit.oak.plugins.index.lucene.directory.BufferedOakDirectory; import org.apache.jackrabbit.oak.plugins.index.lucene.FieldNames; import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition; import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants; @@ -74,7 +75,8 @@ } public static Directory newIndexDirectory(IndexDefinition indexDefinition, - NodeBuilder definition, String dirName, @Nullable GarbageCollectableBlobStore blobStore) + NodeBuilder definition, String dirName, boolean buffered, + @Nullable GarbageCollectableBlobStore blobStore) throws IOException { String path = null; if (LuceneIndexConstants.PERSISTENCE_FILE.equalsIgnoreCase( @@ -82,7 +84,11 @@ path = definition.getString(PERSISTENCE_PATH); } if (path == null) { - return new OakDirectory(definition, dirName, indexDefinition, false, blobStore); + if (buffered) { + return new BufferedOakDirectory(definition, dirName, indexDefinition, blobStore); + } else { + return new OakDirectory(definition, dirName, indexDefinition, false, blobStore); + } } else { // try { File file = new File(path); Index: oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java =================================================================== --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java (revision 1774165) +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java (working copy) @@ -371,13 +371,13 @@ } private void purgeDeletedDocs(NodeBuilder idx, IndexDefinition definition) throws IOException { - IndexWriter writer = new IndexWriter(newIndexDirectory(definition, idx, LuceneIndexConstants.INDEX_DATA_CHILD_NAME, null), getIndexWriterConfig(definition, true)); + IndexWriter writer = new IndexWriter(newIndexDirectory(definition, idx, LuceneIndexConstants.INDEX_DATA_CHILD_NAME, false, null), getIndexWriterConfig(definition, true)); writer.forceMergeDeletes(); writer.close(); } public int getDeletedDocCount(NodeBuilder idx, IndexDefinition definition) throws IOException { - IndexReader reader = DirectoryReader.open(newIndexDirectory(definition, idx, LuceneIndexConstants.INDEX_DATA_CHILD_NAME, null)); + IndexReader reader = DirectoryReader.open(newIndexDirectory(definition, idx, LuceneIndexConstants.INDEX_DATA_CHILD_NAME, false, null)); int numDeletes = reader.numDeletedDocs(); reader.close(); return numDeletes; Index: oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/BufferedOakDirectoryTest.java =================================================================== --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/BufferedOakDirectoryTest.java (nonexistent) +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/BufferedOakDirectoryTest.java (working copy) @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.lucene.directory; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Arrays; +import java.util.Random; +import java.util.Set; + +import com.google.common.collect.Sets; + +import org.apache.commons.io.FileUtils; +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.lucene.OakDirectory; +import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.junit.Test; + +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INDEX_DATA_CHILD_NAME; +import static org.apache.jackrabbit.oak.plugins.index.lucene.directory.BufferedOakDirectory.DELETE_THRESHOLD_UNTIL_REOPEN; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class BufferedOakDirectoryTest { + + private Random rnd = new Random(); + + private NodeState root = EmptyNodeState.EMPTY_NODE; + + private NodeBuilder builder = root.builder(); + + @Test + public void createOutput() throws Exception { + Directory buffered = createDir(builder, true); + byte[] data = writeFile(buffered, "file"); + + // must not be visible yet in base + Directory base = createDir(builder, false); + assertFalse(base.fileExists("file")); + base.close(); + + buffered.close(); + + // now it must exist + base = createDir(builder, false); + assertFile(base, "file", data); + base.close(); + } + + @Test + public void listAll() throws Exception { + Directory buffered = createDir(builder, true); + writeFile(buffered, "file"); + + // must only show up after buffered is closed + Directory base = createDir(builder, false); + assertEquals(0, base.listAll().length); + base.close(); + buffered.close(); + base = createDir(builder, false); + assertEquals(Sets.newHashSet("file"), Sets.newHashSet(base.listAll())); + base.close(); + + buffered = createDir(builder, true); + buffered.deleteFile("file"); + assertEquals(0, buffered.listAll().length); + + // must only disappear after buffered is closed + base = createDir(builder, false); + assertEquals(Sets.newHashSet("file"), Sets.newHashSet(base.listAll())); + base.close(); + buffered.close(); + base = createDir(builder, false); + assertEquals(0, base.listAll().length); + base.close(); + } + + @Test + public void fileLength() throws Exception { + Directory base = createDir(builder, false); + writeFile(base, "file"); + base.close(); + + Directory buffered = createDir(builder, true); + buffered.deleteFile("file"); + try { + buffered.fileLength("file"); + fail("must throw FileNotFoundException"); + } catch (FileNotFoundException expected) { + // expected + } + try { + buffered.fileLength("unknown"); + fail("must throw FileNotFoundException"); + } catch (FileNotFoundException expected) { + // expected + } + buffered.close(); + } + + @Test + public void reopen() throws Exception { + Random rand = new Random(42); + Set names = Sets.newHashSet(); + Directory dir = createDir(builder, true); + for (int i = 0; i < 10 * DELETE_THRESHOLD_UNTIL_REOPEN; i++) { + String name = "file-" + i; + writeFile(dir, name); + if (rand.nextInt(20) != 0) { + dir.deleteFile(name); + } else { + // keep 5% + names.add(name); + } + } + assertEquals(names, Sets.newHashSet(dir.listAll())); + dir.close(); + + // open unbuffered and check list as well + dir = createDir(builder, false); + assertEquals(names, Sets.newHashSet(dir.listAll())); + dir.close(); + } + + private void assertFile(Directory dir, String file, byte[] expected) + throws IOException { + assertTrue(dir.fileExists(file)); + assertEquals(expected.length, dir.fileLength(file)); + IndexInput in = dir.openInput(file, IOContext.DEFAULT); + byte[] data = new byte[expected.length]; + in.readBytes(data, 0, data.length); + in.close(); + assertTrue(Arrays.equals(expected, data)); + } + + private Directory createDir(NodeBuilder builder, boolean buffered) { + IndexDefinition def = new IndexDefinition(root, builder.getNodeState(), "/foo"); + if (buffered) { + return new BufferedOakDirectory(builder, INDEX_DATA_CHILD_NAME, def, null); + } else { + return new OakDirectory(builder, def,false); + } + } + + private byte[] randomBytes(int size) { + byte[] data = new byte[size]; + rnd.nextBytes(data); + return data; + } + + private byte[] writeFile(Directory dir, String name) throws IOException { + byte[] data = randomBytes(rnd.nextInt((int) (16 * FileUtils.ONE_KB))); + IndexOutput out = dir.createOutput(name, IOContext.DEFAULT); + out.writeBytes(data, data.length); + out.close(); + return data; + } +} Property changes on: oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/BufferedOakDirectoryTest.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/CopyOnWriteDirectoryTest.java =================================================================== --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/CopyOnWriteDirectoryTest.java (revision 1774165) +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/CopyOnWriteDirectoryTest.java (working copy) @@ -40,7 +40,6 @@ import org.apache.lucene.store.IndexOutput; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -87,13 +86,12 @@ } // OAK-5238 - @Ignore @Test public void copyOnWrite() throws Exception { IndexDefinition def = new IndexDefinition(ns.getRoot(), ns.getRoot(), "/foo"); NodeBuilder builder = ns.getRoot().builder(); Directory remote = IndexWriterUtils.newIndexDirectory( - def, builder.child("foo"), INDEX_DATA_CHILD_NAME, null); + def, builder.child("foo"), INDEX_DATA_CHILD_NAME, true, null); Directory dir = copier.wrapForWrite(def, remote, false, INDEX_DATA_CHILD_NAME); addFiles(dir); writeTree(builder);