diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java index 94a658e..e809c95 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java @@ -197,6 +197,8 @@ class IndexDefinition implements Aggregate.AggregateMapper{ private final int suggesterUpdateFrequencyMinutes; + private final boolean saveDirListing; + public IndexDefinition(NodeState root, NodeState defn) { this(root, defn, null); } @@ -257,6 +259,7 @@ class IndexDefinition implements Aggregate.AggregateMapper{ this.maxExtractLength = determineMaxExtractLength(); this.suggesterUpdateFrequencyMinutes = getOptionalValue(defn, LuceneIndexConstants.SUGGEST_UPDATE_FREQUENCY_MINUTES, 60); this.scorerProviderName = getOptionalValue(defn, LuceneIndexConstants.PROP_SCORER_PROVIDER, null); + this.saveDirListing = getOptionalValue(defn, LuceneIndexConstants.SAVE_DIR_LISTING, false); } public boolean isFullTextEnabled() { @@ -361,6 +364,10 @@ class IndexDefinition implements Aggregate.AggregateMapper{ return scorerProviderName; } + public boolean isSaveDirListing() { + return saveDirListing; + } + @Override public String toString() { return "Lucene Index : " + indexName; diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java index ee41d5a..89bc068 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java @@ -46,7 +46,7 @@ class IndexNode { IndexDefinition definition = new IndexDefinition(root, defnNodeState, indexPath); NodeState data = defnNodeState.getChildNode(INDEX_DATA_CHILD_NAME); if (data.exists()) { - directory = new OakDirectory(new ReadOnlyBuilder(data), definition); + directory = new OakDirectory(new ReadOnlyBuilder(data), definition, true); if (cloner != null){ directory = cloner.wrap(indexPath, definition, directory); } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java index 0da3f0e..10a75d2 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java @@ -255,4 +255,11 @@ public interface LuceneIndexConstants { * 'IS NOT NULL' constraint */ String PROP_NOT_NULL_CHECK_ENABLED = "notNullCheckEnabled"; + + /** + * Boolean property indicating that Lucene directory content + * should be saved as part of NodeState itself as a multi value property + * to allow faster reads (OAK-2809) + */ + String SAVE_DIR_LISTING = "saveDirectoryListing"; } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java index d3304fc..4503e54 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java @@ -80,7 +80,7 @@ public class LuceneIndexEditorContext { throws IOException { String path = definition.getString(PERSISTENCE_PATH); if (path == null) { - return new OakDirectory(definition.child(INDEX_DATA_CHILD_NAME), indexDefinition); + return new OakDirectory(definition.child(INDEX_DATA_CHILD_NAME), indexDefinition, false); } else { // try { File file = new File(path); @@ -118,6 +118,8 @@ public class LuceneIndexEditorContext { private Parser parser; + private Directory directory; + /** * The media types supported by the parser used. */ @@ -144,7 +146,8 @@ public class LuceneIndexEditorContext { IndexWriter getWriter() throws IOException { if (writer == null) { final long start = PERF_LOGGER.start(); - writer = new IndexWriter(newIndexDirectory(definition, definitionBuilder), config); + directory = newIndexDirectory(definition, definitionBuilder); + writer = new IndexWriter(directory, config); PERF_LOGGER.end(start, -1, "Created IndexWriter for directory {}", definition); } return writer; @@ -168,6 +171,8 @@ public class LuceneIndexEditorContext { writer.close(); + directory.close(); + //OAK-2029 Record the last updated status so //as to make IndexTracker detect changes when index //is stored in file system diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java index 37c71b6..a7eac9c 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java @@ -16,8 +16,6 @@ */ package org.apache.jackrabbit.oak.plugins.index.lucene; -import com.google.common.collect.Iterables; -import com.google.common.io.ByteStreams; import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; import java.io.IOException; @@ -25,11 +23,14 @@ import java.io.InputStream; import java.util.Collection; import java.util.List; +import com.google.common.collect.Iterables; +import com.google.common.io.ByteStreams; import com.google.common.primitives.Ints; import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.util.PerfLogger; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -37,7 +38,9 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockFactory; import org.apache.lucene.store.NoLockFactory; +import org.slf4j.LoggerFactory; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkElementIndex; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkPositionIndexes; @@ -46,28 +49,46 @@ import static com.google.common.collect.Lists.newArrayList; import static org.apache.jackrabbit.JcrConstants.JCR_DATA; import static org.apache.jackrabbit.JcrConstants.JCR_LASTMODIFIED; import static org.apache.jackrabbit.oak.api.Type.BINARIES; +import static org.apache.jackrabbit.oak.api.Type.STRINGS; +import static org.apache.jackrabbit.oak.plugins.memory.PropertyStates.createProperty; /** * Implementation of the Lucene {@link Directory} (a flat list of files) * based on an Oak {@link NodeBuilder}. */ class OakDirectory extends Directory { - + static final PerfLogger PERF_LOGGER = new PerfLogger(LoggerFactory.getLogger(OakDirectory.class.getName() + ".perf")); + static final String PROP_DIR_LISTING = "dirListing"; static final String PROP_BLOB_SIZE = "blobSize"; protected final NodeBuilder directoryBuilder; private final IndexDefinition definition; + private final boolean readOnly; private LockFactory lockFactory; - public OakDirectory(NodeBuilder directoryBuilder, IndexDefinition definition) { + public OakDirectory(NodeBuilder directoryBuilder, IndexDefinition definition, boolean readOnly) { this.lockFactory = NoLockFactory.getNoLockFactory(); this.directoryBuilder = directoryBuilder; this.definition = definition; + this.readOnly = readOnly; } @Override public String[] listAll() throws IOException { - return Iterables.toArray( - directoryBuilder.getChildNodeNames(), String.class); + long start = PERF_LOGGER.start(); + Iterable fileNames = null; + if (readOnly && definition.isSaveDirListing()) { + PropertyState listing = directoryBuilder.getProperty(PROP_DIR_LISTING); + if (listing != null) { + fileNames = listing.getValue(Type.STRINGS); + } + } + + if (fileNames == null){ + fileNames = directoryBuilder.getChildNodeNames(); + } + String[] result = Iterables.toArray(fileNames, String.class); + PERF_LOGGER.end(start, 100, "Directory listing performed. Total {} files", result.length); + return result; } @Override @@ -77,6 +98,7 @@ class OakDirectory extends Directory { @Override public void deleteFile(String name) throws IOException { + checkArgument(!readOnly, "Read only directory"); directoryBuilder.getChildNode(name).remove(); } @@ -94,6 +116,7 @@ class OakDirectory extends Directory { @Override public IndexOutput createOutput(String name, IOContext context) throws IOException { + checkArgument(!readOnly, "Read only directory"); NodeBuilder file; if (!directoryBuilder.hasChildNode(name)) { file = directoryBuilder.child(name); @@ -133,7 +156,9 @@ class OakDirectory extends Directory { @Override public void close() throws IOException { - // do nothing + if (!readOnly && definition.isSaveDirListing()) { + directoryBuilder.setProperty(createProperty(PROP_DIR_LISTING, directoryBuilder.getChildNodeNames(), STRINGS)); + } } @Override diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorTest.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorTest.java index d95d317..2eb2d2b 100644 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorTest.java +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorTest.java @@ -60,6 +60,7 @@ import static org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent.IN import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class LuceneIndexEditorTest { @@ -159,6 +160,24 @@ public class LuceneIndexEditorTest { assertEquals(2, getSearcher().getIndexReader().numDocs()); } + @Test + public void saveDirectoryListing() throws Exception{ + NodeBuilder index = builder.child(INDEX_DEFINITIONS_NAME); + NodeBuilder nb = newLuceneIndexDefinitionV2(index, "lucene", + of(TYPENAME_STRING)); + nb.setProperty(LuceneIndexConstants.SAVE_DIR_LISTING, true); + nb.setProperty(LuceneIndexConstants.FULL_TEXT_ENABLED, false); + nb.setProperty(createProperty(INCLUDE_PROPERTY_NAMES, of("foo"), STRINGS)); + + NodeState before = builder.getNodeState(); + builder.child("test").setProperty("foo", "fox is jumping"); + NodeState after = builder.getNodeState(); + + NodeState indexed = HOOK.processCommit(before, after, CommitInfo.EMPTY); + NodeState dir = indexed.getChildNode("oak:index").getChildNode("lucene").getChildNode(":data"); + assertTrue(dir.hasProperty(OakDirectory.PROP_DIR_LISTING)); + } + /** * 1. Index property foo in /test * 2. Then modify some other property in /test diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectoryTest.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectoryTest.java index 164fe3a..86a6105 100644 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectoryTest.java +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectoryTest.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Random; +import java.util.Set; import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.Type; @@ -38,6 +39,7 @@ import org.apache.lucene.store.IndexOutput; import org.junit.Test; import static com.google.common.collect.Lists.newArrayList; +import static com.google.common.collect.Sets.newHashSet; import static org.apache.jackrabbit.JcrConstants.JCR_DATA; import static org.apache.jackrabbit.oak.api.Type.BINARIES; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INDEX_DATA_CHILD_NAME; @@ -57,21 +59,21 @@ public class OakDirectoryTest { @Test public void writes_DefaultSetup() throws Exception{ - Directory dir = createDir(builder); + Directory dir = createDir(builder, false); assertWrites(dir, IndexDefinition.DEFAULT_BLOB_SIZE); } @Test public void writes_CustomBlobSize() throws Exception{ builder.setProperty(LuceneIndexConstants.BLOB_SIZE, 300); - Directory dir = createDir(builder); + Directory dir = createDir(builder, false); assertWrites(dir, 300); } @Test public void testCompatibility() throws Exception{ builder.setProperty(LuceneIndexConstants.BLOB_SIZE, OakDirectory.DEFAULT_BLOB_SIZE); - Directory dir = createDir(builder); + Directory dir = createDir(builder, false); byte[] data = assertWrites(dir, OakDirectory.DEFAULT_BLOB_SIZE); NodeBuilder testNode = builder.child(INDEX_DATA_CHILD_NAME).child("test"); @@ -90,7 +92,7 @@ public class OakDirectoryTest { @Test //OAK-2388 public void testOverflow() throws Exception{ - Directory dir = createDir(builder); + Directory dir = createDir(builder, false); NodeBuilder file = builder.child(INDEX_DATA_CHILD_NAME).child("test.txt"); int blobSize = 32768; int dataSize = 90844; @@ -103,9 +105,26 @@ public class OakDirectoryTest { file.setProperty(PropertyStates.createProperty("jcr:data", blobs, Type.BINARIES)); IndexInput input = dir.openInput("test.txt", IOContext.DEFAULT); - assertEquals((long)blobSize * (dataSize - 1), input.length()); + assertEquals((long) blobSize * (dataSize - 1), input.length()); } + @Test + public void saveListing() throws Exception{ + builder.setProperty(LuceneIndexConstants.SAVE_DIR_LISTING, true); + Directory dir = createDir(builder, false); + Set fileNames = newHashSet(); + for (int i = 0; i < 10; i++) { + String fileName = "foo" + i; + createFile(dir, fileName); + fileNames.add(fileName); + } + dir.close(); + + dir = createDir(builder, true); + assertEquals(fileNames, newHashSet(dir.listAll())); + } + + byte[] assertWrites(Directory dir, int blobSize) throws IOException { byte[] data = randomBytes(fileSize); IndexOutput o = dir.createOutput("test", IOContext.DEFAULT); @@ -132,8 +151,18 @@ public class OakDirectoryTest { return data; } - private Directory createDir(NodeBuilder builder){ - return new OakDirectory(builder.child(INDEX_DATA_CHILD_NAME), new IndexDefinition(root, builder.getNodeState())); + private int createFile(Directory dir, String fileName) throws IOException { + int size = rnd.nextInt(1000) + 1; + byte[] data = randomBytes(size); + IndexOutput o = dir.createOutput(fileName, IOContext.DEFAULT); + o.writeBytes(data, data.length); + o.close(); + return size; + } + + private Directory createDir(NodeBuilder builder, boolean readOnly){ + return new OakDirectory(builder.child(INDEX_DATA_CHILD_NAME), + new IndexDefinition(root, builder.getNodeState()), readOnly); } byte[] randomBytes(int size) {