diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java index 996d4ec..9a6fcbc 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java @@ -31,6 +31,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import com.google.common.primitives.Ints; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneIndexHelper; @@ -38,6 +39,7 @@ import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.BLOB_SIZE; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.EXCLUDE_PROPERTY_NAMES; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.EXPERIMENTAL_STORAGE; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.FULL_TEXT_ENABLED; @@ -63,6 +65,8 @@ public class IndexDefinition { private final Map propDefns; + private final int blobSize; + public IndexDefinition(NodeBuilder defn) { this.definition = defn; PropertyState pst = defn.getProperty(INCLUDE_PROPERTY_TYPES); @@ -84,6 +88,9 @@ public class IndexDefinition { this.includes = getMultiProperty(defn, INCLUDE_PROPERTY_NAMES); this.orderedProps = getMultiProperty(defn, ORDERED_PROP_NAMES); + //TODO Default blobSize should be 1 MB + this.blobSize = getOptionalValue(defn, BLOB_SIZE, OakDirectory.DEFAULT_BLOB_SIZE); + this.fullTextEnabled = getOptionalValue(defn, FULL_TEXT_ENABLED, true); //Storage is disabled for non full text indexes this.storageEnabled = this.fullTextEnabled && getOptionalValue(defn, EXPERIMENTAL_STORAGE, true); @@ -152,6 +159,10 @@ public class IndexDefinition { return propDefns.containsKey(propName); } + public int getBlobSize() { + return blobSize; + } + //~------------------------------------------< Internal > private static boolean getOptionalValue(NodeBuilder definition, String propName, boolean defaultVal){ @@ -159,6 +170,11 @@ public class IndexDefinition { return ps == null ? defaultVal : ps.getValue(Type.BOOLEAN); } + private static int getOptionalValue(NodeBuilder definition, String propName, int defaultVal){ + PropertyState ps = definition.getProperty(propName); + return ps == null ? defaultVal : Ints.checkedCast(ps.getValue(Type.LONG)); + } + private static Set getMultiProperty(NodeBuilder definition, String propName){ PropertyState pse = definition.getProperty(propName); return pse != null ? ImmutableSet.copyOf(pse.getValue(Type.STRINGS)) : Collections.emptySet(); diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java index 91fb5b8..ae0443b 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java @@ -37,15 +37,15 @@ import org.apache.lucene.store.FSDirectory; class IndexNode { - static IndexNode open(String name, NodeState definition) + static IndexNode open(String name, NodeState defnNodeState) throws IOException { Directory directory = null; - - NodeState data = definition.getChildNode(INDEX_DATA_CHILD_NAME); + IndexDefinition definition = new IndexDefinition(new ReadOnlyBuilder(defnNodeState)); + NodeState data = defnNodeState.getChildNode(INDEX_DATA_CHILD_NAME); if (data.exists()) { - directory = new OakDirectory(new ReadOnlyBuilder(data)); - } else if (PERSISTENCE_FILE.equalsIgnoreCase(definition.getString(PERSISTENCE_NAME))) { - String path = definition.getString(PERSISTENCE_PATH); + directory = new OakDirectory(new ReadOnlyBuilder(data), definition); + } else if (PERSISTENCE_FILE.equalsIgnoreCase(defnNodeState.getString(PERSISTENCE_NAME))) { + String path = defnNodeState.getString(PERSISTENCE_PATH); if (path != null && new File(path).exists()) { directory = FSDirectory.open(new File(path)); } @@ -80,10 +80,10 @@ class IndexNode { private boolean closed = false; - IndexNode(String name, NodeState definition, Directory directory) + IndexNode(String name, IndexDefinition definition, Directory directory) throws IOException { this.name = name; - this.definition = new IndexDefinition(new ReadOnlyBuilder(definition)); + this.definition = definition; this.directory = directory; this.reader = DirectoryReader.open(directory); this.searcher = new IndexSearcher(reader); diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java index 40f3341..07a64ed 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java @@ -78,4 +78,9 @@ public interface LuceneIndexConstants { * be performed with same property then it must be part of include list also */ String ORDERED_PROP_NAMES = "orderedProps"; + + /** + * Size in bytes used for splitting the index files when storing them in NodeStore + */ + String BLOB_SIZE = "blobSize"; } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java index 8efae8b..38d1d02 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java @@ -66,7 +66,7 @@ public class LuceneIndexEditorContext { throws IOException { String path = definition.getString(PERSISTENCE_PATH); if (path == null) { - return new OakDirectory(definition.child(INDEX_DATA_CHILD_NAME)); + return new OakDirectory(definition.child(INDEX_DATA_CHILD_NAME), new IndexDefinition(definition)); } else { // try { File file = new File(path); diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java index 495670f..8e57339 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java @@ -24,8 +24,11 @@ import java.io.IOException; import java.io.InputStream; import java.util.Collection; import java.util.List; + +import com.google.common.primitives.Ints; import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; @@ -50,12 +53,15 @@ import static org.apache.jackrabbit.oak.api.Type.BINARIES; */ class OakDirectory extends Directory { + static final String PROP_BLOB_SIZE = "blobSize"; protected final NodeBuilder directoryBuilder; + private final IndexDefinition definition; private LockFactory lockFactory; - public OakDirectory(NodeBuilder directoryBuilder) { + public OakDirectory(NodeBuilder directoryBuilder, IndexDefinition definition) { this.lockFactory = NoLockFactory.getNoLockFactory(); this.directoryBuilder = directoryBuilder; + this.definition = definition; } @Override @@ -88,7 +94,14 @@ class OakDirectory extends Directory { @Override public IndexOutput createOutput(String name, IOContext context) throws IOException { - return new OakIndexOutput(name, directoryBuilder.child(name)); + NodeBuilder file; + if (!directoryBuilder.hasChildNode(name)) { + file = directoryBuilder.child(name); + file.setProperty(PROP_BLOB_SIZE, definition.getBlobSize()); + } else { + file = directoryBuilder.child(name); + } + return new OakIndexOutput(name, file); } @@ -137,7 +150,7 @@ class OakDirectory extends Directory { * Size of the blob entries to which the Lucene files are split. * Set to higher than the 4kB inline limit for the BlobStore, */ - private static final int BLOB_SIZE = 32 * 1024; + static final int DEFAULT_BLOB_SIZE = 32 * 1024; private static class OakIndexFile { @@ -145,6 +158,8 @@ class OakDirectory extends Directory { private final NodeBuilder file; + private final int blobSize; + private long position = 0; private long length; @@ -155,13 +170,15 @@ class OakDirectory extends Directory { private int index = -1; - private final byte[] blob = new byte[BLOB_SIZE]; + private final byte[] blob; private boolean blobModified = false; public OakIndexFile(String name, NodeBuilder file) { this.name = name; this.file = file; + this.blobSize = determineBlobSize(file); + this.blob = new byte[blobSize]; PropertyState property = file.getProperty(JCR_DATA); if (property != null && property.getType() == BINARIES) { @@ -170,16 +187,18 @@ class OakDirectory extends Directory { this.data = newArrayList(); } - this.length = data.size() * BLOB_SIZE; + this.length = data.size() * blobSize; if (!data.isEmpty()) { Blob last = data.get(data.size() - 1); - this.length -= BLOB_SIZE - last.length(); + this.length -= blobSize - last.length(); } } private OakIndexFile(OakIndexFile that) { this.name = that.name; this.file = that.file; + this.blobSize = that.blobSize; + this.blob = new byte[blobSize]; this.position = that.position; this.length = that.length; @@ -193,7 +212,7 @@ class OakDirectory extends Directory { flushBlob(); checkState(!blobModified); - int n = (int) Math.min(BLOB_SIZE, length - i * BLOB_SIZE); + int n = (int) Math.min(blobSize, length - i * blobSize); InputStream stream = data.get(i).getNewStream(); try { ByteStreams.readFully(stream, blob, 0, n); @@ -206,7 +225,7 @@ class OakDirectory extends Directory { private void flushBlob() throws IOException { if (blobModified) { - int n = (int) Math.min(BLOB_SIZE, length - index * BLOB_SIZE); + int n = (int) Math.min(blobSize, length - index * blobSize); Blob b = file.createBlob(new ByteArrayInputStream(blob, 0, n)); if (index < data.size()) { data.set(index, b); @@ -237,12 +256,12 @@ class OakDirectory extends Directory { throw new IOException("Invalid byte range request"); } - int i = (int) (position / BLOB_SIZE); - int o = (int) (position % BLOB_SIZE); + int i = (int) (position / blobSize); + int o = (int) (position % blobSize); while (len > 0) { loadBlob(i); - int l = Math.min(len, BLOB_SIZE - o); + int l = Math.min(len, blobSize - o); System.arraycopy(blob, o, b, offset, l); offset += l; @@ -256,13 +275,13 @@ class OakDirectory extends Directory { public void writeBytes(byte[] b, int offset, int len) throws IOException { - int i = (int) (position / BLOB_SIZE); - int o = (int) (position % BLOB_SIZE); + int i = (int) (position / blobSize); + int o = (int) (position % blobSize); while (len > 0) { - int l = Math.min(len, BLOB_SIZE - o); + int l = Math.min(len, blobSize - o); if (index != i) { - if (o > 0 || (l < BLOB_SIZE && position + l < length)) { + if (o > 0 || (l < blobSize && position + l < length)) { loadBlob(i); } else { flushBlob(); @@ -282,6 +301,13 @@ class OakDirectory extends Directory { } } + private static int determineBlobSize(NodeBuilder file){ + if (file.hasProperty(PROP_BLOB_SIZE)){ + return Ints.checkedCast(file.getProperty(PROP_BLOB_SIZE).getValue(Type.LONG)); + } + return DEFAULT_BLOB_SIZE; + } + public void flush() throws IOException { flushBlob(); if (dataModified) { diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectoryTest.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectoryTest.java new file mode 100644 index 0000000..919b3d2 --- /dev/null +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectoryTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import org.apache.jackrabbit.oak.api.Blob; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.junit.Test; + +import static com.google.common.collect.Lists.newArrayList; +import static org.apache.jackrabbit.JcrConstants.JCR_DATA; +import static org.apache.jackrabbit.oak.api.Type.BINARIES; +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INDEX_DATA_CHILD_NAME; +import static org.apache.jackrabbit.oak.plugins.index.lucene.OakDirectory.PROP_BLOB_SIZE; +import static org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent.INITIAL_CONTENT; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class OakDirectoryTest { + private Random rnd = new Random(); + + private NodeState root = INITIAL_CONTENT; + + private NodeBuilder builder = root.builder(); + + int fileSize = OakDirectory.DEFAULT_BLOB_SIZE + rnd.nextInt(1000); + + @Test + public void writes_DefaultSetup() throws Exception{ + Directory dir = createDir(builder); + assertWrites(dir, OakDirectory.DEFAULT_BLOB_SIZE); + } + + @Test + public void writes_CustomBlobSize() throws Exception{ + builder.setProperty(LuceneIndexConstants.BLOB_SIZE, 300); + Directory dir = createDir(builder); + assertWrites(dir, 300); + } + + @Test + public void testCompatibility() throws Exception{ + Directory dir = createDir(builder); + byte[] data = assertWrites(dir, OakDirectory.DEFAULT_BLOB_SIZE); + + NodeBuilder testNode = builder.child(INDEX_DATA_CHILD_NAME).child("test"); + testNode.removeProperty(PROP_BLOB_SIZE); + + IndexInput i = dir.openInput("test", IOContext.DEFAULT); + assertEquals(fileSize, i.length()); + + byte[] result = new byte[fileSize]; + i.readBytes(result, 0, result.length); + + assertTrue(Arrays.equals(data, result)); + } + + byte[] assertWrites(Directory dir, int blobSize) throws IOException { + byte[] data = randomBytes(fileSize); + IndexOutput o = dir.createOutput("test", IOContext.DEFAULT); + o.writeBytes(data, data.length); + o.close(); + + assertTrue(dir.fileExists("test")); + assertEquals(fileSize, dir.fileLength("test")); + + IndexInput i = dir.openInput("test", IOContext.DEFAULT); + assertEquals(fileSize, i.length()); + + byte[] result = new byte[fileSize]; + i.readBytes(result, 0, result.length); + + assertTrue(Arrays.equals(data, result)); + + NodeBuilder testNode = builder.child(INDEX_DATA_CHILD_NAME).child("test"); + assertEquals(blobSize, testNode.getProperty(PROP_BLOB_SIZE).getValue(Type.LONG).longValue()); + + List blobs = newArrayList(testNode.getProperty(JCR_DATA).getValue(BINARIES)); + assertEquals(blobSize, blobs.get(0).length()); + + return data; + } + + private Directory createDir(NodeBuilder builder){ + return new OakDirectory(builder.child(INDEX_DATA_CHILD_NAME), new IndexDefinition(builder)); + } + + byte[] randomBytes(int size) { + byte[] data = new byte[size]; + rnd.nextBytes(data); + return data; + } +} diff --git oak-run/src/main/groovy/org/apache/jackrabbit/oak/console/commands/LuceneCommand.groovy oak-run/src/main/groovy/org/apache/jackrabbit/oak/console/commands/LuceneCommand.groovy index 1ff4b36..bd419ff 100644 --- oak-run/src/main/groovy/org/apache/jackrabbit/oak/console/commands/LuceneCommand.groovy +++ oak-run/src/main/groovy/org/apache/jackrabbit/oak/console/commands/LuceneCommand.groovy @@ -21,6 +21,7 @@ package org.apache.jackrabbit.oak.console.commands import com.google.common.base.Stopwatch import org.apache.jackrabbit.oak.commons.PathUtils +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition import org.apache.jackrabbit.oak.plugins.index.lucene.OakDirectory import org.apache.jackrabbit.oak.spi.state.NodeState import org.apache.jackrabbit.oak.spi.state.ReadOnlyBuilder @@ -96,7 +97,7 @@ class LuceneCommand extends ComplexCommandSupport { //OakDirectory is package scope but Groovy allows us //to use it. Good or bad but its helpful debug scripts //can access inner classes and prod code cannot. Win win :) - return new OakDirectory(new ReadOnlyBuilder(data)); + return new OakDirectory(new ReadOnlyBuilder(data), new IndexDefinition(definition)); } return null }