Index: src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java (revision 1700719) +++ src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java (working copy) @@ -79,6 +79,7 @@ import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.DECLARING_NODE_TYPES; import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.ENTRY_COUNT_PROPERTY_NAME; import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_COUNT; +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.ACTIVE_DELETE; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.BLOB_SIZE; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.COMPAT_MODE; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.EVALUATE_PATH_RESTRICTION; @@ -109,6 +110,12 @@ private static final Logger log = LoggerFactory.getLogger(IndexDefinition.class); /** + * Default number of seconds after which to delete actively. Default is 3600 (1 hour). + * To disable active delete, set to -1. + */ + static final int DEFAULT_ACTIVE_DELETE = 60 * 60; + + /** * Blob size to use by default. To avoid issues in OAK-2105 the size should not * be power of 2. */ @@ -151,6 +158,8 @@ private final String funcName; + private final int activeDelete; + private final int blobSize; private final Codec codec; @@ -227,6 +236,7 @@ this.definition = defn; this.indexName = determineIndexName(defn, indexPath); this.blobSize = getOptionalValue(defn, BLOB_SIZE, DEFAULT_BLOB_SIZE); + this.activeDelete = getOptionalValue(defn, ACTIVE_DELETE, DEFAULT_ACTIVE_DELETE); this.testMode = getOptionalValue(defn, LuceneIndexConstants.TEST_MODE, false); this.aggregates = collectAggregates(defn); @@ -1372,4 +1382,12 @@ return 0; } + public boolean getActiveDeleteEnabled() { + return activeDelete != -1; + } + + public NodeBuilder getTrashNodeBuilder() { + return root.builder().child(LuceneIndexConstants.TRASH_CHILD_NAME); + } + } Index: src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java (revision 1700719) +++ src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java (working copy) @@ -26,6 +26,8 @@ String INDEX_DATA_CHILD_NAME = ":data"; + String TRASH_CHILD_NAME = ":trash"; + Version VERSION = Version.LUCENE_47; Analyzer ANALYZER = new OakAnalyzer(VERSION); @@ -81,6 +83,11 @@ String ORDERED_PROP_NAMES = "orderedProps"; /** + * Actively the data store files after this many hours. + */ + String ACTIVE_DELETE = "activeDelete"; + + /** * Size in bytes used for splitting the index files when storing them in NodeStore */ String BLOB_SIZE = "blobSize"; Index: src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java (revision 1700719) +++ src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java (working copy) @@ -20,6 +20,9 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.io.SequenceInputStream; +import java.security.SecureRandom; +import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; import java.util.List; @@ -32,6 +35,7 @@ import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.StringUtils; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.util.PerfLogger; import org.apache.lucene.store.AlreadyClosedException; @@ -65,11 +69,17 @@ static final PerfLogger PERF_LOGGER = new PerfLogger(LoggerFactory.getLogger(OakDirectory.class.getName() + ".perf")); static final String PROP_DIR_LISTING = "dirListing"; static final String PROP_BLOB_SIZE = "blobSize"; + static final String PROP_UNIQUE_KEY = "uniqueKey"; + static final int UNIQUE_KEY_SIZE = 16; + + private final static SecureRandom secureRandom = new SecureRandom(); + protected final NodeBuilder directoryBuilder; private final IndexDefinition definition; private LockFactory lockFactory; private final boolean readOnly; private final Set fileNames = Sets.newConcurrentHashSet(); + private final boolean activeDeleteEnabled; public OakDirectory(NodeBuilder directoryBuilder, IndexDefinition definition, boolean readOnly) { this.lockFactory = NoLockFactory.getNoLockFactory(); @@ -77,6 +87,7 @@ this.definition = definition; this.readOnly = readOnly; this.fileNames.addAll(getListing()); + this.activeDeleteEnabled = definition.getActiveDeleteEnabled(); } @Override @@ -93,7 +104,29 @@ public void deleteFile(String name) throws IOException { checkArgument(!readOnly, "Read only directory"); fileNames.remove(name); - directoryBuilder.getChildNode(name).remove(); + NodeBuilder f = directoryBuilder.getChildNode(name); + if (activeDeleteEnabled) { + PropertyState property = f.getProperty(JCR_DATA); + ArrayList data; + if (property != null && property.getType() == BINARIES) { + data = newArrayList(property.getValue(BINARIES)); + } else { + data = newArrayList(); + } + NodeBuilder t = definition.getTrashNodeBuilder(); + long index; + if (!t.hasProperty("index")) { + index = 1; + } else { + index = t.getProperty("index").getValue(Type.LONG); + } + t.setProperty("index", index); + NodeBuilder b = t.child("run_" + index); + b.setProperty("time", System.currentTimeMillis()); + b.setProperty("name", name); + b.setProperty(JCR_DATA, data, BINARIES); + } + f.remove(); } @Override @@ -114,6 +147,10 @@ NodeBuilder file; if (!directoryBuilder.hasChildNode(name)) { file = directoryBuilder.child(name); + byte[] uniqueKey = new byte[UNIQUE_KEY_SIZE]; + secureRandom.nextBytes(uniqueKey); + String key = StringUtils.convertBytesToHex(uniqueKey); + file.setProperty(PROP_UNIQUE_KEY, key); file.setProperty(PROP_BLOB_SIZE, definition.getBlobSize()); } else { file = directoryBuilder.child(name); @@ -248,6 +285,11 @@ * The data of the currently loaded blob. */ private byte[] blob; + + /** + * The unique key that is used to make the content unique (to allow removing binaries from the blob store without risking to remove binaries that are still needed). + */ + private final byte[] uniqueKey; /** * Whether the currently loaded blob was modified since the blob was @@ -259,6 +301,7 @@ this.name = name; this.file = file; this.blobSize = determineBlobSize(file); + this.uniqueKey = readUniqueKey(file); this.blob = new byte[blobSize]; PropertyState property = file.getProperty(JCR_DATA); @@ -272,6 +315,9 @@ if (!data.isEmpty()) { Blob last = data.get(data.size() - 1); this.length -= blobSize - last.length(); + if (uniqueKey != null) { + this.length -= uniqueKey.length; + } } } @@ -279,6 +325,7 @@ this.name = that.name; this.file = that.file; this.blobSize = that.blobSize; + this.uniqueKey = that.uniqueKey; this.blob = new byte[blobSize]; this.position = that.position; @@ -307,7 +354,12 @@ private void flushBlob() throws IOException { if (blobModified) { int n = (int) Math.min(blobSize, length - index * blobSize); - Blob b = file.createBlob(new ByteArrayInputStream(blob, 0, n)); + InputStream in = new ByteArrayInputStream(blob, 0, n); + if (uniqueKey != null) { + in = new SequenceInputStream(in, + new ByteArrayInputStream(uniqueKey)); + } + Blob b = file.createBlob(in); if (index < data.size()) { data.set(index, b); } else { @@ -391,6 +443,14 @@ return DEFAULT_BLOB_SIZE; } + private static byte[] readUniqueKey(NodeBuilder file) { + if (file.hasProperty(PROP_UNIQUE_KEY)) { + String key = file.getString(PROP_UNIQUE_KEY); + return StringUtils.convertHexToBytes(key); + } + return null; + } + public void flush() throws IOException { flushBlob(); if (dataModified) { Index: src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java (revision 1700719) +++ src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java (working copy) @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -@Version("2.3.0") +@Version("2.4.0") @Export(optional = "provide:=true") package org.apache.jackrabbit.oak.plugins.index.lucene;