Index: oak-auth-external/pom.xml
===================================================================
--- oak-auth-external/pom.xml (revision 1633286)
+++ oak-auth-external/pom.xml (working copy)
@@ -161,7 +161,7 @@
com.h2database
h2
- 1.3.175
+ 1.4.182
test
Index: oak-core/pom.xml
===================================================================
--- oak-core/pom.xml (revision 1633090)
+++ oak-core/pom.xml (working copy)
@@ -289,7 +289,7 @@
com.h2database
h2
- 1.3.175
+ 1.4.182
true
Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentMK.java
===================================================================
--- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentMK.java (revision 1633090)
+++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentMK.java (working copy)
@@ -33,6 +33,7 @@
import org.apache.jackrabbit.mk.api.MicroKernel;
import org.apache.jackrabbit.mk.api.MicroKernelException;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
import org.apache.jackrabbit.oak.commons.json.JsopReader;
import org.apache.jackrabbit.oak.commons.json.JsopStream;
@@ -46,14 +47,27 @@
import org.apache.jackrabbit.oak.plugins.document.mongo.MongoBlobStore;
import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore;
import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore;
+import org.apache.jackrabbit.oak.plugins.document.persistentCache.CacheType;
+import org.apache.jackrabbit.oak.plugins.document.persistentCache.PersistentCache;
import org.apache.jackrabbit.oak.plugins.document.rdb.RDBBlobStore;
import org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore;
+import org.apache.jackrabbit.oak.plugins.document.util.StringValue;
import org.apache.jackrabbit.oak.stats.Clock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A MicroKernel implementation that stores the data in a {@link DocumentStore}.
*/
public class DocumentMK implements MicroKernel {
+
+ static final Logger LOG = LoggerFactory.getLogger(DocumentMK.class);
+
+ /**
+ * The path where the persistent cache is stored.
+ */
+ static final String PERSISTENT_CACHE =
+ System.getProperty("oak.documentMK.persCache");
/**
* The threshold where special handling for many child node starts.
@@ -468,6 +482,7 @@
private boolean disableBranches;
private Clock clock = Clock.SIMPLE;
private Executor executor;
+ private PersistentCache persistentCache;
public Builder() {
memoryCacheSize(DEFAULT_MEMORY_CACHE_SIZE);
@@ -488,7 +503,12 @@
}
if (this.blobStore == null) {
- this.blobStore = new MongoBlobStore(db, blobCacheSizeMB * 1024 * 1024L);
+ GarbageCollectableBlobStore s = new MongoBlobStore(db, blobCacheSizeMB * 1024 * 1024L);
+ PersistentCache p = getPersistentCache();
+ if (p != null) {
+ s = p.wrapBlobStore(s);
+ }
+ this.blobStore = s;
}
if (this.diffCache == null) {
@@ -759,9 +779,62 @@
public DocumentMK open() {
return new DocumentMK(this);
}
+
+ public Cache buildNodeCache(DocumentNodeStore store) {
+ return buildCache(CacheType.NODE, getNodeCacheSize(), store, null);
+ }
+
+ public Cache buildChildrenCache() {
+ return buildCache(CacheType.CHILDREN, getChildrenCacheSize(), null, null);
+ }
+
+ public Cache buildDocChildrenCache() {
+ return buildCache(CacheType.DOC_CHILDREN, getDocChildrenCacheSize(), null, null);
+ }
+
+ public Cache buildDiffCache() {
+ return buildCache(CacheType.DIFF, getDiffCacheSize(), null, null);
+ }
- public Cache buildCache(long maxWeight) {
- if (LIRS_CACHE) {
+ public Cache buildDocumentCache(DocumentStore docStore) {
+ return buildCache(CacheType.DOCUMENT, getDocumentCacheSize(), null, docStore);
+ }
+
+ private Cache buildCache(
+ CacheType cacheType,
+ long maxWeight,
+ DocumentNodeStore docNodeStore,
+ DocumentStore docStore
+ ) {
+ Cache cache = buildCache(maxWeight);
+ PersistentCache p = getPersistentCache();
+ if (p != null) {
+ if (docNodeStore != null) {
+ docNodeStore.setPersistentCache(p);
+ }
+ cache = p.wrap(docNodeStore, docStore, cache, cacheType);
+ }
+ return cache;
+ }
+
+ private PersistentCache getPersistentCache() {
+ if (PERSISTENT_CACHE == null) {
+ return null;
+ }
+ if (persistentCache == null) {
+ try {
+ persistentCache = new PersistentCache(PERSISTENT_CACHE);
+ } catch (Throwable e) {
+ LOG.warn("Persistent cache not available; please disable the configuration", e);
+ throw new IllegalArgumentException(e);
+ }
+ }
+ return persistentCache;
+ }
+
+ private Cache buildCache(
+ long maxWeight) {
+ if (LIRS_CACHE || PERSISTENT_CACHE != null) {
return CacheLIRS.newBuilder().
weigher(weigher).
averageWeight(2000).
@@ -776,6 +849,7 @@
recordStats().
build();
}
+
}
-
+
}
Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeState.java
===================================================================
--- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeState.java (revision 1633090)
+++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeState.java (working copy)
@@ -18,8 +18,10 @@
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.NoSuchElementException;
import java.util.Set;
@@ -59,7 +61,7 @@
/**
* A {@link NodeState} implementation for the {@link DocumentNodeStore}.
*/
-class DocumentNodeState extends AbstractNodeState implements CacheValue {
+public class DocumentNodeState extends AbstractNodeState implements CacheValue {
public static final Children NO_CHILDREN = new Children();
@@ -106,11 +108,15 @@
return true;
} else if (that instanceof DocumentNodeState) {
DocumentNodeState other = (DocumentNodeState) that;
- if (getPath().equals(other.getPath())) {
- if (revisionEquals(other)) {
- return true;
- }
+ if (!getPath().equals(other.getPath())) {
+ // path does not match: not equals
+ // (even if the properties are equal)
+ return false;
}
+ if (revisionEquals(other)) {
+ return true;
+ }
+ // revision does not match: might still be equals
} else if (that instanceof ModifiedNodeState) {
ModifiedNodeState modified = (ModifiedNodeState) that;
if (modified.getBaseState() == this) {
@@ -119,9 +125,8 @@
}
if (that instanceof NodeState) {
return AbstractNodeState.equals(this, (NodeState) that);
- } else {
- return false;
}
+ return false;
}
@Override
@@ -480,6 +485,71 @@
});
}
+ public String asString() {
+ JsopWriter json = new JsopBuilder();
+ json.key("path").value(path);
+ json.key("rev").value(rev.toString());
+ if (lastRevision != null) {
+ json.key("lastRev").value(lastRevision.toString());
+ }
+ if (hasChildren) {
+ json.key("hasChildren").value(hasChildren);
+ }
+ if (properties.size() > 0) {
+ json.key("prop").object();
+ for (String k : properties.keySet()) {
+ json.key(k).value(getPropertyAsString(k));
+ }
+ json.endObject();
+ }
+ return json.toString();
+ }
+
+ public static DocumentNodeState fromString(DocumentNodeStore store, String s) {
+ JsopTokenizer json = new JsopTokenizer(s);
+ String path = null;
+ Revision rev = null;
+ Revision lastRev = null;
+ boolean hasChildren = false;
+ DocumentNodeState state = null;
+ HashMap map = new HashMap();
+ while (true) {
+ String k = json.readString();
+ json.read(':');
+ if ("path".equals(k)) {
+ path = json.readString();
+ } else if ("rev".equals(k)) {
+ rev = Revision.fromString(json.readString());
+ } else if ("lastRev".equals(k)) {
+ lastRev = Revision.fromString(json.readString());
+ } else if ("hasChildren".equals(k)) {
+ hasChildren = json.read() == JsopReader.TRUE;
+ } else if ("prop".equals(k)) {
+ json.read('{');
+ while (true) {
+ if (json.matches('}')) {
+ break;
+ }
+ k = json.readString();
+ json.read(':');
+ String v = json.readString();
+ map.put(k, v);
+ json.matches(',');
+ }
+ }
+ if (json.matches(JsopReader.END)) {
+ break;
+ }
+ json.read(',');
+ }
+ state = new DocumentNodeState(store, path, rev, hasChildren);
+ state.setLastRevision(lastRev);
+ for (Entry e : map.entrySet()) {
+ state.setProperty(e.getKey(), e.getValue());
+ }
+ return state;
+ }
+
/**
* A list of children for a node.
*/
@@ -489,21 +559,71 @@
* Ascending sorted list of names of child nodes.
*/
final ArrayList children = new ArrayList();
+ int cachedMemory;
boolean hasMore;
@Override
public int getMemory() {
- int size = 114;
- for (String c : children) {
- size += c.length() * 2 + 56;
+ if (cachedMemory == 0) {
+ int size = 114;
+ for (String c : children) {
+ size += c.length() * 2 + 56;
+ }
+ cachedMemory = size;
}
- return size;
+ return cachedMemory;
}
@Override
public String toString() {
return children.toString();
}
+
+ public String asString() {
+ JsopWriter json = new JsopBuilder();
+ if (hasMore) {
+ json.key("hasMore").value(true);
+ }
+ if (children.size() > 0) {
+ json.key("children").array();
+ for (String c : children) {
+ json.value(c);
+ }
+ json.endArray();
+ }
+ return json.toString();
+ }
+
+ public static Children fromString(String s) {
+ JsopTokenizer json = new JsopTokenizer(s);
+ Children children = new Children();
+ while (true) {
+ if (json.matches(JsopReader.END)) {
+ break;
+ }
+ String k = json.readString();
+ json.read(':');
+ if ("hasMore".equals(k)) {
+ children.hasMore = json.read() == JsopReader.TRUE;
+ } else if ("children".equals(k)) {
+ json.read('[');
+ while (true) {
+ if (json.matches(']')) {
+ break;
+ }
+ String value = json.readString();
+ children.children.add(value);
+ json.matches(',');
+ }
+ }
+ if (json.matches(JsopReader.END)) {
+ break;
+ }
+ json.read(',');
+ }
+ return children;
+ }
+
}
private class ChildNodeEntryIterator implements Iterator {
Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
===================================================================
--- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java (revision 1633090)
+++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java (working copy)
@@ -64,13 +64,13 @@
import org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector;
import org.apache.jackrabbit.oak.plugins.document.mongo.MongoBlobReferenceIterator;
import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore;
+import org.apache.jackrabbit.oak.plugins.document.persistentCache.PersistentCache;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.commons.json.JsopStream;
import org.apache.jackrabbit.oak.commons.json.JsopWriter;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.cache.CacheStats;
-import org.apache.jackrabbit.oak.cache.CacheValue;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.kernel.BlobSerializer;
import org.apache.jackrabbit.oak.plugins.document.Branch.BranchCommit;
@@ -249,7 +249,7 @@
*
* Key: PathRev, value: DocumentNodeState
*/
- private final Cache nodeCache;
+ private final Cache nodeCache;
private final CacheStats nodeCacheStats;
/**
@@ -257,7 +257,7 @@
*
* Key: PathRev, value: Children
*/
- private final Cache nodeChildrenCache;
+ private final Cache nodeChildrenCache;
private final CacheStats nodeChildrenCacheStats;
/**
@@ -265,7 +265,7 @@
*
* Key: StringValue, value: Children
*/
- private final Cache docChildrenCache;
+ private final Cache docChildrenCache;
private final CacheStats docChildrenCacheStats;
/**
@@ -319,6 +319,8 @@
private final boolean disableBranches;
+ private PersistentCache persistentCache;
+
public DocumentNodeStore(DocumentMK.Builder builder) {
this.blobStore = builder.getBlobStore();
if (builder.isUseSimpleRevision()) {
@@ -361,15 +363,15 @@
//TODO Make stats collection configurable as it add slight overhead
- nodeCache = builder.buildCache(builder.getNodeCacheSize());
+ nodeCache = builder.buildNodeCache(this);
nodeCacheStats = new CacheStats(nodeCache, "Document-NodeState",
builder.getWeigher(), builder.getNodeCacheSize());
- nodeChildrenCache = builder.buildCache(builder.getChildrenCacheSize());
+ nodeChildrenCache = builder.buildChildrenCache();
nodeChildrenCacheStats = new CacheStats(nodeChildrenCache, "Document-NodeChildren",
builder.getWeigher(), builder.getChildrenCacheSize());
- docChildrenCache = builder.buildCache(builder.getDocChildrenCacheSize());
+ docChildrenCache = builder.buildDocChildrenCache();
docChildrenCacheStats = new CacheStats(docChildrenCache, "Document-DocChildren",
builder.getWeigher(), builder.getDocChildrenCacheSize());
@@ -465,6 +467,9 @@
}
}
}
+ if (persistentCache != null) {
+ persistentCache.close();
+ }
}
@Nonnull
@@ -686,7 +691,7 @@
return n;
}
});
- return node == missing ? null : node;
+ return node == missing || node.equals(missing) ? null : node;
} catch (ExecutionException e) {
throw new MicroKernelException(e);
}
@@ -701,32 +706,28 @@
}
final String path = checkNotNull(parent).getPath();
final Revision readRevision = parent.getLastRevision();
- PathRev key = childNodeCacheKey(path, readRevision, name);
- DocumentNodeState.Children children;
- for (;;) {
- try {
- children = nodeChildrenCache.get(key, new Callable() {
- @Override
- public DocumentNodeState.Children call() throws Exception {
- return readChildren(parent, name, limit);
- }
- });
- } catch (ExecutionException e) {
- throw new MicroKernelException(
- "Error occurred while fetching children for path "
- + path, e.getCause());
+ try {
+ PathRev key = childNodeCacheKey(path, readRevision, name);
+ DocumentNodeState.Children children = nodeChildrenCache.get(key, new Callable() {
+ @Override
+ public DocumentNodeState.Children call() throws Exception {
+ return readChildren(parent, name, limit);
+ }
+ });
+ if (children.children.size() < limit && children.hasMore) {
+ // not enough children loaded - load more,
+ // and put that in the cache
+ // (not using nodeChildrenCache.invalidate, because
+ // the generational persistent cache doesn't support that)
+ children = readChildren(parent, name, limit);
+ nodeChildrenCache.put(key, children);
}
- if (children.hasMore && limit > children.children.size()) {
- // there are potentially more children and
- // current cache entry contains less than requested limit
- // -> need to refresh entry with current limit
- nodeChildrenCache.invalidate(key);
- } else {
- // use this cache entry
- break;
- }
+ return children;
+ } catch (ExecutionException e) {
+ throw new MicroKernelException(
+ "Error occurred while fetching children for path "
+ + path, e.getCause());
}
- return children;
}
/**
@@ -812,7 +813,7 @@
// or more than 16k child docs are requested
return store.query(Collection.NODES, from, to, limit);
}
- CacheValue key = new StringValue(path);
+ StringValue key = new StringValue(path);
// check cache
NodeDocument.Children c = docChildrenCache.getIfPresent(key);
if (c == null) {
@@ -937,13 +938,13 @@
}
}
if (isNew) {
- CacheValue key = childNodeCacheKey(path, rev, null);
DocumentNodeState.Children c = new DocumentNodeState.Children();
Set set = Sets.newTreeSet();
for (String p : added) {
set.add(Utils.unshareString(PathUtils.getName(p)));
}
c.children.addAll(set);
+ PathRev key = childNodeCacheKey(path, rev, null);
nodeChildrenCache.put(key, c);
}
@@ -962,7 +963,7 @@
// update docChildrenCache
if (!added.isEmpty()) {
- CacheValue docChildrenKey = new StringValue(path);
+ StringValue docChildrenKey = new StringValue(path);
NodeDocument.Children docChildren = docChildrenCache.getIfPresent(docChildrenKey);
if (docChildren != null) {
int currentSize = docChildren.childNames.size();
@@ -1628,7 +1629,11 @@
if (toNode != null) {
// exists in both revisions
// check if different
- if (!fromNode.getLastRevision().equals(toNode.getLastRevision())) {
+ Revision a = fromNode.getLastRevision();
+ Revision b = toNode.getLastRevision();
+ if (a == null && b == null) {
+ // ok
+ } else if (a == null || b == null || !a.equals(b)) {
w.tag('^').key(name).object().endObject().newline();
}
} else {
@@ -1693,7 +1698,8 @@
private static PathRev childNodeCacheKey(@Nonnull String path,
@Nonnull Revision readRevision,
@Nullable String name) {
- return new PathRev((name == null ? "" : name) + path, readRevision);
+ String p = (name == null ? "" : name) + path;
+ return new PathRev(p, readRevision);
}
private static DocumentRootBuilder asDocumentRootBuilder(NodeBuilder builder)
@@ -1878,4 +1884,8 @@
public LastRevRecoveryAgent getLastRevRecoveryAgent() {
return lastRevRecoveryAgent;
}
+
+ public void setPersistentCache(PersistentCache persistentCache) {
+ this.persistentCache = persistentCache;
+ }
}
Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/MemoryDiffCache.java
===================================================================
--- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/MemoryDiffCache.java (revision 1633090)
+++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/MemoryDiffCache.java (working copy)
@@ -20,7 +20,6 @@
import javax.annotation.Nonnull;
import org.apache.jackrabbit.oak.cache.CacheStats;
-import org.apache.jackrabbit.oak.cache.CacheValue;
import org.apache.jackrabbit.oak.plugins.document.util.StringValue;
import com.google.common.cache.Cache;
@@ -37,12 +36,12 @@
*
* Key: PathRev, value: StringValue
*/
- protected final Cache diffCache;
+ protected final Cache diffCache;
protected final CacheStats diffCacheStats;
MemoryDiffCache(DocumentMK.Builder builder) {
- diffCache = builder.buildCache(builder.getDiffCacheSize());
+ diffCache = builder.buildDiffCache();
diffCacheStats = new CacheStats(diffCache, "Document-Diff",
builder.getWeigher(), builder.getDiffCacheSize());
}
Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java
===================================================================
--- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java (revision 1633090)
+++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java (working copy)
@@ -22,6 +22,7 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.NavigableMap;
import java.util.Queue;
import java.util.Set;
@@ -43,6 +44,10 @@
import com.google.common.collect.Queues;
import org.apache.jackrabbit.oak.cache.CacheValue;
import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.commons.json.JsopBuilder;
+import org.apache.jackrabbit.oak.commons.json.JsopReader;
+import org.apache.jackrabbit.oak.commons.json.JsopTokenizer;
+import org.apache.jackrabbit.oak.commons.json.JsopWriter;
import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore;
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
import org.slf4j.Logger;
@@ -1363,7 +1368,7 @@
return false;
}
int c1 = context.getRevisionComparator().compare(r1, r2);
- int c2 = r1.compareRevisionTimeThenClusterId(r2);
+ int c2 = r1.compareTo(r2);
if (c1 == 0) {
return c2 == 0;
} else if (c1 < 0) {
@@ -1661,12 +1666,89 @@
private Map getDeleted() {
return ValueMap.create(this, DELETED);
}
-
+
+ public String asString() {
+ JsopWriter json = new JsopBuilder();
+ toJson(json, data);
+ return json.toString();
+ }
+
+ @SuppressWarnings("unchecked")
+ private static void toJson(JsopWriter json, Map extends Object, Object> map) {
+ for (Entry extends Object, Object>e : map.entrySet()) {
+ json.key(e.getKey().toString());
+ Object value = e.getValue();
+ if (value == null) {
+ json.value(null);
+ } else if (value instanceof Boolean) {
+ json.value((Boolean) value);
+ } else if (value instanceof Long) {
+ json.value((Long) value);
+ } else if (value instanceof Integer) {
+ json.value((Integer) value);
+ } else if (value instanceof Map) {
+ json.object();
+ toJson(json, (Map