Index: oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Compactor.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Compactor.java (date 1477426500000) +++ oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Compactor.java (date 1477481340000) @@ -18,11 +18,12 @@ import static com.google.common.collect.Lists.newArrayList; import static com.google.common.collect.Maps.newHashMap; +import static java.lang.Long.numberOfLeadingZeros; import static org.apache.jackrabbit.oak.api.Type.BINARIES; import static org.apache.jackrabbit.oak.api.Type.BINARY; import static org.apache.jackrabbit.oak.commons.PathUtils.concat; import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; -import static org.apache.jackrabbit.oak.segment.RecordCache.newRecordCache; +import static org.apache.jackrabbit.oak.segment.file.PriorityCache.nextPowerOfTwo; import java.io.IOException; import java.io.InputStream; @@ -38,6 +39,7 @@ import org.apache.jackrabbit.oak.plugins.memory.MultiBinaryPropertyState; import org.apache.jackrabbit.oak.plugins.memory.PropertyStates; import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions; +import org.apache.jackrabbit.oak.segment.file.PriorityCache; import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.state.ApplyDiff; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; @@ -71,12 +73,6 @@ private final SegmentWriter writer; - /** - * Filters nodes that will be included in the compaction map, allowing for - * optimization in case of an offline compaction - */ - private final Predicate includeInMap = new OfflineCompactionPredicate(); - private final ProgressTracker progress = new ProgressTracker(); /** @@ -126,7 +122,10 @@ } } - private final RecordCache cache = newRecordCache(cacheSize); + private final PriorityCache blobCache = + new PriorityCache<>((int) nextPowerOfTwo(cacheSize/2)); + private final PriorityCache nodeCache = + new PriorityCache<>((int) nextPowerOfTwo(cacheSize/2)); public Compactor(SegmentReader reader, SegmentWriter writer, BlobStore blobStore, Supplier cancel, SegmentGCOptions gc) { @@ -232,7 +231,7 @@ RecordId id = null; if (after instanceof SegmentNodeState) { id = ((SegmentNodeState) after).getRecordId(); - RecordId compactedId = cache.get(id); + RecordId compactedId = nodeCache.get(id, 0); if (compactedId != null) { builder.setChildNode(name, new SegmentNodeState(reader, writer, compactedId)); @@ -251,11 +250,10 @@ boolean success = new CompactDiff(child, path, name).diff( EMPTY_NODE, after); if (success) { - SegmentNodeState state = writer.writeNode(child - .getNodeState()); + SegmentNodeState state = writer.writeNode(child.getNodeState()); builder.setChildNode(name, state); - if (id != null && includeInMap.apply(after)) { - cache.put(id, state.getRecordId()); + if (id != null) { + nodeCache.put(id, state.getRecordId(), 0, cost(state)); } } return success; @@ -275,7 +273,7 @@ RecordId id = null; if (after instanceof SegmentNodeState) { id = ((SegmentNodeState) after).getRecordId(); - RecordId compactedId = cache.get(id); + RecordId compactedId = nodeCache.get(id, 0); if (compactedId != null) { builder.setChildNode(name, new SegmentNodeState(reader, writer, compactedId)); @@ -293,10 +291,9 @@ boolean success = new CompactDiff(child, path, name).diff( before, after); if (success) { - RecordId compactedId = writer.writeNode( - child.getNodeState()).getRecordId(); + SegmentNodeState state = writer.writeNode(child.getNodeState()); if (id != null) { - cache.put(id, compactedId); + nodeCache.put(id, state.getRecordId(), 0, cost(state)); } } return success; @@ -307,6 +304,20 @@ } } + private static byte cost(SegmentNodeState node) { + long childCount = node.getChildNodeCount(Long.MAX_VALUE); + return cost(childCount); + } + + private static byte cost(SegmentBlob blob) { + long length = blob.length(); + return cost(length); + } + + private static byte cost(long n) { + return (byte) (Byte.MIN_VALUE + 64 - numberOfLeadingZeros(n)); + } + private PropertyState compact(PropertyState property) { String name = property.getName(); Type type = property.getType(); @@ -340,7 +351,7 @@ RecordId id = sb.getRecordId(); // TODO verify binary impact on cache - RecordId compactedId = cache.get(id); + RecordId compactedId = blobCache.get(id, 0); if (compactedId != null) { return new SegmentBlob(blobStore, compactedId); } @@ -349,14 +360,12 @@ // if the blob is external, just clone it if (sb.isExternal()) { - SegmentBlob clone = writer.writeBlob(sb); - cache.put(id, clone.getRecordId()); - return clone; + return writer.writeBlob(sb); } // if the blob is inlined, just clone it if (sb.length() < Segment.MEDIUM_LIMIT) { SegmentBlob clone = writer.writeBlob(blob); - cache.put(id, clone.getRecordId()); + blobCache.put(id, clone.getRecordId(), 0, cost(clone)); return clone; } @@ -373,7 +382,6 @@ for (RecordId duplicateId : ids) { if (new SegmentBlob(blobStore, duplicateId) .equals(sb)) { - cache.put(id, duplicateId); return new SegmentBlob(blobStore, duplicateId); } } @@ -382,7 +390,7 @@ // if not, clone the large blob and keep track of the result sb = writer.writeBlob(blob); - cache.put(id, sb.getRecordId()); + blobCache.put(id, sb.getRecordId(), 0, cost(sb)); if (dedup) { if (ids == null) { Index: oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/PriorityCache.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/PriorityCache.java (date 1477426500000) +++ oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/PriorityCache.java (date 1477481340000) @@ -22,6 +22,8 @@ import static com.google.common.base.Preconditions.checkArgument; import static java.lang.Integer.bitCount; import static java.lang.Integer.numberOfTrailingZeros; +import static java.lang.Long.numberOfLeadingZeros; +import static java.lang.Math.max; import static java.util.Arrays.fill; import javax.annotation.CheckForNull; @@ -101,6 +103,15 @@ } } + /** + * Round {@code size} up to the next power of two or 1 for negative values. + * @param size + * @return the next power of two starting from {@code size}. + */ + public static long nextPowerOfTwo(int size) { + return 1L << (64L - numberOfLeadingZeros((long)max(1, size) - 1L)); + } + /** * Create a new instance of the given {@code size}. {@code rehash} specifies the number * of rehashes to resolve a clash. Index: oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/PriorityCacheTest.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/PriorityCacheTest.java (date 1477426500000) +++ oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/PriorityCacheTest.java (date 1477481340000) @@ -150,4 +150,17 @@ assertEquals(200, cache.size()); } + @Test + public void nextPowerOfTwo() { + assertEquals(1, PriorityCache.nextPowerOfTwo(-1)); + assertEquals(1, PriorityCache.nextPowerOfTwo(-123)); + assertEquals(1, PriorityCache.nextPowerOfTwo(0)); + assertEquals(1, PriorityCache.nextPowerOfTwo(1)); + assertEquals(2, PriorityCache.nextPowerOfTwo(2)); + assertEquals(4, PriorityCache.nextPowerOfTwo(3)); + assertEquals(4, PriorityCache.nextPowerOfTwo(4)); + assertEquals(512, PriorityCache.nextPowerOfTwo(500)); + assertEquals(2147483648L, PriorityCache.nextPowerOfTwo(Integer.MAX_VALUE)); + } + }