Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java (revision 1599671) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java (working copy) @@ -52,7 +52,7 @@ /** Logger instance */ private static final Logger log = LoggerFactory.getLogger(Compactor.class); - public static void compact(SegmentStore store) { + public static Map compact(SegmentStore store) { SegmentWriter writer = store.getTracker().getWriter(); Compactor compactor = new Compactor(writer); @@ -74,6 +74,7 @@ before = head; after = builder.getNodeState(); } + return compactor.getCompacted(); } private final SegmentWriter writer; @@ -255,4 +256,8 @@ } } + public Map getCompacted() { + return compacted; + } + } Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MapRecord.java =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MapRecord.java (revision 1599671) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/MapRecord.java (working copy) @@ -364,7 +364,7 @@ } boolean compare(MapRecord before, final NodeStateDiff diff) { - if (fastEquals(this, before)) { + if (fastEquals(this, before, getStore())) { return true; } Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Record.java =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Record.java (revision 1599671) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Record.java (working copy) @@ -16,6 +16,8 @@ */ package org.apache.jackrabbit.oak.plugins.segment; +import java.util.Map; + import javax.annotation.Nonnull; /** @@ -23,16 +25,29 @@ */ class Record { - static boolean fastEquals(Object a, Object b) { - return a instanceof Record && fastEquals((Record) a, b); + static boolean fastEquals(Object a, Object b, SegmentStore store) { + return a instanceof Record && fastEquals((Record) a, b, store); } - static boolean fastEquals(Record a, Object b) { - return b instanceof Record && fastEquals(a, (Record) b); + static boolean fastEquals(Record a, Object b, SegmentStore store) { + return b instanceof Record && fastEquals(a, (Record) b, store); } - static boolean fastEquals(Record a, Record b) { - return a.segmentId == b.segmentId && a.offset == b.offset; + static boolean fastEquals(Record a, Record b, SegmentStore store) { + Map compaction = store.getCompactionMap(); + if (compaction == null) { + return a.segmentId == b.segmentId && a.offset == b.offset; + } + RecordId aId = a.getRecordId(); + if (compaction.containsKey(aId)) { + aId = compaction.get(aId); + } + RecordId bId = b.getRecordId(); + if (compaction.containsKey(bId)) { + bId = compaction.get(bId); + } + return aId.getSegmentId() == bId.getSegmentId() + && aId.getOffset() == bId.getOffset(); } /** @@ -69,6 +84,15 @@ } /** + * Returns the segment store. + * + * @return segment store + */ + public SegmentStore getStore() { + return segmentId.getTracker().getStore(); + } + + /** * Returns the identifier of this record. * * @return record identifier @@ -113,7 +137,7 @@ @Override public boolean equals(Object that) { - return fastEquals(this, that); + return fastEquals(this, that, getStore()); } @Override Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java (revision 1599671) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java (working copy) @@ -178,7 +178,7 @@ @Override public boolean equals(Object object) { - if (object == this || fastEquals(this, object)) { + if (object == this || fastEquals(this, object, getStore())) { return true; } else { return object instanceof Blob Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeState.java =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeState.java (revision 1599671) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeState.java (working copy) @@ -381,7 +381,7 @@ @Override public boolean compareAgainstBaseState(NodeState base, NodeStateDiff diff) { - if (this == base || fastEquals(this, base)) { + if (this == base || fastEquals(this, base, getStore())) { return true; // no changes } else if (base == EMPTY_NODE || !base.exists()) { // special case return EmptyNodeState.compareAgainstEmptyState(this, diff); @@ -476,7 +476,7 @@ if (!diff.childNodeAdded(afterChildName, afterNode)) { return false; } - } else if (!fastEquals(afterNode, beforeNode)) { + } else if (!fastEquals(afterNode, beforeNode, getStore())) { if (!diff.childNodeChanged( afterChildName, beforeNode, afterNode)) { return false; @@ -512,7 +512,7 @@ NodeState beforeChild = beforeTemplate.getChildNode(beforeChildName, beforeId); if (beforeChild.exists()) { - if (!fastEquals(afterChild, beforeChild) + if (!fastEquals(afterChild, beforeChild, getStore()) && !diff.childNodeChanged( childName, beforeChild, afterChild)) { return false; @@ -550,7 +550,7 @@ @Override public boolean equals(Object object) { - if (this == object || fastEquals(this, object)) { + if (this == object || fastEquals(this, object, getStore())) { return true; } else if (object instanceof SegmentNodeState) { SegmentNodeState that = (SegmentNodeState) object; Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java (revision 1599671) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java (working copy) @@ -150,7 +150,7 @@ NodeState root = getRoot(); SegmentNodeState before = snb.getBaseState(); - if (!fastEquals(before, root)) { + if (!fastEquals(before, root, store)) { SegmentNodeState after = snb.getNodeState(); snb.reset(root); after.compareAgainstBaseState( @@ -328,7 +328,7 @@ private SegmentNodeBuilder prepare() throws CommitFailedException { SegmentNodeState state = head.get(); SegmentNodeBuilder builder = state.builder(); - if (fastEquals(before, state.getChildNode(ROOT))) { + if (fastEquals(before, state.getChildNode(ROOT), store)) { // use a shortcut when there are no external changes builder.setChildNode( ROOT, hook.processCommit(before, after, info)); @@ -416,7 +416,7 @@ NodeState execute() throws CommitFailedException, InterruptedException { // only do the merge if there are some changes to commit - if (!fastEquals(before, after)) { + if (!fastEquals(before, after, store)) { long timeout = optimisticMerge(); if (timeout >= 0) { pessimisticMerge(timeout); Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java (revision 1599671) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java (working copy) @@ -16,6 +16,8 @@ */ package org.apache.jackrabbit.oak.plugins.segment; +import java.util.Map; + import javax.annotation.CheckForNull; import javax.annotation.Nonnull; @@ -84,4 +86,7 @@ */ void gc(); + @CheckForNull + Map getCompactionMap(); + } Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Template.java =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Template.java (revision 1599671) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Template.java (working copy) @@ -240,7 +240,7 @@ // TODO: Leverage the HAMT data structure for the comparison MapRecord thisMap = getChildNodeMap(thisId); MapRecord thatMap = getChildNodeMap(thatId); - if (fastEquals(thisMap, thatMap)) { + if (fastEquals(thisMap, thatMap, thisMap.getStore())) { return true; // shortcut } else if (thisMap.size() != thatMap.size()) { return false; // shortcut Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (revision 1599671) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (working copy) @@ -137,6 +137,12 @@ private int compactThreshold = 10; /** + * Map that contains the link between the old record ids and new record ids + * of the compacted states. + */ + private final AtomicReference> compactionMap; + + /** * List of old tar file generations that are waiting to be removed. */ private final LinkedList toBeRemoved = newLinkedList(); @@ -266,6 +272,8 @@ flushThread.setPriority(Thread.MIN_PRIORITY); flushThread.start(); + compactionMap = new AtomicReference>(null); + log.info("TarMK opened: {} (mmap={})", directory, memoryMapping); } @@ -410,9 +418,11 @@ private void compact() { if (compactNeeded.getAndSet(false)) { long start = System.nanoTime(); - Compactor.compact(this); - log.info("TarMK Compaction: Completed in {}ms", MILLISECONDS - .convert(System.nanoTime() - start, NANOSECONDS)); + compactionMap.set(Compactor.compact(this)); + // TODO remove size + log.info("TarMK Compaction (" + compactionMap.get().size() + + "): Completed in {}ms", MILLISECONDS.convert( + System.nanoTime() - start, NANOSECONDS)); cleanupNeeded.set(true); } } @@ -627,4 +637,9 @@ return index; } + @Override + public Map getCompactionMap() { + return compactionMap.get(); + } + } Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/http/HttpStore.java =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/http/HttpStore.java (revision 1599671) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/http/HttpStore.java (working copy) @@ -27,6 +27,7 @@ import java.net.URL; import java.net.URLConnection; import java.nio.ByteBuffer; +import java.util.Map; import javax.annotation.CheckForNull; @@ -39,6 +40,7 @@ import org.apache.jackrabbit.oak.plugins.segment.SegmentStore; import com.google.common.io.ByteStreams; + import org.apache.jackrabbit.oak.spi.blob.BlobStore; public class HttpStore implements SegmentStore { @@ -167,4 +169,9 @@ // TODO: distributed gc } + @Override + public Map getCompactionMap() { + return null; + } + } Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java (revision 1599671) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java (working copy) @@ -1,128 +1,135 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.jackrabbit.oak.plugins.segment.memory; - -import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; - -import java.nio.ByteBuffer; -import java.util.concurrent.ConcurrentMap; - -import javax.annotation.Nonnull; - -import org.apache.jackrabbit.oak.api.Blob; -import org.apache.jackrabbit.oak.plugins.segment.Segment; -import org.apache.jackrabbit.oak.plugins.segment.SegmentId; -import org.apache.jackrabbit.oak.plugins.segment.SegmentTracker; -import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState; -import org.apache.jackrabbit.oak.plugins.segment.SegmentStore; -import org.apache.jackrabbit.oak.plugins.segment.SegmentWriter; -import org.apache.jackrabbit.oak.spi.blob.BlobStore; -import org.apache.jackrabbit.oak.spi.state.NodeBuilder; -import org.apache.jackrabbit.oak.spi.state.NodeState; - -import com.google.common.collect.Maps; - -public class MemoryStore implements SegmentStore { - - private final SegmentTracker tracker = new SegmentTracker(this); - - private SegmentNodeState head; - - private final ConcurrentMap segments = - Maps.newConcurrentMap(); - - public MemoryStore(NodeState root) { - NodeBuilder builder = EMPTY_NODE.builder(); - builder.setChildNode("root", root); - - SegmentWriter writer = tracker.getWriter(); - this.head = writer.writeNode(builder.getNodeState()); - writer.flush(); - } - - public MemoryStore() { - this(EMPTY_NODE); - } - - @Override - public SegmentTracker getTracker() { - return tracker; - } - - @Override - public synchronized SegmentNodeState getHead() { - return head; - } - - @Override - public synchronized boolean setHead(SegmentNodeState base, SegmentNodeState head) { - if (this.head.getRecordId().equals(base.getRecordId())) { - this.head = head; - return true; - } else { - return false; - } - } - - @Override - public boolean containsSegment(SegmentId id) { - return id.getTracker() == tracker || segments.containsKey(id); - } - - @Override @Nonnull - public Segment readSegment(SegmentId id) { - Segment segment = segments.get(id); - if (segment != null) { - return segment; - } else { - throw new IllegalArgumentException("Segment not found: " + id); - } - } - - @Override - public void writeSegment( - SegmentId id, byte[] data, int offset, int length) { - ByteBuffer buffer = ByteBuffer.allocate(length); - buffer.put(data, offset, length); - buffer.rewind(); - Segment segment = new Segment(tracker, id, buffer); - if (segments.putIfAbsent(id, segment) != null) { - throw new IllegalStateException("Segment override: " + id); - } - } - - @Override - public void close() { - } - - @Override - public Blob readBlob(String reference) { - return null; - } - - @Override - public BlobStore getBlobStore() { - return null; - } - - @Override - public void gc() { - System.gc(); - segments.keySet().retainAll(tracker.getReferencedSegmentIds()); - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.segment.memory; + +import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; + +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; + +import javax.annotation.Nonnull; + +import org.apache.jackrabbit.oak.api.Blob; +import org.apache.jackrabbit.oak.plugins.segment.RecordId; +import org.apache.jackrabbit.oak.plugins.segment.Segment; +import org.apache.jackrabbit.oak.plugins.segment.SegmentId; +import org.apache.jackrabbit.oak.plugins.segment.SegmentTracker; +import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState; +import org.apache.jackrabbit.oak.plugins.segment.SegmentStore; +import org.apache.jackrabbit.oak.plugins.segment.SegmentWriter; +import org.apache.jackrabbit.oak.spi.blob.BlobStore; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; + +import com.google.common.collect.Maps; + +public class MemoryStore implements SegmentStore { + + private final SegmentTracker tracker = new SegmentTracker(this); + + private SegmentNodeState head; + + private final ConcurrentMap segments = + Maps.newConcurrentMap(); + + public MemoryStore(NodeState root) { + NodeBuilder builder = EMPTY_NODE.builder(); + builder.setChildNode("root", root); + + SegmentWriter writer = tracker.getWriter(); + this.head = writer.writeNode(builder.getNodeState()); + writer.flush(); + } + + public MemoryStore() { + this(EMPTY_NODE); + } + + @Override + public SegmentTracker getTracker() { + return tracker; + } + + @Override + public synchronized SegmentNodeState getHead() { + return head; + } + + @Override + public synchronized boolean setHead(SegmentNodeState base, SegmentNodeState head) { + if (this.head.getRecordId().equals(base.getRecordId())) { + this.head = head; + return true; + } else { + return false; + } + } + + @Override + public boolean containsSegment(SegmentId id) { + return id.getTracker() == tracker || segments.containsKey(id); + } + + @Override @Nonnull + public Segment readSegment(SegmentId id) { + Segment segment = segments.get(id); + if (segment != null) { + return segment; + } else { + throw new IllegalArgumentException("Segment not found: " + id); + } + } + + @Override + public void writeSegment( + SegmentId id, byte[] data, int offset, int length) { + ByteBuffer buffer = ByteBuffer.allocate(length); + buffer.put(data, offset, length); + buffer.rewind(); + Segment segment = new Segment(tracker, id, buffer); + if (segments.putIfAbsent(id, segment) != null) { + throw new IllegalStateException("Segment override: " + id); + } + } + + @Override + public void close() { + } + + @Override + public Blob readBlob(String reference) { + return null; + } + + @Override + public BlobStore getBlobStore() { + return null; + } + + @Override + public void gc() { + System.gc(); + segments.keySet().retainAll(tracker.getReferencedSegmentIds()); + } + + @Override + public Map getCompactionMap() { + return null; + } + +}