diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BinaryUtils.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BinaryUtils.java new file mode 100644 index 0000000..2bc0833 --- /dev/null +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BinaryUtils.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jackrabbit.oak.segment; + +class BinaryUtils { + + private BinaryUtils() { + // Prevent instantiation + } + + static int writeByte(byte[] buffer, int position, byte value) { + buffer[position++] = value; + return position; + } + + static int writeShort(byte[] buffer, int position, short value) { + position = writeByte(buffer, position, (byte) (value >> 8)); + position = writeByte(buffer, position, (byte) (value)); + return position; + } + + static int writeInt(byte[] buffer, int position, int value) { + position = writeShort(buffer, position, (short) (value >> 16)); + position = writeShort(buffer, position, (short) (value)); + return position; + } + + static int writeLong(byte[] buffer, int position, long value) { + position = writeInt(buffer, position, (int) (value >> 32)); + position = writeInt(buffer, position, (int) (value)); + return position; + } + +} diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java index 2c970be..23c9222 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java @@ -22,7 +22,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkPositionIndexes; import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.Lists.newArrayListWithCapacity; +import static com.google.common.collect.Maps.newHashMap; import static org.apache.jackrabbit.oak.commons.IOUtils.closeQuietly; import static org.apache.jackrabbit.oak.segment.SegmentId.isDataSegmentId; import static org.apache.jackrabbit.oak.segment.SegmentVersion.LATEST_VERSION; @@ -34,7 +34,7 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.nio.ByteBuffer; import java.util.Arrays; -import java.util.List; +import java.util.Map; import java.util.UUID; import javax.annotation.CheckForNull; @@ -57,12 +57,14 @@ import org.apache.jackrabbit.oak.plugins.memory.PropertyStates; */ public class Segment { + static final int HEADER_SIZE = 18; + /** * Number of bytes used for storing a record identifier. One byte * is used for identifying the segment and two for the record offset * within that segment. */ - static final int RECORD_ID_BYTES = 1 + 2; + static final int RECORD_ID_BYTES = 8 + 8 + 2; /** * The limit on segment references within one segment. Since record @@ -111,12 +113,12 @@ public class Segment { */ public static final int BLOB_ID_SMALL_LIMIT = 1 << 12; - public static final int REF_COUNT_OFFSET = 5; - static final int ROOT_COUNT_OFFSET = 6; public static final int GC_GENERATION_OFFSET = 10; + public static final int REFERENCED_SEGMENT_ID_COUNT_OFFSET = 14; + @Nonnull private final SegmentStore store; @@ -135,12 +137,7 @@ public class Segment { @Nonnull private final SegmentVersion version; - /** - * Referenced segment identifiers. Entries are initialized lazily in - * {@link #getRefId(int)}. Set to {@code null} for bulk segments. - */ - @CheckForNull - private final SegmentId[] refids; + private final Map recordIdCache = newHashMap(); /** * Unpacks a 4 byte aligned segment offset. @@ -193,11 +190,8 @@ public class Segment { + toHex(data.array()); } }); - this.refids = new SegmentId[getRefCount()]; - this.refids[0] = id; this.version = SegmentVersion.fromByte(segmentVersion); } else { - this.refids = null; this.version = LATEST_VERSION; } } @@ -223,8 +217,6 @@ public class Segment { this.id = store.newDataSegmentId(); this.info = checkNotNull(info); this.data = ByteBuffer.wrap(checkNotNull(buffer)); - this.refids = new SegmentId[SEGMENT_REFERENCE_LIMIT + 1]; - this.refids[0] = id; this.version = SegmentVersion.fromByte(buffer[3]); id.loaded(this); } @@ -253,14 +245,28 @@ public class Segment { return id; } - int getRefCount() { - return (data.get(REF_COUNT_OFFSET) & 0xff) + 1; - } - public int getRootCount() { return data.getShort(ROOT_COUNT_OFFSET) & 0xffff; } + public int getReferencedSegmentIdCount() { + return data.getInt(REFERENCED_SEGMENT_ID_COUNT_OFFSET); + } + + public UUID getReferencedSegmentId(int index) { + checkArgument(index < getReferencedSegmentIdCount()); + + int position = data.position(); + + position += HEADER_SIZE; + position += index * 16; + + long msb = data.getLong(position); + long lsb = data.getLong(position + 8); + + return new UUID(msb, lsb); + } + /** * Determine the gc generation a segment from its data. Note that bulk segments don't have * generations (i.e. stay at 0). @@ -285,16 +291,28 @@ public class Segment { } public RecordType getRootType(int index) { - int refCount = getRefCount(); checkArgument(index < getRootCount()); - return RecordType.values()[data.get(data.position() + refCount * 16 + index * 3) & 0xff]; + + int position = data.position(); + + position += HEADER_SIZE; + position += getReferencedSegmentIdCount() * 16; + position += index * 3; + + return RecordType.values()[data.get(position) & 0xff]; } public int getRootOffset(int index) { - int refCount = getRefCount(); checkArgument(index < getRootCount()); - return (data.getShort(data.position() + refCount * 16 + index * 3 + 1) & 0xffff) - << RECORD_ALIGN_BITS; + + int position = data.position(); + + position += HEADER_SIZE; + position += getReferencedSegmentIdCount() * 16; + position += index * 3; + position += 1; + + return (data.getShort(position) & 0xffff) << RECORD_ALIGN_BITS; } private volatile String info; @@ -316,48 +334,12 @@ public class Segment { */ @CheckForNull public String getSegmentInfo() { - if (info == null && getRefCount() != 0) { + if (info == null && id.isDataSegmentId()) { info = readString(getRootOffset(0)); } return info; } - SegmentId getRefId(int index) { - if (refids == null || index >= refids.length) { - String type = "data"; - if (!id.isDataSegmentId()) { - type = "bulk"; - } - long delta = System.currentTimeMillis() - id.getCreationTime(); - throw new IllegalStateException("RefId '" + index - + "' doesn't exist in " + type + " segment " + id - + ". Creation date delta is " + delta + " ms."); - } - SegmentId refid = refids[index]; - if (refid == null) { - synchronized (this) { - refid = refids[index]; - if (refid == null) { - int refpos = data.position() + index * 16; - long msb = data.getLong(refpos); - long lsb = data.getLong(refpos + 8); - refid = store.newSegmentId(msb, lsb); - refids[index] = refid; - } - } - } - return refid; - } - - public List getReferencedIds() { - int refcount = getRefCount(); - List ids = newArrayListWithCapacity(refcount); - for (int refid = 0; refid < refcount; refid++) { - ids.add(getRefId(refid)); - } - return ids; - } - public int size() { return data.remaining(); } @@ -401,9 +383,28 @@ public class Segment { } private RecordId internalReadRecordId(int pos) { - SegmentId refid = getRefId(data.get(pos) & 0xff); - int offset = ((data.get(pos + 1) & 0xff) << 8) | (data.get(pos + 2) & 0xff); - return new RecordId(refid, offset << RECORD_ALIGN_BITS); + RecordId recordId = recordIdCache.get(pos); + + if (recordId != null) { + return recordId; + } + + synchronized (recordIdCache) { + recordId = recordIdCache.get(pos); + + if (recordId != null) { + return recordId; + } + + long msb = data.getLong(pos); + long lsb = data.getLong(pos + 8); + int offset = (data.getShort(pos + 16) & 0xffff) << RECORD_ALIGN_BITS; + recordId = new RecordId(store.newSegmentId(msb, lsb), offset); + + recordIdCache.put(pos, recordId); + + return recordId; + } } @Nonnull @@ -538,17 +539,13 @@ public class Segment { } if (id.isDataSegmentId()) { writer.println("--------------------------------------------------------------------------"); - int refcount = getRefCount(); - for (int refid = 0; refid < refcount; refid++) { - writer.format("reference %02x: %s%n", refid, getRefId(refid)); + + for (int i = 0; i < getReferencedSegmentIdCount(); i++) { + writer.format("reference %02x: %s%n", i, getReferencedSegmentId(i)); } - int rootcount = data.getShort(ROOT_COUNT_OFFSET) & 0xffff; - int pos = data.position() + refcount * 16; - for (int rootid = 0; rootid < rootcount; rootid++) { - writer.format( - "root %d: %s at %04x%n", rootid, - RecordType.values()[data.get(pos + rootid * 3) & 0xff], - data.getShort(pos + rootid * 3 + 1) & 0xffff); + + for (int i = 0; i < getRootCount(); i++) { + writer.format("root %d: %s at %04x%n", i, getRootType(i), getRootOffset(i)); } } writer.println("--------------------------------------------------------------------------"); diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java index cc9a164..b94f279 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java @@ -29,19 +29,18 @@ import static java.lang.System.arraycopy; import static java.lang.System.currentTimeMillis; import static java.lang.System.identityHashCode; import static org.apache.jackrabbit.oak.segment.Segment.GC_GENERATION_OFFSET; +import static org.apache.jackrabbit.oak.segment.Segment.HEADER_SIZE; import static org.apache.jackrabbit.oak.segment.Segment.MAX_SEGMENT_SIZE; import static org.apache.jackrabbit.oak.segment.Segment.RECORD_ID_BYTES; -import static org.apache.jackrabbit.oak.segment.Segment.SEGMENT_REFERENCE_LIMIT; import static org.apache.jackrabbit.oak.segment.Segment.align; import static org.apache.jackrabbit.oak.segment.SegmentId.isDataSegmentId; import static org.apache.jackrabbit.oak.segment.SegmentVersion.LATEST_VERSION; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.Collection; -import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.UUID; import javax.annotation.CheckForNull; import javax.annotation.Nonnull; @@ -109,6 +108,8 @@ public class SegmentBufferWriter implements WriteOperationHandler { */ private final Map roots = newLinkedHashMap(); + private final Set referencedSegmentIds = newHashSet(); + @Nonnull private final SegmentStore store; @@ -208,6 +209,7 @@ public class SegmentBufferWriter implements WriteOperationHandler { length = 0; position = buffer.length; roots.clear(); + referencedSegmentIds.clear(); String metaInfo = "{\"wid\":\"" + wid + '"' + @@ -229,27 +231,23 @@ public class SegmentBufferWriter implements WriteOperationHandler { } public void writeByte(byte value) { - buffer[position++] = value; + position = BinaryUtils.writeByte(buffer, position, value); dirty = true; } public void writeShort(short value) { - buffer[position++] = (byte) (value >> 8); - buffer[position++] = (byte) value; + position = BinaryUtils.writeShort(buffer, position, value); dirty = true; } public void writeInt(int value) { - buffer[position++] = (byte) (value >> 24); - buffer[position++] = (byte) (value >> 16); - buffer[position++] = (byte) (value >> 8); - buffer[position++] = (byte) value; + position = BinaryUtils.writeInt(buffer, position, value); dirty = true; } public void writeLong(long value) { - writeInt((int) (value >> 32)); - writeInt((int) value); + position = BinaryUtils.writeLong(buffer, position, value); + dirty = true; } /** @@ -279,12 +277,20 @@ public class SegmentBufferWriter implements WriteOperationHandler { } int offset = recordId.getOffset(); + checkState(0 <= offset && offset < MAX_SEGMENT_SIZE); checkState(offset == align(offset, 1 << Segment.RECORD_ALIGN_BITS)); - buffer[position++] = (byte) getSegmentRef(recordId.getSegmentId()); - buffer[position++] = (byte) (offset >> (8 + Segment.RECORD_ALIGN_BITS)); - buffer[position++] = (byte) (offset >> Segment.RECORD_ALIGN_BITS); + long msb = recordId.getSegmentId().getMostSignificantBits(); + long lsb = recordId.getSegmentId().getLeastSignificantBits(); + + writeLong(msb); + writeLong(lsb); + writeShort((short)((offset >> Segment.RECORD_ALIGN_BITS) & 0xffff)); + + if (!recordId.getSegmentId().equals(segment.getSegmentId())) { + referencedSegmentIds.add(recordId.getSegmentId().asUUID()); + } statistics.recordIdCount++; @@ -317,27 +323,6 @@ public class SegmentBufferWriter implements WriteOperationHandler { return info; } - private int getSegmentRef(SegmentId segmentId) { - checkGCGeneration(segmentId); - - int refCount = segment.getRefCount(); - if (refCount > SEGMENT_REFERENCE_LIMIT) { - throw new SegmentOverflowException( - "Segment cannot have more than 255 references " + segment.getSegmentId()); - } - for (int index = 0; index < refCount; index++) { - if (segmentId.equals(segment.getRefId(index))) { - return index; - } - } - - ByteBuffer.wrap(buffer, refCount * 16, 16) - .putLong(segmentId.getMostSignificantBits()) - .putLong(segmentId.getLeastSignificantBits()); - buffer[Segment.REF_COUNT_OFFSET] = (byte) refCount; - return refCount; - } - public void writeBytes(byte[] data, int offset, int length) { arraycopy(data, offset, buffer, position, length); position += length; @@ -352,19 +337,22 @@ public class SegmentBufferWriter implements WriteOperationHandler { @Override public void flush() throws IOException { if (dirty) { - int refcount = segment.getRefCount(); - statistics.segmentIdCount = refcount; - int rootcount = roots.size(); - buffer[Segment.ROOT_COUNT_OFFSET] = (byte) (rootcount >> 8); - buffer[Segment.ROOT_COUNT_OFFSET + 1] = (byte) rootcount; + BinaryUtils.writeShort(buffer, Segment.ROOT_COUNT_OFFSET, (short) rootcount); + + int referencedSegmentIdCount = referencedSegmentIds.size(); + BinaryUtils.writeInt(buffer, Segment.REFERENCED_SEGMENT_ID_COUNT_OFFSET, referencedSegmentIdCount); + statistics.segmentIdCount = referencedSegmentIdCount; + + int totalLength = align(HEADER_SIZE + referencedSegmentIdCount * 16 + rootcount * 3 + length, 16); - length = align(refcount * 16 + rootcount * 3 + length, 16); - statistics.size = length; + if (totalLength > buffer.length) { + throw new IllegalStateException("too much data for a segment"); + } - checkState(length <= buffer.length); + statistics.size = length = totalLength; - int pos = refcount * 16; + int pos = HEADER_SIZE; if (pos + length <= buffer.length) { // the whole segment fits to the space *after* the referenced // segment identifiers we've already written, so we can safely @@ -380,11 +368,14 @@ public class SegmentBufferWriter implements WriteOperationHandler { length = buffer.length; } + for (UUID id : referencedSegmentIds) { + pos = BinaryUtils.writeLong(buffer, pos, id.getMostSignificantBits()); + pos = BinaryUtils.writeLong(buffer, pos, id.getLeastSignificantBits()); + } + for (Map.Entry entry : roots.entrySet()) { - int offset = entry.getKey().getOffset(); - buffer[pos++] = (byte) entry.getValue().ordinal(); - buffer[pos++] = (byte) (offset >> (8 + Segment.RECORD_ALIGN_BITS)); - buffer[pos++] = (byte) (offset >> Segment.RECORD_ALIGN_BITS); + pos = BinaryUtils.writeByte(buffer, pos, (byte) entry.getValue().ordinal()); + pos = BinaryUtils.writeShort(buffer, pos, (short) (entry.getKey().getOffset() >> Segment.RECORD_ALIGN_BITS)); } SegmentId segmentId = segment.getSegmentId(); @@ -421,25 +412,27 @@ public class SegmentBufferWriter implements WriteOperationHandler { // First compute the header and segment sizes based on the assumption // that *all* identifiers stored in this record point to previously // unreferenced segments. - int refCount = segment.getRefCount() + idCount; + int rootCount = roots.size() + 1; - int headerSize = refCount * 16 + rootCount * 3; + int referencedIdCount = referencedSegmentIds.size() + ids.size(); + int headerSize = HEADER_SIZE + rootCount * 3 + referencedIdCount * 16; int segmentSize = align(headerSize + recordSize + length, 16); // If the size estimate looks too big, recompute it with a more // accurate refCount value. We skip doing this when possible to // avoid the somewhat expensive list and set traversals. - if (segmentSize > buffer.length - 1 || refCount > Segment.SEGMENT_REFERENCE_LIMIT) { - refCount -= idCount; - Set segmentIds = newHashSet(); + if (segmentSize > buffer.length) { // The set of old record ids in this segment // that were previously root record ids, but will no longer be, // because the record to be written references them. // This needs to be a set, because the list of ids can // potentially reference the same record multiple times - Set notRoots = new HashSet(); + + Set segmentIds = newHashSet(); + Set notRoots = newHashSet(); + for (RecordId recordId : ids) { SegmentId segmentId = recordId.getSegmentId(); if (!(segmentId.equals(segment.getSegmentId()))) { @@ -450,20 +443,19 @@ public class SegmentBufferWriter implements WriteOperationHandler { } rootCount -= notRoots.size(); - if (!segmentIds.isEmpty()) { - for (int refid = 1; refid < refCount; refid++) { - segmentIds.remove(segment.getRefId(refid)); + // Adjust the estimation of the new referenced segment ID count. + + for (SegmentId segmentId : segmentIds) { + if (referencedSegmentIds.contains(segmentId.asUUID())) { + referencedIdCount--; } - refCount += segmentIds.size(); } - headerSize = refCount * 16 + rootCount * 3; + headerSize = HEADER_SIZE + rootCount * 3 + referencedIdCount * 16; segmentSize = align(headerSize + recordSize + length, 16); } - if (segmentSize > buffer.length - 1 - || rootCount > 0xffff - || refCount > Segment.SEGMENT_REFERENCE_LIMIT) { + if (segmentSize > buffer.length || rootCount > 0xffff) { flush(); } diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java index 6005e17..a0effcd 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java @@ -1316,13 +1316,54 @@ public class FileStore implements SegmentStore, Closeable { @Override public void writeSegment(SegmentId id, byte[] buffer, int offset, int length) throws IOException { + Segment segment = null; + + // If the segment is a data segment, create a new instance of Segment to + // access some internal information stored in the segment and to store + // in an in-memory cache for later use. + + if (id.isDataSegmentId()) { + ByteBuffer data; + + if (offset > 4096) { + data = ByteBuffer.allocate(length); + data.put(buffer, offset, length); + data.rewind(); + } else { + data = ByteBuffer.wrap(buffer, offset, length); + } + + segment = new Segment(this, segmentReader, id, data); + } + fileStoreLock.writeLock().lock(); try { int generation = Segment.getGcGeneration(wrap(buffer, offset, length), id.asUUID()); + + // Flush the segment to disk + long size = tarWriter.writeEntry( id.getMostSignificantBits(), id.getLeastSignificantBits(), - buffer, offset, length, generation); + buffer, + offset, + length, + generation + ); + + // If the segment is a data segment, update the graph before + // (potentially) flushing the TAR file. + + if (segment != null) { + UUID from = segment.getSegmentId().asUUID(); + + for (int i = 0; i < segment.getReferencedSegmentIdCount(); i++) { + tarWriter.addGraphEdge(from, segment.getReferencedSegmentId(i)); + } + } + + // Close the TAR file if the size exceeds the maximum. + if (size >= maxFileSize) { newWriter(); } @@ -1330,17 +1371,10 @@ public class FileStore implements SegmentStore, Closeable { fileStoreLock.writeLock().unlock(); } - // Keep this data segment in memory as it's likely to be accessed soon - if (id.isDataSegmentId()) { - ByteBuffer data; - if (offset > 4096) { - data = ByteBuffer.allocate(length); - data.put(buffer, offset, length); - data.rewind(); - } else { - data = ByteBuffer.wrap(buffer, offset, length); - } - segmentCache.putSegment(new Segment(this, segmentReader, id, data)); + // Keep this data segment in memory as it's likely to be accessed soon. + + if (segment != null) { + segmentCache.putSegment(segment); } } diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java index 2f6abca..1b929c6 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java @@ -22,7 +22,6 @@ import static com.google.common.base.Charsets.UTF_8; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.collect.Lists.newArrayList; import static com.google.common.collect.Lists.newArrayListWithCapacity; -import static com.google.common.collect.Maps.newHashMap; import static com.google.common.collect.Maps.newHashMapWithExpectedSize; import static com.google.common.collect.Maps.newLinkedHashMap; import static com.google.common.collect.Maps.newTreeMap; @@ -30,7 +29,6 @@ import static com.google.common.collect.Sets.newHashSet; import static com.google.common.collect.Sets.newHashSetWithExpectedSize; import static java.nio.ByteBuffer.wrap; import static java.util.Collections.singletonList; -import static org.apache.jackrabbit.oak.segment.Segment.REF_COUNT_OFFSET; import static org.apache.jackrabbit.oak.segment.Segment.getGcGeneration; import static org.apache.jackrabbit.oak.segment.SegmentId.isDataSegmentId; import static org.apache.jackrabbit.oak.segment.file.TarWriter.BINARY_REFERENCES_MAGIC; @@ -62,6 +60,7 @@ import com.google.common.collect.Sets; import org.apache.commons.io.FileUtils; import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector; import org.apache.jackrabbit.oak.segment.SegmentGraph.SegmentGraphVisitor; +import org.apache.jackrabbit.oak.segment.SegmentId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -659,26 +658,13 @@ class TarReader implements Closeable { @Nonnull private List getReferences(TarEntry entry, UUID id, Map> graph) throws IOException { - if (graph != null) { - List uuids = graph.get(id); - return uuids == null ? Collections.emptyList() : uuids; - } else { - // a pre-compiled graph is not available, so read the - // references directly from this segment - ByteBuffer segment = access.read( - entry.offset(), - Math.min(entry.size(), 16 * 256)); - int pos = segment.position(); - int refCount = segment.get(pos + REF_COUNT_OFFSET) & 0xff; - int refEnd = pos + 16 * (refCount + 1); - List refIds = newArrayList(); - for (int refPos = pos + 16; refPos < refEnd; refPos += 16) { - refIds.add(new UUID( - segment.getLong(refPos), - segment.getLong(refPos + 8))); - } - return refIds; + List references = graph.get(id); + + if (references == null) { + return Collections.emptyList(); } + + return references; } /** @@ -872,6 +858,30 @@ class TarReader implements Closeable { } } + // Reconstruct the graph index for non-cleaned segments. + + Map> graph = getGraph(false); + + for (Entry> e : graph.entrySet()) { + if (cleaned.contains(e.getKey())) { + continue; + } + + Set vertices = newHashSet(); + + for (UUID vertex : e.getValue()) { + if (cleaned.contains(vertex)) { + continue; + } + + vertices.add(vertex); + } + + for (UUID vertex : vertices) { + writer.addGraphEdge(e.getKey(), vertex); + } + } + // Reconstruct the binary reference index for non-cleaned segments. Map>> references = getBinaryReferences(); @@ -1080,68 +1090,77 @@ class TarReader implements Closeable { * @throws IOException if the tar file could not be read */ private ByteBuffer loadGraph() throws IOException { - // read the graph metadata just before the tar index entry int pos = access.length() - 2 * BLOCK_SIZE - getIndexEntrySize(); + ByteBuffer meta = access.read(pos - 16, 16); + int crc32 = meta.getInt(); int count = meta.getInt(); int bytes = meta.getInt(); int magic = meta.getInt(); if (magic != GRAPH_MAGIC) { - return null; // magic byte mismatch + log.warn("Invalid graph magic number in {}", file); + return null; } - if (count < 0 || bytes < count * 16 + 16 || BLOCK_SIZE + bytes > pos) { - log.warn("Invalid graph metadata in tar file {}", file); - return null; // impossible uuid and/or byte counts + if (count < 0) { + log.warn("Invalid number of entries in {}", file); + return null; + } + + if (bytes < 4 + count * 34) { + log.warn("Invalid entry size in {}", file); + return null; } - // this involves seeking backwards in the file, which might not - // perform well, but that's OK since we only do this once per file ByteBuffer graph = access.read(pos - bytes, bytes); byte[] b = new byte[bytes - 16]; + graph.mark(); graph.get(b); graph.reset(); CRC32 checksum = new CRC32(); checksum.update(b); + if (crc32 != (int) checksum.getValue()) { log.warn("Invalid graph checksum in tar file {}", file); - return null; // checksum mismatch + return null; } hasGraph = true; + return graph; } - private static Map> parseGraph(ByteBuffer graphByteBuffer, boolean bulkOnly) { - int count = graphByteBuffer.getInt(graphByteBuffer.limit() - 12); + private static Map> parseGraph(ByteBuffer buffer, boolean bulkOnly) { + int nEntries = buffer.getInt(buffer.limit() - 12); - ByteBuffer buffer = graphByteBuffer.duplicate(); - buffer.limit(graphByteBuffer.limit() - 16); + Map> graph = newHashMapWithExpectedSize(nEntries); - List uuids = newArrayListWithCapacity(count); - for (int i = 0; i < count; i++) { - uuids.add(new UUID(buffer.getLong(), buffer.getLong())); - } + for (int i = 0; i < nEntries; i++) { + long msb = buffer.getLong(); + long lsb = buffer.getLong(); + int nVertices = buffer.getInt(); + + List vertices = newArrayListWithCapacity(nVertices); - Map> graph = newHashMap(); - while (buffer.hasRemaining()) { - UUID uuid = uuids.get(buffer.getInt()); - List list = newArrayList(); - int refid = buffer.getInt(); - while (refid != -1) { - UUID ref = uuids.get(refid); - if (!bulkOnly || !isDataSegmentId(ref.getLeastSignificantBits())) { - list.add(ref); + for (int j = 0; j < nVertices; j++) { + long vmsb = buffer.getLong(); + long vlsb = buffer.getLong(); + + if (bulkOnly && SegmentId.isDataSegmentId(vlsb)) { + continue; } - refid = buffer.getInt(); + + vertices.add(new UUID(vmsb, vlsb)); } - graph.put(uuid, list); + + graph.put(new UUID(msb, lsb), vertices); } + return graph; } diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java index eed9ba8..e6a1b5f 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java @@ -24,10 +24,7 @@ import static com.google.common.base.Preconditions.checkPositionIndexes; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.Maps.newHashMap; import static com.google.common.collect.Maps.newLinkedHashMap; -import static com.google.common.collect.Maps.newTreeMap; import static com.google.common.collect.Sets.newHashSet; -import static org.apache.jackrabbit.oak.segment.Segment.REF_COUNT_OFFSET; -import static org.apache.jackrabbit.oak.segment.SegmentId.isDataSegmentId; import java.io.Closeable; import java.io.File; @@ -37,17 +34,13 @@ import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.util.Arrays; -import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; -import java.util.SortedMap; import java.util.UUID; import java.util.zip.CRC32; import com.google.common.base.Charsets; -import com.google.common.collect.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -152,17 +145,15 @@ class TarWriter implements Closeable { */ private final Map index = newLinkedHashMap(); - private final Set references = newHashSet(); - /** - * Segment graph of the entries that have already been written. + * List of binary references contained in this TAR file. */ - private final SortedMap> graph = newTreeMap(); + private final Map>> binaryReferences = newHashMap(); /** - * List of binary references contained in this TAR file. + * Graph of references between segments. */ - private final Map>> binaryReferences = newHashMap(); + private final Map> graph = newHashMap(); TarWriter(File file) { this(file, FileStoreMonitor.DEFAULT); @@ -257,27 +248,6 @@ class TarWriter implements Closeable { (int) (currentLength - size - padding), size, generation); index.put(uuid, entry); - if (isDataSegmentId(uuid.getLeastSignificantBits())) { - ByteBuffer segment = ByteBuffer.wrap(data, offset, size); - int pos = segment.position(); - int refcount = segment.get(pos + REF_COUNT_OFFSET) & 0xff; - if (refcount != 0) { - int refend = pos + 16 * (refcount + 1); - List list = Lists.newArrayListWithCapacity(refcount); - for (int refpos = pos + 16; refpos < refend; refpos += 16) { - UUID refid = new UUID( - segment.getLong(refpos), - segment.getLong(refpos + 8)); - if (!index.containsKey(refid)) { - references.add(refid); - } - list.add(refid); - } - Collections.sort(list); - graph.put(uuid, list); - } - } - monitor.written(currentLength - initialLength); return currentLength; } @@ -300,6 +270,17 @@ class TarWriter implements Closeable { references.add(reference); } + void addGraphEdge(UUID from, UUID to) { + Set adj = graph.get(from); + + if (adj == null) { + adj = newHashSet(); + graph.put(from, adj); + } + + adj.add(to); + } + /** * Flushes the entries that have so far been written to the disk. * This method is not synchronized to allow concurrent reads @@ -464,52 +445,73 @@ class TarWriter implements Closeable { } private void writeGraph() throws IOException { - List uuids = Lists.newArrayListWithCapacity( - index.size() + references.size()); - uuids.addAll(index.keySet()); - uuids.addAll(references); - Collections.sort(uuids); - - int graphSize = uuids.size() * 16 + 16; - for (List list : graph.values()) { - graphSize += 4 + list.size() * 4 + 4; - } - int padding = getPaddingSize(graphSize); + int graphSize = 0; - String graphName = file.getName() + ".gph"; - byte[] header = newEntryHeader(graphName, graphSize + padding); + // The following information are stored in the footer as meta- + // information about the entry. - ByteBuffer buffer = ByteBuffer.allocate(graphSize); + // 4 bytes to store a magic number identifying this entry as containing + // references to binary values. + graphSize += 4; - Map refmap = newHashMap(); + // 4 bytes to store the CRC32 checksum of the data in this entry. + graphSize += 4; - int index = 0; - for (UUID uuid : uuids) { - buffer.putLong(uuid.getMostSignificantBits()); - buffer.putLong(uuid.getLeastSignificantBits()); - refmap.put(uuid, index++); + // 4 bytes to store the length of this entry, without including the + // optional padding. + graphSize += 4; + + // 4 bytes to store the number of entries in the graph map. + graphSize += 4; + + // The following information are stored as part of the main content of + // this entry, after the optional padding. + + for (Entry> entry : graph.entrySet()) { + // 16 bytes to store the key of the map. + graphSize += 16; + + // 4 bytes for the number of entries in the adjacency list. + graphSize += 4; + + // 16 bytes for every element in the adjacency list. + graphSize += 16 * entry.getValue().size(); } - for (Map.Entry> entry : graph.entrySet()) { - buffer.putInt(refmap.get(entry.getKey())); - for (UUID refid : entry.getValue()) { - buffer.putInt(refmap.get(refid)); + ByteBuffer buffer = ByteBuffer.allocate(graphSize); + + for (Entry> entry : graph.entrySet()) { + UUID from = entry.getKey(); + + buffer.putLong(from.getMostSignificantBits()); + buffer.putLong(from.getLeastSignificantBits()); + + Set adj = entry.getValue(); + + buffer.putInt(adj.size()); + + for (UUID to : adj) { + buffer.putLong(to.getMostSignificantBits()); + buffer.putLong(to.getLeastSignificantBits()); } - buffer.putInt(-1); } CRC32 checksum = new CRC32(); checksum.update(buffer.array(), 0, buffer.position()); + buffer.putInt((int) checksum.getValue()); - buffer.putInt(uuids.size()); + buffer.putInt(graph.size()); buffer.putInt(graphSize); buffer.putInt(GRAPH_MAGIC); - access.write(header); + int padding = getPaddingSize(graphSize); + + access.write(newEntryHeader(file.getName() + ".gph", graphSize + padding)); + if (padding > 0) { - // padding comes *before* the graph! access.write(ZERO_BYTES, 0, padding); } + access.write(buffer.array()); } diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java index 42ade59..8648096 100644 --- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java @@ -125,7 +125,8 @@ public class CompactionAndCleanupIT { fileStore.flush(); long size2 = fileStore.getStats().getApproximateSize(); - assertSize("1st blob added", size2, size1 + blobSize, size1 + blobSize + (blobSize / 100)); + assertTrue("the store should grow", size2 > size1); + assertTrue("the store should grow of at least the size of the blob", size2 - size1 >= blobSize); // Now remove the property. No gc yet -> size doesn't shrink builder = nodeStore.getRoot().builder(); @@ -134,14 +135,13 @@ public class CompactionAndCleanupIT { fileStore.flush(); long size3 = fileStore.getStats().getApproximateSize(); - assertSize("1st blob removed", size3, size2, size2 + 4096); + assertTrue("the store should grow", size3 > size2); // 1st gc cycle -> no reclaimable garbage... fileStore.compact(); fileStore.cleanup(); long size4 = fileStore.getStats().getApproximateSize(); - assertSize("1st gc", size4, size3, size3 + size1); // Add another 5MB binary doubling the blob size builder = nodeStore.getRoot().builder(); @@ -150,21 +150,22 @@ public class CompactionAndCleanupIT { fileStore.flush(); long size5 = fileStore.getStats().getApproximateSize(); - assertSize("2nd blob added", size5, size4 + blobSize, size4 + blobSize + (blobSize / 100)); + assertTrue("the store should grow", size5 > size4); + assertTrue("the store should grow of at least the size of the blob", size5 - size4 >= blobSize); // 2st gc cycle -> 1st blob should get collected fileStore.compact(); fileStore.cleanup(); long size6 = fileStore.getStats().getApproximateSize(); - assertSize("2nd gc", size6, size5 - blobSize - size1, size5 - blobSize); + assertTrue("the store should shrink", size6 < size5); + assertTrue("the store should shrink of at least the size of the blob", size5 - size6 >= blobSize); // 3rtd gc cycle -> no significant change fileStore.compact(); fileStore.cleanup(); long size7 = fileStore.getStats().getApproximateSize(); - assertSize("3rd gc", size7, size6 * 10/11 , size6 * 10/9); // No data loss byte[] blob = ByteStreams.toByteArray(nodeStore.getRoot() @@ -176,8 +177,7 @@ public class CompactionAndCleanupIT { } @Test - public void offlineCompaction() - throws IOException, CommitFailedException { + public void offlineCompaction() throws IOException, CommitFailedException { SegmentGCOptions gcOptions = defaultGCOptions().setOffline(); ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); FileStore fileStore = fileStoreBuilder(getFileStoreFolder()) @@ -213,7 +213,8 @@ public class CompactionAndCleanupIT { fileStore.flush(); long size2 = fileStore.getStats().getApproximateSize(); - assertSize("1st blob added", size2, size1 + blobSize, size1 + blobSize + (blobSize / 100)); + assertTrue("the store should grow", size2 > size1); + assertTrue("the store should grow of at least the size of the blob", size2 - size1 > blobSize); // Now remove the property. No gc yet -> size doesn't shrink builder = nodeStore.getRoot().builder(); @@ -222,15 +223,15 @@ public class CompactionAndCleanupIT { fileStore.flush(); long size3 = fileStore.getStats().getApproximateSize(); - assertSize("1st blob removed", size3, size2, size2 + 4096); + assertTrue("the size should grow", size3 > size2); // 1st gc cycle -> 1st blob should get collected fileStore.compact(); fileStore.cleanup(); long size4 = fileStore.getStats().getApproximateSize(); - assertSize("1st gc", size4, size3 - blobSize - size1, size3 - - blobSize); + assertTrue("the store should shrink", size4 < size3); + assertTrue("the store should shrink of at least the size of the blob", size3 - size4 >= blobSize); // Add another 5MB binary builder = nodeStore.getRoot().builder(); @@ -239,21 +240,22 @@ public class CompactionAndCleanupIT { fileStore.flush(); long size5 = fileStore.getStats().getApproximateSize(); - assertSize("2nd blob added", size5, size4 + blobSize, size4 + blobSize + (blobSize / 100)); + assertTrue("the store should grow", size5 > size4); + assertTrue("the store should grow of at least the size of the blob", size5 - size4 > blobSize); // 2st gc cycle -> 2nd blob should *not* be collected fileStore.compact(); fileStore.cleanup(); long size6 = fileStore.getStats().getApproximateSize(); - assertSize("2nd gc", size6, size5 * 10/11, size5 * 10/9); + assertTrue("the blob should not be collected", Math.abs(size5 - size6) < blobSize); // 3rd gc cycle -> no significant change fileStore.compact(); fileStore.cleanup(); long size7 = fileStore.getStats().getApproximateSize(); - assertSize("3rd gc", size7, size6 * 10/11 , size6 * 10/9); + assertTrue("the blob should not be collected", Math.abs(size6 - size7) < blobSize); // No data loss byte[] blob = ByteStreams.toByteArray(nodeStore.getRoot() diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/RecordUsageAnalyserTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/RecordUsageAnalyserTest.java deleted file mode 100644 index 12a04c3..0000000 --- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/RecordUsageAnalyserTest.java +++ /dev/null @@ -1,320 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.jackrabbit.oak.segment; - -import static com.google.common.base.Strings.repeat; -import static java.util.Collections.nCopies; -import static org.apache.jackrabbit.oak.api.Type.LONGS; -import static org.apache.jackrabbit.oak.api.Type.NAME; -import static org.apache.jackrabbit.oak.api.Type.NAMES; -import static org.apache.jackrabbit.oak.api.Type.STRINGS; -import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; -import static org.apache.jackrabbit.oak.segment.ListRecord.LEVEL_SIZE; -import static org.apache.jackrabbit.oak.segment.Segment.MEDIUM_LIMIT; -import static org.apache.jackrabbit.oak.segment.Segment.SMALL_LIMIT; -import static org.apache.jackrabbit.oak.segment.SegmentWriterBuilder.segmentWriterBuilder; -import static org.junit.Assert.assertEquals; - -import java.io.IOException; -import java.util.Random; - -import com.google.common.collect.ImmutableList; -import org.apache.jackrabbit.oak.api.Blob; -import org.apache.jackrabbit.oak.plugins.memory.ArrayBasedBlob; -import org.apache.jackrabbit.oak.segment.memory.MemoryStore; -import org.apache.jackrabbit.oak.spi.state.NodeBuilder; -import org.junit.Before; -import org.junit.Test; - -public class RecordUsageAnalyserTest { - private SegmentWriter writer; - private RecordUsageAnalyser analyser; - - @Before - public void setup() throws IOException { - MemoryStore store = new MemoryStore(); - writer = segmentWriterBuilder("").build(store); - analyser = new RecordUsageAnalyser(store.getReader()); - } - - @Test - public void emptyNode() throws IOException { - SegmentNodeState node = writer.writeNode(EMPTY_NODE); - analyser.analyseNode(node.getRecordId()); - assertSizes(analyser, 0, 0, 0, 4, 3); - } - - @Test - public void nodeWithInt() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - builder.setProperty("one", 1); - - SegmentNodeState node = writer.writeNode(builder.getNodeState()); - analyser.analyseNode(node.getRecordId()); - assertSizes(analyser, 0, 0, 6, 8, 6); - } - - @Test - public void nodeWithString() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - builder.setProperty("two", "222"); - - SegmentNodeState node = writer.writeNode(builder.getNodeState()); - analyser.analyseNode(node.getRecordId()); - assertSizes(analyser, 0, 0, 8, 8, 6); - } - - @Test - public void nodeWithMultipleProperties() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - builder.setProperty("one", "11"); - builder.setProperty("two", "22"); - builder.setProperty("three", "33"); - - SegmentNodeState node = writer.writeNode(builder.getNodeState()); - analyser.analyseNode(node.getRecordId()); - assertSizes(analyser, 0, 18, 23, 10, 6); - } - - @Test - public void nodeWithMediumString() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - builder.setProperty("medium", repeat("a", SMALL_LIMIT + 1)); - - SegmentNodeState node = writer.writeNode(builder.getNodeState()); - analyser.analyseNode(node.getRecordId()); - assertSizes(analyser, 0, 0, 138, 8, 6); - } - - @Test - public void nodeWithLargeString() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - builder.setProperty("large", repeat("b", MEDIUM_LIMIT + 1)); - - SegmentNodeState node = writer.writeNode(builder.getNodeState()); - analyser.analyseNode(node.getRecordId()); - assertSizes(analyser, 0, 15, 16530, 8, 6); - } - - @Test - public void nodeWithSameString() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - builder.setProperty("two", "two"); - - SegmentNodeState node = writer.writeNode(builder.getNodeState()); - analyser.analyseNode(node.getRecordId()); - assertSizes(analyser, 0, 0, 4, 8, 6); - } - - @Test - public void nodeWithInts() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - builder.setProperty("multi", ImmutableList.of(1L, 2L, 3L, 4L), LONGS); - - SegmentNodeState node = writer.writeNode(builder.getNodeState()); - analyser.analyseNode(node.getRecordId()); - assertSizes(analyser, 0, 12, 21, 8, 6); - } - - @Test - public void nodeWithManyInts() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - builder.setProperty("multi", nCopies(LEVEL_SIZE + 1, 1L), LONGS); - - SegmentNodeState node = writer.writeNode(builder.getNodeState()); - analyser.analyseNode(node.getRecordId()); - assertSizes(analyser, 0, 771, 15, 8, 6); - } - - @Test - public void nodeWithManyIntsAndOne() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - builder.setProperty("multi", nCopies(LEVEL_SIZE + 2, 1L), LONGS); - - SegmentNodeState node = writer.writeNode(builder.getNodeState()); - analyser.analyseNode(node.getRecordId()); - assertSizes(analyser, 0, 777, 15, 8, 6); - } - - @Test - public void nodeWithStrings() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - builder.setProperty("multi", ImmutableList.of("one", "one", "two", "two", "three"), STRINGS); - - SegmentNodeState node = writer.writeNode(builder.getNodeState()); - analyser.analyseNode(node.getRecordId()); - assertSizes(analyser, 0, 15, 27, 8, 6); - } - - @Test - public void nodeWithBlob() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - builder.setProperty("blob", createRandomBlob(4)); - - SegmentNodeState node = writer.writeNode(builder.getNodeState()); - analyser.analyseNode(node.getRecordId()); - assertSizes(analyser, 0, 0, 10, 8, 6); - } - - @Test - public void nodeWithMediumBlob() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - builder.setProperty("mediumBlob", createRandomBlob(SMALL_LIMIT + 1)); - - SegmentNodeState node = writer.writeNode(builder.getNodeState()); - analyser.analyseNode(node.getRecordId()); - assertSizes(analyser, 0, 0, 142, 8, 6); - } - - @Test - public void nodeWithLargeBlob() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - builder.setProperty("largeBlob", createRandomBlob(MEDIUM_LIMIT + 1)); - - SegmentNodeState node = writer.writeNode(builder.getNodeState()); - analyser.analyseNode(node.getRecordId()); - assertSizes(analyser, 0, 15, 16534, 8, 6); - } - - @Test - public void nodeWithPrimaryType() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - builder.setProperty("jcr:primaryType", "type", NAME); - - SegmentNodeState node = writer.writeNode(builder.getNodeState()); - analyser.analyseNode(node.getRecordId()); - assertSizes(analyser, 0, 0, 5, 7, 3); - } - - @Test - public void nodeWithMixinTypes() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - builder.setProperty("jcr:mixinTypes", ImmutableList.of("type1", "type2"), NAMES); - - SegmentNodeState node = writer.writeNode(builder.getNodeState()); - analyser.analyseNode(node.getRecordId()); - assertSizes(analyser, 0, 0, 12, 10, 3); - } - - @Test - public void singleChild() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - builder.setChildNode("child"); - - SegmentNodeState node = writer.writeNode(builder.getNodeState()); - analyser.analyseNode(node.getRecordId()); - assertSizes(analyser, 0, 0, 6, 11, 9); - } - - @Test - public void multiChild() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - builder.setChildNode("child1"); - builder.setChildNode("child2"); - - SegmentNodeState node = writer.writeNode(builder.getNodeState()); - analyser.analyseNode(node.getRecordId()); - assertSizes(analyser, 24, 0, 14, 8, 12); - } - - @Test - public void manyChild() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - for (int k = 0; k < MapRecord.BUCKETS_PER_LEVEL + 1; k++) { - builder.setChildNode("child" + k); - } - - SegmentNodeState node = writer.writeNode(builder.getNodeState()); - analyser.analyseNode(node.getRecordId()); - assertSizes(analyser, 457, 0, 254, 8, 105); - } - - @Test - public void changedChild() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - builder.setChildNode("child1"); - builder.setChildNode("child2"); - - SegmentNodeState node = writer.writeNode(builder.getNodeState()); - analyser.analyseNode(node.getRecordId()); - assertSizes(analyser, 24, 0, 14, 8, 12); - - builder = node.builder(); - builder.child("child1").setProperty("p", "q"); - - node = (SegmentNodeState) builder.getNodeState(); - - analyser.analyseNode(node.getRecordId()); - assertSizes(analyser, 41, 0, 18, 16, 24); - } - - @Test - public void counts() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - builder.setChildNode("child1"); - builder.setChildNode("child2"); - builder.setProperty("prop", ImmutableList.of("a", "b"), STRINGS); - builder.setProperty("mediumString", repeat("m", SMALL_LIMIT)); - builder.setProperty("longString", repeat("l", MEDIUM_LIMIT)); - builder.setProperty("smallBlob", createRandomBlob(4)); - builder.setProperty("mediumBlob", createRandomBlob(SMALL_LIMIT)); - builder.setProperty("longBlob", createRandomBlob(MEDIUM_LIMIT)); - - SegmentNodeState node = writer.writeNode(builder.getNodeState()); - analyser.analyseNode(node.getRecordId()); - assertCounts(analyser, 1, 5, 6, 1, 1, 1, 0, 10, 1, 1, 2, 3); - } - - private static Blob createRandomBlob(int size) { - byte[] bytes = new byte[size]; - new Random().nextBytes(bytes); - return new ArrayBasedBlob(bytes); - } - - private static void assertSizes(RecordUsageAnalyser analyser, - long maps, long lists, long values, long templates, long nodes) { - assertEquals("maps sizes mismatch", maps, analyser.getMapSize()); - assertEquals("lists sizes mismatch", lists, analyser.getListSize()); - assertEquals("value sizes mismatch", values, analyser.getValueSize()); - assertEquals("template sizes mismatch", templates, analyser.getTemplateSize()); - assertEquals("nodes sizes mismatch", nodes, analyser.getNodeSize()); - } - - private static void assertCounts(RecordUsageAnalyser analyser, - long mapCount, long listCount, long propertyCount, - long smallBlobCount, long mediumBlobCount, long longBlobCount, long externalBlobCount, - long smallStringCount, long mediumStringCount, long longStringCount, - long templateCount, long nodeCount) { - assertEquals("map count mismatch", mapCount, analyser.getMapCount()); - assertEquals("list count mismatch", listCount, analyser.getListCount()); - assertEquals("property count mismatch", propertyCount, analyser.getPropertyCount()); - assertEquals("small blob count mismatch", smallBlobCount, analyser.getSmallBlobCount()); - assertEquals("medium blob mismatch", mediumBlobCount, analyser.getMediumBlobCount()); - assertEquals("long blob count mismatch", longBlobCount, analyser.getLongBlobCount()); - assertEquals("external blob count mismatch", externalBlobCount, analyser.getExternalBlobCount()); - assertEquals("small string count mismatch", smallStringCount, analyser.getSmallStringCount()); - assertEquals("medium string count mismatch", mediumStringCount, analyser.getMediumStringCount()); - assertEquals("long string count mismatch", longStringCount, analyser.getLongStringCount()); - assertEquals("template count mismatch", templateCount, analyser.getTemplateCount()); - assertEquals("node count mismatch", nodeCount, analyser.getNodeCount()); - - } - -} diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentIdFactoryTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentIdFactoryTest.java index c536f8c..2a9d44b 100644 --- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentIdFactoryTest.java +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentIdFactoryTest.java @@ -92,31 +92,4 @@ public class SegmentIdFactoryTest { assertTrue(ids.contains(b)); } - /** - * OAK-2049 - error for data segments - */ - @Test(expected = IllegalStateException.class) - public void dataAIOOBE() throws IOException { - MemoryStore store = new MemoryStore(); - Segment segment = store.getRevisions().getHead().getSegment(); - byte[] buffer = new byte[segment.size()]; - segment.readBytes(Segment.MAX_SEGMENT_SIZE - segment.size(), buffer, 0, segment.size()); - - SegmentId id = store.newDataSegmentId(); - ByteBuffer data = ByteBuffer.wrap(buffer); - Segment s = new Segment(store, store.getReader(), id, data); - s.getRefId(1); - } - - /** - * OAK-2049 - error for bulk segments - */ - @Test(expected = IllegalStateException.class) - public void bulkAIOOBE() { - SegmentId id = store.newBulkSegmentId(); - ByteBuffer data = ByteBuffer.allocate(4); - Segment s = new Segment(store, store.getReader(), id, data); - s.getRefId(1); - } - } diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentParserTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentParserTest.java index 2153912..51052d6 100644 --- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentParserTest.java +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentParserTest.java @@ -56,6 +56,7 @@ import org.apache.jackrabbit.oak.segment.SegmentParser.ValueInfo; import org.apache.jackrabbit.oak.segment.memory.MemoryStore; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; public class SegmentParserTest { @@ -150,7 +151,6 @@ public class SegmentParserTest { assertEquals(node.getRecordId(), info.nodeId); assertEquals(0, info.nodeCount); assertEquals(0, info.propertyCount); - assertEquals(3, info.size); assertEquals(info.nodeId.toString10(), info.stableId); } @@ -166,7 +166,6 @@ public class SegmentParserTest { assertEquals(node.getRecordId(), info.nodeId); assertEquals(1, info.nodeCount); assertEquals(0, info.propertyCount); - assertEquals(6, info.size); assertEquals(info.nodeId.toString10(), info.stableId); } @@ -187,7 +186,6 @@ public class SegmentParserTest { assertEquals(node.getRecordId(), info.nodeId); assertEquals(2, info.nodeCount); assertEquals(1, info.propertyCount); - assertEquals(9, info.size); assertEquals(info.nodeId.toString10(), info.stableId); } @@ -210,7 +208,6 @@ public class SegmentParserTest { assertFalse(info.manyChildNodes); assertEquals(2, info.mixinCount); assertEquals(1, info.propertyCount); - assertEquals(20, info.size); } @Override protected void onString(RecordId parentId, RecordId stringId) { } @Override protected void onNode(RecordId parentId, RecordId nodeId) { } @@ -227,7 +224,6 @@ public class SegmentParserTest { @Override protected void onMapLeaf(RecordId parentId, RecordId mapId, MapRecord map) { } }.parseMap(null, map.getRecordId(), map); assertEquals(map.getRecordId(), mapInfo.mapId); - assertEquals(-1, mapInfo.size); } @Test @@ -235,37 +231,30 @@ public class SegmentParserTest { Random rnd = new Random(); MapRecord base = writer.writeMap(null, createMap(33, rnd)); MapRecord map = writer.writeMap(base, createMap(1, rnd)); - final AtomicInteger size = new AtomicInteger(); MapInfo mapInfo = new TestParser(store.getReader(), "nonEmptyMap") { @Override protected void onMapDiff(RecordId parentId, RecordId mapId, MapRecord map) { MapInfo mapInfo = parseMapDiff(mapId, map); assertEquals(mapId, mapInfo.mapId); - size.addAndGet(mapInfo.size); } @Override protected void onMap(RecordId parentId, RecordId mapId, MapRecord map) { MapInfo mapInfo = parseMap(parentId, mapId, map); assertEquals(mapId, mapInfo.mapId); - size.addAndGet(mapInfo.size); } @Override protected void onMapBranch(RecordId parentId, RecordId mapId, MapRecord map) { MapInfo mapInfo = parseMapBranch(mapId, map); assertEquals(mapId, mapInfo.mapId); - size.addAndGet(mapInfo.size); } @Override protected void onMapLeaf(RecordId parentId, RecordId mapId, MapRecord map) { MapInfo mapInfo = parseMapLeaf(mapId, map); assertEquals(mapId, mapInfo.mapId); - size.addAndGet(mapInfo.size); } @Override protected void onString(RecordId parentId, RecordId stringId) { } }.parseMap(null, map.getRecordId(), map); assertEquals(map.getRecordId(), mapInfo.mapId); - assertEquals(-1, mapInfo.size); - assertEquals(456, size.get()); } private Map createMap(int size, Random rnd) throws IOException { @@ -287,7 +276,6 @@ public class SegmentParserTest { PropertyInfo propertyInfo = parseProperty(parentId, propertyId, template); assertEquals(propertyId, propertyInfo.propertyId); assertEquals(-1, propertyInfo.count); - assertEquals(0, propertyInfo.size); } @Override protected void onTemplate(RecordId parentId, RecordId templateId) { } @Override protected void onValue(RecordId parentId, RecordId valueId, Type type) { } @@ -306,7 +294,6 @@ public class SegmentParserTest { PropertyInfo propertyInfo = parseProperty(parentId, propertyId, template); assertEquals(propertyId, propertyInfo.propertyId); assertEquals(4, propertyInfo.count); - assertEquals(7, propertyInfo.size); } @Override protected void onTemplate(RecordId parentId, RecordId templateId) { } @Override protected void onValue(RecordId parentId, RecordId valueId, Type type) { } @@ -323,7 +310,6 @@ public class SegmentParserTest { BlobInfo blobInfo = parseBlob(blobId); assertEquals(blobId, blobInfo.blobId); assertEquals(SMALL, blobInfo.blobType); - assertEquals(5, blobInfo.size); } }.parseValue(null, blob.getRecordId(), BINARY); assertEquals(blob.getRecordId(), valueInfo.valueId); @@ -339,7 +325,6 @@ public class SegmentParserTest { BlobInfo blobInfo = parseBlob(blobId); assertEquals(blobId, blobInfo.blobId); assertEquals(MEDIUM, blobInfo.blobType); - assertEquals(SMALL_LIMIT + 2, blobInfo.size); } }.parseValue(null, blob.getRecordId(), BINARY); assertEquals(blob.getRecordId(), valueInfo.valueId); @@ -355,7 +340,6 @@ public class SegmentParserTest { BlobInfo blobInfo = parseBlob(blobId); assertEquals(blobId, blobInfo.blobId); assertEquals(LONG, blobInfo.blobType); - assertEquals(MEDIUM_LIMIT + 11, blobInfo.size); } @Override protected void onList(RecordId parentId, RecordId listId, int count) { } }.parseValue(null, blob.getRecordId(), BINARY); @@ -375,7 +359,6 @@ public class SegmentParserTest { BlobInfo blobInfo = new TestParser(store.getReader(), "shortString").parseString(stringId); assertEquals(stringId, blobInfo.blobId); assertEquals(SMALL, blobInfo.blobType); - assertEquals(6, blobInfo.size); } @Test @@ -384,7 +367,6 @@ public class SegmentParserTest { BlobInfo blobInfo = new TestParser(store.getReader(), "mediumString").parseString(stringId); assertEquals(stringId, blobInfo.blobId); assertEquals(MEDIUM, blobInfo.blobType); - assertEquals(SMALL_LIMIT + 2, blobInfo.size); } @Test @@ -395,7 +377,6 @@ public class SegmentParserTest { }.parseString(stringId); assertEquals(stringId, blobInfo.blobId); assertEquals(LONG, blobInfo.blobType); - assertEquals(MEDIUM_LIMIT + 11, blobInfo.size); } @Test @@ -404,7 +385,6 @@ public class SegmentParserTest { ListInfo listInfo = new TestParser(store.getReader(), "emptyList").parseList(null, listId, 0); assertEquals(listId, listInfo.listId); assertEquals(0, listInfo.count); - assertEquals(0, listInfo.size); } @Test @@ -424,7 +404,6 @@ public class SegmentParserTest { }.parseList(null, listId, count); assertEquals(listId, listInfo.listId); assertEquals(count, listInfo.count); - assertEquals(301185, listInfo.size); } } diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentReferencesTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentReferencesTest.java new file mode 100644 index 0000000..12c26f8 --- /dev/null +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentReferencesTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jackrabbit.oak.segment; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +import java.io.File; +import java.util.Arrays; + +import org.apache.jackrabbit.oak.segment.file.FileStore; +import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class SegmentReferencesTest { + + @Rule + public TemporaryFolder folder = new TemporaryFolder(new File("target")); + + private FileStore newFileStore() throws Exception { + return FileStoreBuilder.fileStoreBuilder(folder.getRoot()).build(); + } + + @Test + public void segmentShouldNotReferenceItself() throws Exception { + try (FileStore store = newFileStore()) { + + // Write two records, one referencing the other. + + SegmentWriter writer = SegmentWriterBuilder.segmentWriterBuilder("test").build(store); + RecordId stringId = writer.writeString("test"); + RecordId listId = writer.writeList(Arrays.asList(stringId, stringId)); + writer.flush(); + + // The two records should be living in the same segment. + + assertEquals(listId.getSegmentId(), stringId.getSegmentId()); + + // This inter-segment reference shouldn't generate a reference from + // this segment to itself. + + assertEquals(0, listId.getSegment().getReferencedSegmentIdCount()); + } + } + + @Test + public void segmentShouldExposeReferencedSegments() throws Exception { + try (FileStore store = newFileStore()) { + + // Write two records, one referencing the other. + + SegmentWriter writer = SegmentWriterBuilder.segmentWriterBuilder("test").build(store); + + RecordId stringId = writer.writeString("test"); + writer.flush(); + + RecordId listId = writer.writeList(Arrays.asList(stringId, stringId)); + writer.flush(); + + // The two records should be living in two different segments. + + assertNotEquals(listId.getSegmentId(), stringId.getSegmentId()); + + // This intra-segment reference should generate a reference from the + // segment containing the list to the segment containing the string. + + assertEquals(1, listId.getSegment().getReferencedSegmentIdCount()); + assertEquals(stringId.getSegmentId().asUUID(), listId.getSegment().getReferencedSegmentId(0)); + } + } + +} diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentSizeTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentSizeTest.java deleted file mode 100644 index 111cf6d..0000000 --- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentSizeTest.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.jackrabbit.oak.segment; - -import static junit.framework.Assert.assertEquals; -import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; -import static org.apache.jackrabbit.oak.segment.SegmentWriterBuilder.segmentWriterBuilder; -import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder; - -import java.io.File; -import java.io.IOException; -import java.util.Calendar; -import java.util.Collections; - -import com.google.common.collect.ImmutableList; -import org.apache.jackrabbit.oak.api.Type; -import org.apache.jackrabbit.oak.plugins.memory.PropertyStates; -import org.apache.jackrabbit.oak.segment.file.FileStore; -import org.apache.jackrabbit.oak.segment.memory.MemoryStore; -import org.apache.jackrabbit.oak.spi.state.NodeBuilder; -import org.apache.jackrabbit.oak.spi.state.NodeState; -import org.apache.jackrabbit.util.ISO8601; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - - -/** - * Test case for ensuring that segment size remains within bounds. - */ -public class SegmentSizeTest { - @Rule - public TemporaryFolder folder = new TemporaryFolder(new File("target")); - - private FileStore store; - - @Before - public void setup() throws IOException { - store = fileStoreBuilder(folder.getRoot()).build(); - } - - @After - public void tearDown() { - store.close(); - } - - @Test - public void testNodeSize() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - expectSize(80, builder); - expectAmortizedSize(8, builder); - - builder = EMPTY_NODE.builder(); - builder.setProperty("foo", "bar"); - expectSize(96, builder); - expectAmortizedSize(12, builder); - - builder = EMPTY_NODE.builder(); - builder.setProperty("foo", "bar"); - builder.setProperty("baz", 123); - expectSize(128, builder); - expectAmortizedSize(20, builder); - - builder = EMPTY_NODE.builder(); - builder.child("foo"); - expectSize(112, builder); - expectAmortizedSize(20, builder); - - builder = EMPTY_NODE.builder(); - builder.child("foo"); - builder.child("bar"); - expectSize(144, builder); - expectAmortizedSize(52, builder); - } - - @Test - public void testDuplicateStrings() throws IOException { - String string = "More than just a few bytes of example content."; - - SegmentWriter writer = new MemoryStore().getWriter(); - SegmentNodeBuilder builder = writer.writeNode(EMPTY_NODE).builder(); - - builder.setProperty(PropertyStates.createProperty( - "test", Collections.nCopies(1, string), Type.STRINGS)); - RecordId id1 = builder.getNodeState().getRecordId(); - - builder.setProperty(PropertyStates.createProperty( - "test", Collections.nCopies(12, string), Type.STRINGS)); - RecordId id2 = builder.getNodeState().getRecordId(); - assertEquals(20 + 12 * Segment.RECORD_ID_BYTES, - id1.getOffset() - id2.getOffset()); - - builder.setProperty(PropertyStates.createProperty( - "test", Collections.nCopies(100, string), Type.STRINGS)); - RecordId id3 = builder.getNodeState().getRecordId(); - assertEquals(20 + 100 * Segment.RECORD_ID_BYTES, - id2.getOffset() - id3.getOffset()); - } - - @Test - public void testDuplicateDates() throws IOException { - String now = ISO8601.format(Calendar.getInstance()); - - SegmentWriter writer = new MemoryStore().getWriter(); - SegmentNodeBuilder builder = writer.writeNode(EMPTY_NODE).builder(); - - builder.setProperty(PropertyStates.createProperty( - "test", Collections.nCopies(1, now), Type.DATES)); - RecordId id1 = builder.getNodeState().getRecordId(); - - builder.setProperty(PropertyStates.createProperty( - "test", Collections.nCopies(12, now), Type.DATES)); - RecordId id2 = builder.getNodeState().getRecordId(); - assertEquals(20 + 12 * Segment.RECORD_ID_BYTES, - id1.getOffset() - id2.getOffset()); - - builder.setProperty(PropertyStates.createProperty( - "test", Collections.nCopies(100, now), Type.DATES)); - RecordId id3 = builder.getNodeState().getRecordId(); - assertEquals(20 + 100 * Segment.RECORD_ID_BYTES, - id2.getOffset() - id3.getOffset()); - } - - @Test - public void testAccessControlNodes() throws IOException { - NodeBuilder builder = EMPTY_NODE.builder(); - builder.setProperty("jcr:primaryType", "rep:ACL", Type.NAME); - expectSize(96, builder); - expectAmortizedSize(8, builder); - - NodeBuilder deny = builder.child("deny"); - deny.setProperty("jcr:primaryType", "rep:DenyACE", Type.NAME); - deny.setProperty("rep:principalName", "everyone"); - deny.setProperty(PropertyStates.createProperty( - "rep:privileges", ImmutableList.of("jcr:read"), Type.NAMES)); - expectSize(240, builder); - expectAmortizedSize(40, builder); - - NodeBuilder allow = builder.child("allow"); - allow.setProperty("jcr:primaryType", "rep:GrantACE"); - allow.setProperty("rep:principalName", "administrators"); - allow.setProperty(PropertyStates.createProperty( - "rep:privileges", ImmutableList.of("jcr:all"), Type.NAMES)); - expectSize(368, builder); - expectAmortizedSize(96, builder); - - NodeBuilder deny0 = builder.child("deny0"); - deny0.setProperty("jcr:primaryType", "rep:DenyACE", Type.NAME); - deny0.setProperty("rep:principalName", "everyone"); - deny0.setProperty("rep:glob", "*/activities/*"); - builder.setProperty(PropertyStates.createProperty( - "rep:privileges", ImmutableList.of("jcr:read"), Type.NAMES)); - expectSize(480, builder); - expectAmortizedSize(136, builder); - - NodeBuilder allow0 = builder.child("allow0"); - allow0.setProperty("jcr:primaryType", "rep:GrantACE"); - allow0.setProperty("rep:principalName", "user-administrators"); - allow0.setProperty(PropertyStates.createProperty( - "rep:privileges", ImmutableList.of("jcr:all"), Type.NAMES)); - expectSize(544, builder); - expectAmortizedSize(176, builder); - } - - @Test - public void testFlatNodeUpdate() throws IOException { - MemoryStore store = new MemoryStore(); - SegmentWriter writer = store.getWriter(); - - NodeBuilder builder = EMPTY_NODE.builder(); - for (int i = 0; i < 1000; i++) { - builder.child("child" + i); - } - - SegmentNodeState state = writer.writeNode(builder.getNodeState()); - writer.flush(); - Segment segment = store.readSegment(state.getRecordId().getSegmentId()); - assertEquals(31584, segment.size()); - - writer.flush(); // force flushing of the previous segment - - builder = state.builder(); - builder.child("child1000"); - state = writer.writeNode(builder.getNodeState()); - writer.flush(); - segment = store.readSegment(state.getRecordId().getSegmentId()); - assertEquals(560, segment.size()); - } - - private void expectSize(int expectedSize, NodeBuilder builder) throws IOException { - SegmentWriter writer = segmentWriterBuilder("test").build(store); - RecordId id = writer.writeNode(builder.getNodeState()).getRecordId(); - writer.flush(); - Segment segment = id.getSegment(); - assertEquals("Unexpected size of segment " + id + " info=" + segment.getSegmentInfo(), - expectedSize, segment.size()); - } - - private void expectAmortizedSize(int expectedSize, NodeBuilder builder) throws IOException { - SegmentWriter writer = segmentWriterBuilder("test").build(store); - NodeState state = builder.getNodeState(); - RecordId id1 = writer.writeNode(state).getRecordId(); - RecordId id2 = writer.writeNode(state).getRecordId(); - assertEquals(expectedSize, id1.getOffset() - id2.getOffset()); - } - -} diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarFileTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarFileTest.java index fd5a779..b6eb2ad 100644 --- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarFileTest.java +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarFileTest.java @@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.segment.file; import static com.google.common.base.Charsets.UTF_8; +import static com.google.common.collect.Lists.newArrayList; import static com.google.common.collect.Maps.newHashMap; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -28,6 +29,7 @@ import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -164,4 +166,34 @@ public class TarFileTest { } } + @Test + public void graphShouldBeTrimmedDownOnSweep() throws Exception { + try (TarWriter writer = new TarWriter(file)) { + writer.writeEntry(1, 1, new byte[] {1}, 0, 1, 1); + writer.writeEntry(1, 2, new byte[] {1}, 0, 1, 1); + writer.writeEntry(1, 3, new byte[] {1}, 0, 1, 1); + writer.writeEntry(2, 1, new byte[] {1}, 0, 1, 2); + writer.writeEntry(2, 2, new byte[] {1}, 0, 1, 2); + writer.writeEntry(2, 3, new byte[] {1}, 0, 1, 2); + + writer.addGraphEdge(new UUID(1, 1), new UUID(1, 2)); + writer.addGraphEdge(new UUID(1, 2), new UUID(1, 3)); + writer.addGraphEdge(new UUID(2, 1), new UUID(2, 2)); + writer.addGraphEdge(new UUID(2, 2), new UUID(2, 3)); + } + + Set sweep = newSet(new UUID(1, 2), new UUID(2, 3)); + + try (TarReader reader = TarReader.open(file, false)) { + try (TarReader swept = reader.sweep(sweep, new HashSet())) { + assertNotNull(swept); + + Map> graph = newHashMap(); + graph.put(new UUID(2, 1), newArrayList(new UUID(2, 2))); + + assertEquals(graph, swept.getGraph(false)); + } + } + } + }