diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BlockRecord.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BlockRecord.java index 5ded313..22731ae 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BlockRecord.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BlockRecord.java @@ -54,7 +54,7 @@ class BlockRecord extends Record { length = size - position; } if (length > 0) { - getSegment().readBytes(getOffset(position), buffer, offset, length); + getSegment().readBytes(getRecordNumber(), position, buffer, offset, length); } return length; } diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/CachingSegmentReader.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/CachingSegmentReader.java index a23fd89..08268f3 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/CachingSegmentReader.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/CachingSegmentReader.java @@ -87,7 +87,7 @@ public class CachingSegmentReader implements SegmentReader { final SegmentId segmentId = id.getSegmentId(); long msb = segmentId.getMostSignificantBits(); long lsb = segmentId.getLeastSignificantBits(); - return stringCache.get(msb, lsb, id.getOffset(), new Function() { + return stringCache.get(msb, lsb, id.getRecordNumber(), new Function() { @Nonnull @Override public String apply(Integer offset) { @@ -111,7 +111,7 @@ public class CachingSegmentReader implements SegmentReader { final SegmentId segmentId = id.getSegmentId(); long msb = segmentId.getMostSignificantBits(); long lsb = segmentId.getLeastSignificantBits(); - return templateCache.get(msb, lsb, id.getOffset(), new Function() { + return templateCache.get(msb, lsb, id.getRecordNumber(), new Function() { @Nonnull @Override public Template apply(Integer offset) { diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/IdentityRecordNumbers.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/IdentityRecordNumbers.java new file mode 100644 index 0000000..2134dbe --- /dev/null +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/IdentityRecordNumbers.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jackrabbit.oak.segment; + +import java.util.Iterator; + +/** + * An implementation of a record number to offset table that assumes that a + * record number is also a valid offset in the segment. This implementation is + * useful when an instance of a table has still to be provided, but record + * numbers have no logical semantics (e.g. for bulk segments). + *

+ * This implementation is trivially thread-safe. + */ +class IdentityRecordNumbers implements RecordNumbers { + + @Override + public int getOffset(int recordNumber) { + return recordNumber; + } + + @Override + public Iterator iterator() { + throw new UnsupportedOperationException("invalid usage of the record-number-to-offset table"); + } + +} diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ImmutableRecordNumbers.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ImmutableRecordNumbers.java new file mode 100644 index 0000000..8f332cc --- /dev/null +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ImmutableRecordNumbers.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jackrabbit.oak.segment; + +import java.util.Iterator; +import java.util.Map; + +import com.google.common.collect.Maps; + +/** + * An immutable record number to offset table. It is initialized at construction + * time and can never be changed afterwards. + *

+ * This implementation is trivially thread-safe. + */ +class ImmutableRecordNumbers implements RecordNumbers { + + private final Map recordNumbers; + + /** + * Create a new immutable record number to offset table. + * + * @param recordNumbers a map of record numbers to offsets. It can't be + * {@code null}. + */ + public ImmutableRecordNumbers(Map recordNumbers) { + this.recordNumbers = Maps.newHashMap(recordNumbers); + } + + @Override + public int getOffset(int recordNumber) { + Integer offset = recordNumbers.get(recordNumber); + + if (offset == null) { + return -1; + } + + return offset; + } + + @Override + public Iterator iterator() { + return new RecordNumbersIterator(recordNumbers.entrySet().iterator()); + } + +} diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ListRecord.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ListRecord.java index 5d5f3d1..a15fb73 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ListRecord.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ListRecord.java @@ -62,7 +62,7 @@ class ListRecord extends Record { int bucketIndex = index / bucketSize; int bucketOffset = index % bucketSize; Segment segment = getSegment(); - RecordId id = segment.readRecordId(getOffset(0, bucketIndex)); + RecordId id = segment.readRecordId(getRecordNumber(), 0, bucketIndex); ListRecord bucket = new ListRecord( id, Math.min(bucketSize, size - bucketIndex * bucketSize)); return bucket.getEntry(bucketOffset); @@ -95,13 +95,13 @@ class ListRecord extends Record { ids.add(getRecordId()); } else if (bucketSize == 1) { for (int i = 0; i < count; i++) { - ids.add(segment.readRecordId(getOffset(0, index + i))); + ids.add(segment.readRecordId(getRecordNumber(), 0, index + i)); } } else { while (count > 0) { int bucketIndex = index / bucketSize; int bucketOffset = index % bucketSize; - RecordId id = segment.readRecordId(getOffset(0, bucketIndex)); + RecordId id = segment.readRecordId(getRecordNumber(), 0, bucketIndex); ListRecord bucket = new ListRecord( id, Math.min(bucketSize, size - bucketIndex * bucketSize)); int n = Math.min(bucket.size() - bucketOffset, count); diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/MapRecord.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/MapRecord.java index 10db6fc..56917e0 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/MapRecord.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/MapRecord.java @@ -104,26 +104,26 @@ public class MapRecord extends Record { boolean isLeaf() { Segment segment = getSegment(); - int head = segment.readInt(getOffset(0)); + int head = segment.readInt(getRecordNumber()); if (isDiff(head)) { - RecordId base = segment.readRecordId(getOffset(8, 2)); + RecordId base = segment.readRecordId(getRecordNumber(), 8, 2); return reader.readMap(base).isLeaf(); } return !isBranch(head); } public boolean isDiff() { - return isDiff(getSegment().readInt(getOffset(0))); + return isDiff(getSegment().readInt(getRecordNumber())); } MapRecord[] getBuckets() { Segment segment = getSegment(); MapRecord[] buckets = new MapRecord[BUCKETS_PER_LEVEL]; - int bitmap = segment.readInt(getOffset(4)); + int bitmap = segment.readInt(getRecordNumber(), 4); int ids = 0; for (int i = 0; i < BUCKETS_PER_LEVEL; i++) { if ((bitmap & (1 << i)) != 0) { - buckets[i] = reader.readMap(segment.readRecordId(getOffset(8, ids++))); + buckets[i] = reader.readMap(segment.readRecordId(getRecordNumber(), 8, ids++)); } else { buckets[i] = null; } @@ -133,11 +133,11 @@ public class MapRecord extends Record { private List getBucketList(Segment segment) { List buckets = newArrayListWithCapacity(BUCKETS_PER_LEVEL); - int bitmap = segment.readInt(getOffset(4)); + int bitmap = segment.readInt(getRecordNumber(), 4); int ids = 0; for (int i = 0; i < BUCKETS_PER_LEVEL; i++) { if ((bitmap & (1 << i)) != 0) { - RecordId id = segment.readRecordId(getOffset(8, ids++)); + RecordId id = segment.readRecordId(getRecordNumber(), 8, ids++); buckets.add(reader.readMap(id)); } } @@ -146,9 +146,9 @@ public class MapRecord extends Record { int size() { Segment segment = getSegment(); - int head = segment.readInt(getOffset(0)); + int head = segment.readInt(getRecordNumber()); if (isDiff(head)) { - RecordId base = segment.readRecordId(getOffset(8, 2)); + RecordId base = segment.readRecordId(getRecordNumber(), 8, 2); return reader.readMap(base).size(); } return getSize(head); @@ -159,16 +159,16 @@ public class MapRecord extends Record { int hash = getHash(name); Segment segment = getSegment(); - int head = segment.readInt(getOffset(0)); + int head = segment.readInt(getRecordNumber()); if (isDiff(head)) { - if (hash == segment.readInt(getOffset(4))) { - RecordId key = segment.readRecordId(getOffset(8)); + if (hash == segment.readInt(getRecordNumber(), 4)) { + RecordId key = segment.readRecordId(getRecordNumber(), 8); if (name.equals(reader.readString(key))) { - RecordId value = segment.readRecordId(getOffset(8, 1)); + RecordId value = segment.readRecordId(getRecordNumber(), 8, 1); return new MapEntry(reader, name, key, value); } } - RecordId base = segment.readRecordId(getOffset(8, 2)); + RecordId base = segment.readRecordId(getRecordNumber(), 8, 2); return reader.readMap(base).getEntry(name); } @@ -181,14 +181,14 @@ public class MapRecord extends Record { if (isBranch(size, level)) { // this is an intermediate branch record // check if a matching bucket exists, and recurse - int bitmap = segment.readInt(getOffset(4)); + int bitmap = segment.readInt(getRecordNumber(), 4); int mask = (1 << BITS_PER_LEVEL) - 1; int shift = 32 - (level + 1) * BITS_PER_LEVEL; int index = (hash >> shift) & mask; int bit = 1 << index; if ((bitmap & bit) != 0) { int ids = bitCount(bitmap & (bit - 1)); - RecordId id = segment.readRecordId(getOffset(8, ids)); + RecordId id = segment.readRecordId(getRecordNumber(), 8, ids); return reader.readMap(id).getEntry(name); } else { return null; @@ -211,13 +211,11 @@ public class MapRecord extends Record { int i = p + (int) ((q - p) * (h - pH) / (qH - pH)); assert p <= i && i <= q; - long iH = segment.readInt(getOffset(4 + i * 4)) & HASH_MASK; + long iH = segment.readInt(getRecordNumber(), 4 + i * 4) & HASH_MASK; int diff = Long.valueOf(iH).compareTo(Long.valueOf(h)); if (diff == 0) { - RecordId keyId = segment.readRecordId( - getOffset(4 + size * 4, i * 2)); - RecordId valueId = segment.readRecordId( - getOffset(4 + size * 4, i * 2 + 1)); + RecordId keyId = segment.readRecordId(getRecordNumber(), 4 + size * 4, i * 2); + RecordId valueId = segment.readRecordId(getRecordNumber(), 4 + size * 4, i * 2 + 1); diff = reader.readString(keyId).compareTo(name); if (diff == 0) { return new MapEntry(reader, name, keyId, valueId); @@ -239,13 +237,13 @@ public class MapRecord extends Record { checkNotNull(key); Segment segment = getSegment(); - int head = segment.readInt(getOffset(0)); + int head = segment.readInt(getRecordNumber()); if (isDiff(head)) { - if (hash == segment.readInt(getOffset(4)) - && key.equals(segment.readRecordId(getOffset(8)))) { - return segment.readRecordId(getOffset(8, 1)); + if (hash == segment.readInt(getRecordNumber(), 4) + && key.equals(segment.readRecordId(getRecordNumber(), 8))) { + return segment.readRecordId(getRecordNumber(), 8, 1); } - RecordId base = segment.readRecordId(getOffset(8, 2)); + RecordId base = segment.readRecordId(getRecordNumber(), 8, 2); return reader.readMap(base).getValue(hash, key); } @@ -258,14 +256,14 @@ public class MapRecord extends Record { if (isBranch(size, level)) { // this is an intermediate branch record // check if a matching bucket exists, and recurse - int bitmap = segment.readInt(getOffset(4)); + int bitmap = segment.readInt(getRecordNumber(), 4); int mask = (1 << BITS_PER_LEVEL) - 1; int shift = 32 - (level + 1) * BITS_PER_LEVEL; int index = (hash >> shift) & mask; int bit = 1 << index; if ((bitmap & bit) != 0) { int ids = bitCount(bitmap & (bit - 1)); - RecordId id = segment.readRecordId(getOffset(8, ids)); + RecordId id = segment.readRecordId(getRecordNumber(), 8, ids); return reader.readMap(id).getValue(hash, key); } else { return null; @@ -275,15 +273,12 @@ public class MapRecord extends Record { // this is a leaf record; scan the list to find a matching entry Long h = hash & HASH_MASK; for (int i = 0; i < size; i++) { - int hashOffset = getOffset(4 + i * 4); - int diff = h.compareTo(segment.readInt(hashOffset) & HASH_MASK); + int diff = h.compareTo(segment.readInt(getRecordNumber(), 4 + i * 4) & HASH_MASK); if (diff > 0) { return null; } else if (diff == 0) { - int keyOffset = getOffset(4 + size * 4, i * 2); - if (key.equals(segment.readRecordId(keyOffset))) { - int valueOffset = getOffset(4 + size * 4, i * 2 + 1); - return segment.readRecordId(valueOffset); + if (key.equals(segment.readRecordId(getRecordNumber(), 4 + size * 4, i * 2))) { + return segment.readRecordId(getRecordNumber(), 4 + size * 4, i * 2 + 1); } } } @@ -293,9 +288,9 @@ public class MapRecord extends Record { Iterable getKeys() { Segment segment = getSegment(); - int head = segment.readInt(getOffset(0)); + int head = segment.readInt(getRecordNumber()); if (isDiff(head)) { - RecordId base = segment.readRecordId(getOffset(8, 2)); + RecordId base = segment.readRecordId(getRecordNumber(), 8, 2); return reader.readMap(base).getKeys(); } @@ -317,7 +312,7 @@ public class MapRecord extends Record { RecordId[] ids = new RecordId[size]; for (int i = 0; i < size; i++) { - ids[i] = segment.readRecordId(getOffset(4 + size * 4, i * 2)); + ids[i] = segment.readRecordId(getRecordNumber(), 4 + size * 4, i * 2); } String[] keys = new String[size]; @@ -335,11 +330,11 @@ public class MapRecord extends Record { final RecordId diffKey, final RecordId diffValue) { Segment segment = getSegment(); - int head = segment.readInt(getOffset(0)); + int head = segment.readInt(getRecordNumber()); if (isDiff(head)) { - RecordId key = segment.readRecordId(getOffset(8)); - RecordId value = segment.readRecordId(getOffset(8, 1)); - RecordId base = segment.readRecordId(getOffset(8, 2)); + RecordId key = segment.readRecordId(getRecordNumber(), 8); + RecordId value = segment.readRecordId(getRecordNumber(), 8, 1); + RecordId base = segment.readRecordId(getRecordNumber(), 8, 2); return reader.readMap(base).getEntries(key, value); } @@ -366,12 +361,12 @@ public class MapRecord extends Record { MapEntry[] entries = new MapEntry[size]; for (int i = 0; i < size; i++) { - RecordId key = segment.readRecordId(getOffset(4 + size * 4, i * 2)); + RecordId key = segment.readRecordId(getRecordNumber(), 4 + size * 4, i * 2); RecordId value; if (key.equals(diffKey)) { value = diffValue; } else { - value = segment.readRecordId(getOffset(4 + size * 4, i * 2 + 1)); + value = segment.readRecordId(getRecordNumber(), 4 + size * 4, i * 2 + 1); } String name = reader.readString(key); entries[i] = new MapEntry(reader, name, key, value); @@ -385,13 +380,13 @@ public class MapRecord extends Record { } Segment segment = getSegment(); - int head = segment.readInt(getOffset(0)); + int head = segment.readInt(getRecordNumber()); if (isDiff(head)) { - int hash = segment.readInt(getOffset(4)); - RecordId keyId = segment.readRecordId(getOffset(8)); + int hash = segment.readInt(getRecordNumber(), 4); + RecordId keyId = segment.readRecordId(getRecordNumber(), 8); final String key = reader.readString(keyId); - final RecordId value = segment.readRecordId(getOffset(8, 1)); - MapRecord base = reader.readMap(segment.readRecordId(getOffset(8, 2))); + final RecordId value = segment.readRecordId(getRecordNumber(), 8, 1); + MapRecord base = reader.readMap(segment.readRecordId(getRecordNumber(), 8, 2)); boolean rv = base.compare(before, new DefaultNodeStateDiff() { @Override @@ -427,13 +422,13 @@ public class MapRecord extends Record { } Segment beforeSegment = before.getSegment(); - int beforeHead = beforeSegment.readInt(before.getOffset(0)); + int beforeHead = beforeSegment.readInt(before.getRecordNumber()); if (isDiff(beforeHead)) { - int hash = beforeSegment.readInt(before.getOffset(4)); - RecordId keyId = beforeSegment.readRecordId(before.getOffset(8)); + int hash = beforeSegment.readInt(before.getRecordNumber(), 4); + RecordId keyId = beforeSegment.readRecordId(before.getRecordNumber(), 8); final String key = reader.readString(keyId); - final RecordId value = beforeSegment.readRecordId(before.getOffset(8, 1)); - MapRecord base = reader.readMap(beforeSegment.readRecordId(before.getOffset(8, 2))); + final RecordId value = beforeSegment.readRecordId(before.getRecordNumber(), 8, 1); + MapRecord base = reader.readMap(beforeSegment.readRecordId(before.getRecordNumber(), 8, 2)); boolean rv = this.compare(base, new DefaultNodeStateDiff() { @Override diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/MutableRecordNumbers.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/MutableRecordNumbers.java new file mode 100644 index 0000000..aa70e85 --- /dev/null +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/MutableRecordNumbers.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jackrabbit.oak.segment; + +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import com.google.common.collect.Maps; + +/** + * A thread-safe, mutable record numbers to offset table. + */ +class MutableRecordNumbers implements RecordNumbers { + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + + private final Map recordNumbers = Maps.newHashMap(); + + @Override + public int getOffset(int recordNumber) { + lock.readLock().lock(); + + Integer offset = null; + + try { + offset = recordNumbers.get(recordNumber); + } finally { + lock.readLock().unlock(); + } + + if (offset == null) { + return -1; + } + + return offset; + } + + @Override + public Iterator iterator() { + Map recordNumbers; + + lock.readLock().lock(); + + try { + recordNumbers = Maps.newHashMap(this.recordNumbers); + } finally { + lock.readLock().unlock(); + } + + return new RecordNumbersIterator(recordNumbers.entrySet().iterator()); + } + + /** + * Return the size of this table. + * + * @return the size of this table. + */ + public int size() { + lock.readLock().lock(); + + try { + return recordNumbers.size(); + } finally { + lock.readLock().unlock(); + } + } + + /** + * Add a new offset to this table and generate a record number for it. + * + * @param offset an offset to be added to this table. + * @return the record number associated to the offset. + */ + public int addOffset(int offset) { + lock.writeLock().lock(); + + int recordNumber; + + try { + recordNumber = recordNumbers.size(); + recordNumbers.put(recordNumber, offset); + } finally { + lock.writeLock().unlock(); + } + + return recordNumber; + } + +} diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Record.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Record.java index 65731ea..48a4b4e 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Record.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Record.java @@ -34,7 +34,7 @@ class Record { } private static boolean fastEquals(@Nonnull Record a, @Nonnull Record b) { - return a == b || (a.offset == b.offset && a.segmentId.equals(b.segmentId)); + return a == b || (a.recordNumber == b.recordNumber && a.segmentId.equals(b.segmentId)); } /** @@ -43,9 +43,9 @@ class Record { private final SegmentId segmentId; /** - * Segment offset of this record. + * Segment recordNumber of this record. */ - private final int offset; + private final int recordNumber; /** * Creates a new object for the identified record. @@ -53,12 +53,12 @@ class Record { * @param id record identified */ protected Record(@Nonnull RecordId id) { - this(id.getSegmentId(), id.getOffset()); + this(id.getSegmentId(), id.getRecordNumber()); } - protected Record(@Nonnull SegmentId segmentId, int offset) { + protected Record(@Nonnull SegmentId segmentId, int recordNumber) { this.segmentId = segmentId; - this.offset = offset; + this.recordNumber = recordNumber; } /** @@ -70,45 +70,17 @@ class Record { return segmentId.getSegment(); } + protected int getRecordNumber() { + return recordNumber; + } + /** * Returns the identifier of this record. * * @return record identifier */ public RecordId getRecordId() { - return new RecordId(segmentId, offset); - } - - /** - * Returns the segment offset of this record. - * - * @return segment offset of this record - */ - protected final int getOffset() { - return offset; - } - - /** - * Returns the segment offset of the given byte position in this record. - * - * @param position byte position within this record - * @return segment offset of the given byte position - */ - protected final int getOffset(int position) { - return getOffset() + position; - } - - /** - * Returns the segment offset of a byte position in this record. - * The position is calculated from the given number of raw bytes and - * record identifiers. - * - * @param bytes number of raw bytes before the position - * @param ids number of record identifiers before the position - * @return segment offset of the specified byte position - */ - protected final int getOffset(int bytes, int ids) { - return getOffset(bytes + ids * Segment.RECORD_ID_BYTES); + return new RecordId(segmentId, recordNumber); } //------------------------------------------------------------< Object >-- @@ -120,7 +92,7 @@ class Record { @Override public int hashCode() { - return segmentId.hashCode() ^ offset; + return segmentId.hashCode() ^ recordNumber; } @Override diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordId.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordId.java index 4fbc03e..d867253 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordId.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordId.java @@ -22,7 +22,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static java.lang.Integer.parseInt; import static org.apache.jackrabbit.oak.segment.Segment.RECORD_ALIGN_BITS; -import static org.apache.jackrabbit.oak.segment.Segment.pack; +import static org.apache.jackrabbit.oak.segment.Segment.RECORD_ID_BYTES; import java.util.UUID; import java.util.regex.Matcher; @@ -38,7 +38,7 @@ public final class RecordId implements Comparable { private static final Pattern PATTERN = Pattern.compile( "([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})" - + "(:(0|[1-9][0-9]*)|\\.([0-9a-f]{4}))"); + + "(:(0|[1-9][0-9]*)|\\.([0-9a-f]{8}))"); public static RecordId[] EMPTY_ARRAY = new RecordId[0]; @@ -54,7 +54,7 @@ public final class RecordId implements Comparable { if (matcher.group(3) != null) { offset = parseInt(matcher.group(3)); } else { - offset = parseInt(matcher.group(4), 16) << RECORD_ALIGN_BITS; + offset = parseInt(matcher.group(4), 16); } return new RecordId(segmentId, offset); @@ -68,8 +68,6 @@ public final class RecordId implements Comparable { private final int offset; public RecordId(SegmentId segmentId, int offset) { - checkArgument(offset < Segment.MAX_SEGMENT_SIZE); - checkArgument((offset % (1 << RECORD_ALIGN_BITS)) == 0); this.segmentId = checkNotNull(segmentId); this.offset = offset; } @@ -78,7 +76,7 @@ public final class RecordId implements Comparable { return segmentId; } - public int getOffset() { + public int getRecordNumber() { return offset; } @@ -94,27 +92,16 @@ public final class RecordId implements Comparable { return segmentId.getSegment(); } - private static void writeLong(byte[] buffer, int pos, long value) { - for (int k = 0; k < 8; k++) { - buffer[pos + k] = (byte) (value >> (56 - (k << 3))); - } - } - - private static void writeShort(byte[] buffer, int pos, short value) { - buffer[pos] = (byte) (value >> 8); - buffer[pos + 1] = (byte) value; - } - /** * Serialise this record id into an array of bytes: {@code (msb, lsb, offset >> 2)} * @return this record id as byte array */ @Nonnull byte[] getBytes() { - byte[] buffer = new byte[18]; - writeLong(buffer, 0, segmentId.getMostSignificantBits()); - writeLong(buffer, 8, segmentId.getLeastSignificantBits()); - writeShort(buffer, 16, pack(offset)); + byte[] buffer = new byte[RECORD_ID_BYTES]; + BinaryUtils.writeLong(buffer, 0, segmentId.getMostSignificantBits()); + BinaryUtils.writeLong(buffer, 8, segmentId.getLeastSignificantBits()); + BinaryUtils.writeInt(buffer, 16, offset); return buffer; } @@ -134,7 +121,7 @@ public final class RecordId implements Comparable { @Override public String toString() { - return String.format("%s.%04x", segmentId, offset >> RECORD_ALIGN_BITS); + return String.format("%s.%08x", segmentId, offset); } /** diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordIdSet.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordIdSet.java index 3d4e5d8..a7b5aab 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordIdSet.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordIdSet.java @@ -22,7 +22,6 @@ package org.apache.jackrabbit.oak.segment; import static com.google.common.collect.Maps.newHashMap; import static java.lang.System.arraycopy; import static java.util.Arrays.binarySearch; -import static org.apache.jackrabbit.oak.segment.Segment.pack; import java.util.Map; @@ -33,7 +32,7 @@ import java.util.Map; * it contains. */ public class RecordIdSet { - private final Map seenIds = newHashMap(); + private final Map seenIds = newHashMap(); /** * Add {@code id} to this set if not already present @@ -42,12 +41,12 @@ public class RecordIdSet { */ public boolean addIfNotPresent(RecordId id) { String segmentId = id.getSegmentId().toString(); - ShortSet offsets = seenIds.get(segmentId); + IntSet offsets = seenIds.get(segmentId); if (offsets == null) { - offsets = new ShortSet(); + offsets = new IntSet(); seenIds.put(segmentId, offsets); } - return offsets.add(pack(id.getOffset())); + return offsets.add(id.getRecordNumber()); } /** @@ -57,23 +56,23 @@ public class RecordIdSet { */ public boolean contains(RecordId id) { String segmentId = id.getSegmentId().toString(); - ShortSet offsets = seenIds.get(segmentId); - return offsets != null && offsets.contains(pack(id.getOffset())); + IntSet offsets = seenIds.get(segmentId); + return offsets != null && offsets.contains(id.getRecordNumber()); } - static class ShortSet { - short[] elements; + static class IntSet { + int[] elements; - boolean add(short n) { + boolean add(int n) { if (elements == null) { - elements = new short[1]; + elements = new int[1]; elements[0] = n; return true; } else { int k = binarySearch(elements, n); if (k < 0) { int l = -k - 1; - short[] e = new short[elements.length + 1]; + int[] e = new int[elements.length + 1]; arraycopy(elements, 0, e, 0, l); e[l] = n; int c = elements.length - l; @@ -88,7 +87,7 @@ public class RecordIdSet { } } - boolean contains(short n) { + boolean contains(int n) { return elements != null && binarySearch(elements, n) >= 0; } } diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordNumbers.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordNumbers.java new file mode 100644 index 0000000..7021ddb --- /dev/null +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordNumbers.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jackrabbit.oak.segment; + +import org.apache.jackrabbit.oak.segment.RecordNumbers.Entry; + +/** + * A table to translate record numbers to offsets. + */ +interface RecordNumbers extends Iterable { + + /** + * Translate a record number to an offset. + * + * @param recordNumber A record number. + * @return the offset corresponding to the record number, or {@code -1} if + * no offset is associated to the record number. + */ + int getOffset(int recordNumber); + + /** + * Represents a pair of a record number and its corresponding offset. + */ + interface Entry { + + /** + * The record number part of this pair. + * + * @return a record number. + */ + int getRecordNumber(); + + /** + * The offset part of this pair. + * + * @return an offset. + */ + int getOffset(); + + } + +} diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordNumbersIterator.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordNumbersIterator.java new file mode 100644 index 0000000..4db106f --- /dev/null +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordNumbersIterator.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jackrabbit.oak.segment; + +import java.util.Iterator; +import java.util.Map; + +import org.apache.jackrabbit.oak.segment.RecordNumbers.Entry; + +/** + * Utility class implementing an iterator over record numbers to offset pairs. + * It wraps an underlying iterator looping over map entries, where each entry is + * a tuple of integers. + */ +class RecordNumbersIterator implements Iterator { + + private static class Entry implements RecordNumbers.Entry { + + private final Map.Entry entry; + + public Entry(Map.Entry entry) { + this.entry = entry; + } + + @Override + public int getRecordNumber() { + return entry.getKey(); + } + + @Override + public int getOffset() { + return entry.getValue(); + } + + } + + private final Iterator> iterator; + + public RecordNumbersIterator(Iterator> iterator) { + this.iterator = iterator; + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public RecordNumbers.Entry next() { + return new Entry(iterator.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + +} diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java index 3f9b54a..a2fb369 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java @@ -22,6 +22,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkPositionIndexes; import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.Maps.newHashMapWithExpectedSize; import static org.apache.jackrabbit.oak.commons.IOUtils.closeQuietly; import static org.apache.jackrabbit.oak.segment.SegmentId.isDataSegmentId; import static org.apache.jackrabbit.oak.segment.SegmentVersion.LATEST_VERSION; @@ -33,6 +34,7 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.Map; import java.util.UUID; import javax.annotation.CheckForNull; @@ -44,6 +46,7 @@ import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.plugins.memory.PropertyStates; +import org.apache.jackrabbit.oak.segment.RecordNumbers.Entry; /** * A list of records. @@ -55,14 +58,14 @@ import org.apache.jackrabbit.oak.plugins.memory.PropertyStates; */ public class Segment { - static final int HEADER_SIZE = 18; + static final int HEADER_SIZE = 22; /** * Number of bytes used for storing a record identifier. One byte * is used for identifying the segment and two for the record offset * within that segment. */ - static final int RECORD_ID_BYTES = 8 + 8 + 2; + static final int RECORD_ID_BYTES = 8 + 8 + 4; /** * The limit on segment references within one segment. Since record @@ -117,6 +120,8 @@ public class Segment { public static final int REFERENCED_SEGMENT_ID_COUNT_OFFSET = 14; + public static final int RECORD_NUMBER_COUNT_OFFSET = 18; + @Nonnull private final SegmentStore store; @@ -136,22 +141,9 @@ public class Segment { private final SegmentVersion version; /** - * Unpacks a 4 byte aligned segment offset. - * @param offset 4 byte aligned segment offset - * @return unpacked segment offset - */ - public static int unpack(short offset) { - return (offset & 0xffff) << RECORD_ALIGN_BITS; - } - - /** - * Packs a segment offset into a 4 byte aligned address packed into a {@code short}. - * @param offset segment offset - * @return encoded segment offset packed into a {@code short} + * The table translating record numbers to offsets. */ - public static short pack(int offset) { - return (short) (offset >> RECORD_ALIGN_BITS); - } + private final RecordNumbers recordNumbers; /** * Align an {@code address} on the given {@code boundary} @@ -187,8 +179,10 @@ public class Segment { } }); this.version = SegmentVersion.fromByte(segmentVersion); + this.recordNumbers = readRecordNumberOffsets(); } else { this.version = LATEST_VERSION; + this.recordNumbers = new IdentityRecordNumbers(); } } @@ -204,9 +198,34 @@ public class Segment { } } + /** + * Read the serialized table mapping record numbers to offsets. + * + * @return An instance of {@link RecordNumbers}, never {@code null}. + */ + private RecordNumbers readRecordNumberOffsets() { + Map recordNumberOffsets = newHashMapWithExpectedSize(getRecordNumberCount()); + + int position = data.position(); + + position += HEADER_SIZE; + position += getReferencedSegmentIdCount() * 16; + + for (int i = 0; i < getRecordNumberCount(); i++) { + int recordNumber = data.getInt(position); + position += 4; + int offset = data.getInt(position); + position += 4; + recordNumberOffsets.put(recordNumber, offset); + } + + return new ImmutableRecordNumbers(recordNumberOffsets); + } + Segment(@Nonnull SegmentStore store, @Nonnull SegmentReader reader, @Nonnull byte[] buffer, + @Nonnull RecordNumbers recordNumbers, @Nonnull String info) { this.store = checkNotNull(store); this.reader = checkNotNull(reader); @@ -214,6 +233,7 @@ public class Segment { this.info = checkNotNull(info); this.data = ByteBuffer.wrap(checkNotNull(buffer)); this.version = SegmentVersion.fromByte(buffer[3]); + this.recordNumbers = recordNumbers; id.loaded(this); } @@ -221,18 +241,36 @@ public class Segment { return version; } + private int pos(int recordNumber, int length) { + return pos(recordNumber, 0, 0, length); + } + + private int pos(int recordNumber, int rawOffset, int length) { + return pos(recordNumber, rawOffset, 0, length); + } + /** - * Maps the given record offset to the respective position within the + * Maps the given record number to the respective position within the * internal {@link #data} array. The validity of a record with the given - * length at the given offset is also verified. + * length at the given record number is also verified. * - * @param offset record offset - * @param length record length + * @param recordNumber record number + * @param rawOffset offset to add to the base position of the record + * @param recordIdOffset offset to add to to the base position of the + * record, multiplied by the length of a record ID + * @param length record length * @return position within the data array */ - private int pos(int offset, int length) { - checkPositionIndexes(offset, offset + length, MAX_SEGMENT_SIZE); - int pos = data.limit() - MAX_SEGMENT_SIZE + offset; + private int pos(int recordNumber, int rawOffset, int recordIdOffset, int length) { + int offset = recordNumbers.getOffset(recordNumber); + + if (offset == -1) { + throw new IllegalStateException("invalid record number"); + } + + int base = offset + rawOffset + recordIdOffset * RECORD_ID_BYTES; + checkPositionIndexes(base, base + length, MAX_SEGMENT_SIZE); + int pos = data.limit() - MAX_SEGMENT_SIZE + base; checkState(pos >= data.position()); return pos; } @@ -249,6 +287,10 @@ public class Segment { return data.getInt(REFERENCED_SEGMENT_ID_COUNT_OFFSET); } + public int getRecordNumberCount() { + return data.getInt(RECORD_NUMBER_COUNT_OFFSET); + } + public UUID getReferencedSegmentId(int index) { checkArgument(index < getReferencedSegmentIdCount()); @@ -293,7 +335,8 @@ public class Segment { position += HEADER_SIZE; position += getReferencedSegmentIdCount() * 16; - position += index * 3; + position += getRecordNumberCount() * 8; + position += index * 5; return RecordType.values()[data.get(position) & 0xff]; } @@ -305,10 +348,11 @@ public class Segment { position += HEADER_SIZE; position += getReferencedSegmentIdCount() * 16; - position += index * 3; + position += getRecordNumberCount() * 8; + position += index * 5; position += 1; - return (data.getShort(position) & 0xffff) << RECORD_ALIGN_BITS; + return data.getInt(position); } private volatile String info; @@ -340,48 +384,67 @@ public class Segment { return data.remaining(); } - byte readByte(int offset) { - return data.get(pos(offset, 1)); + byte readByte(int recordNumber) { + return readByte(recordNumber, 0); + } + + byte readByte(int recordNumber, int offset) { + return data.get(pos(recordNumber, offset, 1)); + } + + short readShort(int recordNumber) { + return data.getShort(pos(recordNumber, 2)); } - short readShort(int offset) { - return data.getShort(pos(offset, 2)); + int readInt(int recordNumber) { + return data.getInt(pos(recordNumber, 4)); } - int readInt(int offset) { - return data.getInt(pos(offset, 4)); + int readInt(int recordNumber, int offset) { + return data.getInt(pos(recordNumber, offset, 4)); } - long readLong(int offset) { - return data.getLong(pos(offset, 8)); + long readLong(int recordNumber) { + return data.getLong(pos(recordNumber, 8)); } /** * Reads the given number of bytes starting from the given position * in this segment. * - * @param position position within segment + * @param recordNumber position within segment * @param buffer target buffer * @param offset offset within target buffer * @param length number of bytes to read */ - void readBytes(int position, byte[] buffer, int offset, int length) { + void readBytes(int recordNumber, byte[] buffer, int offset, int length) { + readBytes(recordNumber, 0, buffer, offset, length); + } + + void readBytes(int recordNumber, int position, byte[] buffer, int offset, int length) { checkNotNull(buffer); checkPositionIndexes(offset, offset + length, buffer.length); ByteBuffer d = data.duplicate(); - d.position(pos(position, length)); + d.position(pos(recordNumber, position, length)); d.get(buffer, offset, length); } - RecordId readRecordId(int offset) { - int pos = pos(offset, RECORD_ID_BYTES); - return internalReadRecordId(pos); + RecordId readRecordId(int recordNumber, int rawOffset, int recordIdOffset) { + return internalReadRecordId(pos(recordNumber, rawOffset, recordIdOffset, RECORD_ID_BYTES)); + } + + RecordId readRecordId(int recordNumber, int rawOffset) { + return readRecordId(recordNumber, rawOffset, 0); + } + + RecordId readRecordId(int recordNumber) { + return readRecordId(recordNumber, 0, 0); } private RecordId internalReadRecordId(int pos) { long msb = data.getLong(pos); long lsb = data.getLong(pos + 8); - int offset = (data.getShort(pos + 16) & 0xffff) << RECORD_ALIGN_BITS; + int offset = data.getInt(pos + 16); return new RecordId(store.newSegmentId(msb, lsb), offset); } @@ -413,19 +476,20 @@ public class Segment { } @Nonnull - Template readTemplate(int offset) { - int head = readInt(offset); + Template readTemplate(int recordNumber) { + int head = readInt(recordNumber); boolean hasPrimaryType = (head & (1 << 31)) != 0; boolean hasMixinTypes = (head & (1 << 30)) != 0; boolean zeroChildNodes = (head & (1 << 29)) != 0; boolean manyChildNodes = (head & (1 << 28)) != 0; int mixinCount = (head >> 18) & ((1 << 10) - 1); int propertyCount = head & ((1 << 18) - 1); - offset += 4; + + int offset = 4; PropertyState primaryType = null; if (hasPrimaryType) { - RecordId primaryId = readRecordId(offset); + RecordId primaryId = readRecordId(recordNumber, offset); primaryType = PropertyStates.createProperty( "jcr:primaryType", reader.readString(primaryId), Type.NAME); offset += RECORD_ID_BYTES; @@ -435,7 +499,7 @@ public class Segment { if (hasMixinTypes) { String[] mixins = new String[mixinCount]; for (int i = 0; i < mixins.length; i++) { - RecordId mixinId = readRecordId(offset); + RecordId mixinId = readRecordId(recordNumber, offset); mixins[i] = reader.readString(mixinId); offset += RECORD_ID_BYTES; } @@ -447,24 +511,24 @@ public class Segment { if (manyChildNodes) { childName = Template.MANY_CHILD_NODES; } else if (!zeroChildNodes) { - RecordId childNameId = readRecordId(offset); + RecordId childNameId = readRecordId(recordNumber, offset); childName = reader.readString(childNameId); offset += RECORD_ID_BYTES; } PropertyTemplate[] properties; - properties = readProps(propertyCount, offset); + properties = readProps(propertyCount, recordNumber, offset); return new Template(reader, primaryType, mixinTypes, properties, childName); } - private PropertyTemplate[] readProps(int propertyCount, int offset) { + private PropertyTemplate[] readProps(int propertyCount, int recordNumber, int offset) { PropertyTemplate[] properties = new PropertyTemplate[propertyCount]; if (propertyCount > 0) { - RecordId id = readRecordId(offset); + RecordId id = readRecordId(recordNumber, offset); ListRecord propertyNames = new ListRecord(id, properties.length); offset += RECORD_ID_BYTES; for (int i = 0; i < propertyCount; i++) { - byte type = readByte(offset++); + byte type = readByte(recordNumber, offset++); properties[i] = new PropertyTemplate(i, reader.readString(propertyNames.getEntry(i)), Type.fromTag( Math.abs(type), type < 0)); @@ -474,11 +538,11 @@ public class Segment { } long readLength(RecordId id) { - return id.getSegment().readLength(id.getOffset()); + return id.getSegment().readLength(id.getRecordNumber()); } - long readLength(int offset) { - return internalReadLength(pos(offset, 1)); + long readLength(int recordNumber) { + return internalReadLength(pos(recordNumber, 1)); } private long internalReadLength(int pos) { @@ -522,6 +586,10 @@ public class Segment { writer.format("reference %02x: %s%n", i, getReferencedSegmentId(i)); } + for (Entry entry : recordNumbers) { + writer.format("record number %08x: %08x", entry.getRecordNumber(), entry.getOffset()); + } + for (int i = 0; i < getRootCount(); i++) { writer.format("root %d: %s at %04x%n", i, getRootType(i), getRootOffset(i)); } diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java index b70ac5a..831d261 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java @@ -59,38 +59,35 @@ public class SegmentBlob extends Record implements Blob { this.blobStore = blobStore; } - private InputStream getInlineStream( - Segment segment, int offset, int length) { + private InputStream getInlineStream(Segment segment, int offset, int length) { byte[] inline = new byte[length]; - segment.readBytes(offset, inline, 0, length); + segment.readBytes(getRecordNumber(), offset, inline, 0, length); return new SegmentStream(getRecordId(), inline); } @Override @Nonnull public InputStream getNewStream() { Segment segment = getSegment(); - int offset = getOffset(); - byte head = segment.readByte(offset); + byte head = segment.readByte(getRecordNumber()); if ((head & 0x80) == 0x00) { // 0xxx xxxx: small value - return getInlineStream(segment, offset + 1, head); + return getInlineStream(segment, 1, head); } else if ((head & 0xc0) == 0x80) { // 10xx xxxx: medium value - int length = (segment.readShort(offset) & 0x3fff) + SMALL_LIMIT; - return getInlineStream(segment, offset + 2, length); + int length = (segment.readShort(getRecordNumber()) & 0x3fff) + SMALL_LIMIT; + return getInlineStream(segment, 2, length); } else if ((head & 0xe0) == 0xc0) { // 110x xxxx: long value - long length = (segment.readLong(offset) & 0x1fffffffffffffffL) + MEDIUM_LIMIT; + long length = (segment.readLong(getRecordNumber()) & 0x1fffffffffffffffL) + MEDIUM_LIMIT; int listSize = (int) ((length + BLOCK_SIZE - 1) / BLOCK_SIZE); - ListRecord list = new ListRecord( - segment.readRecordId(offset + 8), listSize); + ListRecord list = new ListRecord(segment.readRecordId(getRecordNumber(), 8), listSize); return new SegmentStream(getRecordId(), list, length); } else if ((head & 0xf0) == 0xe0) { // 1110 xxxx: external value, short blob ID - return getNewStream(readShortBlobId(segment, offset, head)); + return getNewStream(readShortBlobId(segment, getRecordNumber(), head)); } else if ((head & 0xf8) == 0xf0) { // 1111 0xxx: external value, long blob ID - return getNewStream(readLongBlobId(segment, offset)); + return getNewStream(readLongBlobId(segment, getRecordNumber())); } else { throw new IllegalStateException(String.format( "Unexpected value record type: %02x", head & 0xff)); @@ -100,23 +97,22 @@ public class SegmentBlob extends Record implements Blob { @Override public long length() { Segment segment = getSegment(); - int offset = getOffset(); - byte head = segment.readByte(offset); + byte head = segment.readByte(getRecordNumber()); if ((head & 0x80) == 0x00) { // 0xxx xxxx: small value return head; } else if ((head & 0xc0) == 0x80) { // 10xx xxxx: medium value - return (segment.readShort(offset) & 0x3fff) + SMALL_LIMIT; + return (segment.readShort(getRecordNumber()) & 0x3fff) + SMALL_LIMIT; } else if ((head & 0xe0) == 0xc0) { // 110x xxxx: long value - return (segment.readLong(offset) & 0x1fffffffffffffffL) + MEDIUM_LIMIT; + return (segment.readLong(getRecordNumber()) & 0x1fffffffffffffffL) + MEDIUM_LIMIT; } else if ((head & 0xf0) == 0xe0) { // 1110 xxxx: external value, short blob ID - return getLength(readShortBlobId(segment, offset, head)); + return getLength(readShortBlobId(segment, getRecordNumber(), head)); } else if ((head & 0xf8) == 0xf0) { // 1111 0xxx: external value, long blob ID - return getLength(readLongBlobId(segment, offset)); + return getLength(readLongBlobId(segment, getRecordNumber())); } else { throw new IllegalStateException(String.format( "Unexpected value record type: %02x", head & 0xff)); @@ -150,26 +146,25 @@ public class SegmentBlob extends Record implements Blob { public boolean isExternal() { Segment segment = getSegment(); - int offset = getOffset(); - byte head = segment.readByte(offset); + byte head = segment.readByte(getRecordNumber()); // 1110 xxxx or 1111 0xxx: external value return (head & 0xf0) == 0xe0 || (head & 0xf8) == 0xf0; } @CheckForNull public String getBlobId() { - return readBlobId(getSegment(), getOffset()); + return readBlobId(getSegment(), getRecordNumber()); } @CheckForNull - static String readBlobId(@Nonnull Segment segment, int offset) { - byte head = segment.readByte(offset); + static String readBlobId(@Nonnull Segment segment, int recordNumber) { + byte head = segment.readByte(recordNumber); if ((head & 0xf0) == 0xe0) { // 1110 xxxx: external value, small blob ID - return readShortBlobId(segment, offset, head); + return readShortBlobId(segment, recordNumber, head); } else if ((head & 0xf8) == 0xf0) { // 1111 0xxx: external value, long blob ID - return readLongBlobId(segment, offset); + return readLongBlobId(segment, recordNumber); } else { return null; } @@ -205,28 +200,27 @@ public class SegmentBlob extends Record implements Blob { //-----------------------------------------------------------< private >-- - private static String readShortBlobId(Segment segment, int offset, byte head) { - int length = (head & 0x0f) << 8 | (segment.readByte(offset + 1) & 0xff); + private static String readShortBlobId(Segment segment, int recordNumber, byte head) { + int length = (head & 0x0f) << 8 | (segment.readByte(recordNumber, 1) & 0xff); byte[] bytes = new byte[length]; - segment.readBytes(offset + 2, bytes, 0, length); + segment.readBytes(recordNumber, 2, bytes, 0, length); return new String(bytes, UTF_8); } - private static String readLongBlobId(Segment segment, int offset) { - RecordId blobId = segment.readRecordId(offset + 1); - return blobId.getSegment().readString(blobId.getOffset()); + private static String readLongBlobId(Segment segment, int recordNumber) { + RecordId blobId = segment.readRecordId(recordNumber, 1); + return blobId.getSegment().readString(blobId.getRecordNumber()); } private List getBulkRecordIds() { Segment segment = getSegment(); - int offset = getOffset(); - byte head = segment.readByte(offset); + byte head = segment.readByte(getRecordNumber()); if ((head & 0xe0) == 0xc0) { // 110x xxxx: long value - long length = (segment.readLong(offset) & 0x1fffffffffffffffL) + MEDIUM_LIMIT; + long length = (segment.readLong(getRecordNumber()) & 0x1fffffffffffffffL) + MEDIUM_LIMIT; int listSize = (int) ((length + BLOCK_SIZE - 1) / BLOCK_SIZE); ListRecord list = new ListRecord( - segment.readRecordId(offset + 8), listSize); + segment.readRecordId(getRecordNumber(), 8), listSize); return list.getEntries(); } else { return null; diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java index b94f279..3ddb761 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java @@ -30,7 +30,6 @@ import static java.lang.System.currentTimeMillis; import static java.lang.System.identityHashCode; import static org.apache.jackrabbit.oak.segment.Segment.GC_GENERATION_OFFSET; import static org.apache.jackrabbit.oak.segment.Segment.HEADER_SIZE; -import static org.apache.jackrabbit.oak.segment.Segment.MAX_SEGMENT_SIZE; import static org.apache.jackrabbit.oak.segment.Segment.RECORD_ID_BYTES; import static org.apache.jackrabbit.oak.segment.Segment.align; import static org.apache.jackrabbit.oak.segment.SegmentId.isDataSegmentId; @@ -45,6 +44,7 @@ import java.util.UUID; import javax.annotation.CheckForNull; import javax.annotation.Nonnull; +import org.apache.jackrabbit.oak.segment.RecordNumbers.Entry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -110,6 +110,8 @@ public class SegmentBufferWriter implements WriteOperationHandler { private final Set referencedSegmentIds = newHashSet(); + private MutableRecordNumbers recordNumbers = new MutableRecordNumbers(); + @Nonnull private final SegmentStore store; @@ -210,13 +212,14 @@ public class SegmentBufferWriter implements WriteOperationHandler { position = buffer.length; roots.clear(); referencedSegmentIds.clear(); + recordNumbers = new MutableRecordNumbers(); String metaInfo = "{\"wid\":\"" + wid + '"' + ",\"sno\":" + tracker.getSegmentCount() + ",\"t\":" + currentTimeMillis() + "}"; try { - segment = new Segment(store, reader, buffer, metaInfo); + segment = new Segment(store, reader, buffer, recordNumbers, metaInfo); statistics = new Statistics(); statistics.id = segment.getSegmentId(); @@ -276,17 +279,13 @@ public class SegmentBufferWriter implements WriteOperationHandler { roots.remove(recordId); } - int offset = recordId.getOffset(); - - checkState(0 <= offset && offset < MAX_SEGMENT_SIZE); - checkState(offset == align(offset, 1 << Segment.RECORD_ALIGN_BITS)); - + int offset = recordId.getRecordNumber(); long msb = recordId.getSegmentId().getMostSignificantBits(); long lsb = recordId.getSegmentId().getLeastSignificantBits(); writeLong(msb); writeLong(lsb); - writeShort((short)((offset >> Segment.RECORD_ALIGN_BITS) & 0xffff)); + writeInt(offset); if (!recordId.getSegmentId().equals(segment.getSegmentId())) { referencedSegmentIds.add(recordId.getSegmentId().asUUID()); @@ -344,7 +343,10 @@ public class SegmentBufferWriter implements WriteOperationHandler { BinaryUtils.writeInt(buffer, Segment.REFERENCED_SEGMENT_ID_COUNT_OFFSET, referencedSegmentIdCount); statistics.segmentIdCount = referencedSegmentIdCount; - int totalLength = align(HEADER_SIZE + referencedSegmentIdCount * 16 + rootcount * 3 + length, 16); + int recordNumberCount = recordNumbers.size(); + BinaryUtils.writeInt(buffer, Segment.RECORD_NUMBER_COUNT_OFFSET, recordNumberCount); + + int totalLength = align(HEADER_SIZE + referencedSegmentIdCount * 16 + rootcount * 5 + recordNumberCount * 8 + length, 16); if (totalLength > buffer.length) { throw new IllegalStateException("too much data for a segment"); @@ -373,9 +375,14 @@ public class SegmentBufferWriter implements WriteOperationHandler { pos = BinaryUtils.writeLong(buffer, pos, id.getLeastSignificantBits()); } + for (Entry entry : recordNumbers) { + pos = BinaryUtils.writeInt(buffer, pos, entry.getRecordNumber()); + pos = BinaryUtils.writeInt(buffer, pos, entry.getOffset()); + } + for (Map.Entry entry : roots.entrySet()) { pos = BinaryUtils.writeByte(buffer, pos, (byte) entry.getValue().ordinal()); - pos = BinaryUtils.writeShort(buffer, pos, (short) (entry.getKey().getOffset() >> Segment.RECORD_ALIGN_BITS)); + pos = BinaryUtils.writeInt(buffer, pos, entry.getKey().getRecordNumber()); } SegmentId segmentId = segment.getSegmentId(); @@ -414,8 +421,9 @@ public class SegmentBufferWriter implements WriteOperationHandler { // unreferenced segments. int rootCount = roots.size() + 1; + int recordNumbersCount = recordNumbers.size() + 1; int referencedIdCount = referencedSegmentIds.size() + ids.size(); - int headerSize = HEADER_SIZE + rootCount * 3 + referencedIdCount * 16; + int headerSize = HEADER_SIZE + rootCount * 5 + referencedIdCount * 16 + recordNumbersCount * 8; int segmentSize = align(headerSize + recordSize + length, 16); // If the size estimate looks too big, recompute it with a more @@ -451,7 +459,7 @@ public class SegmentBufferWriter implements WriteOperationHandler { } } - headerSize = HEADER_SIZE + rootCount * 3 + referencedIdCount * 16; + headerSize = HEADER_SIZE + rootCount * 5 + referencedIdCount * 16 + recordNumbersCount * 8; segmentSize = align(headerSize + recordSize + length, 16); } @@ -465,7 +473,8 @@ public class SegmentBufferWriter implements WriteOperationHandler { position = buffer.length - length; checkState(position >= 0); - RecordId id = new RecordId(segment.getSegmentId(), position); + int recordNumber = recordNumbers.addOffset(position); + RecordId id = new RecordId(segment.getSegmentId(), recordNumber); roots.put(id, type); return id; } diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeState.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeState.java index 338eac5..4d3597c 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeState.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeState.java @@ -34,7 +34,7 @@ import static org.apache.jackrabbit.oak.api.Type.STRING; import static org.apache.jackrabbit.oak.api.Type.STRINGS; import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.MISSING_NODE; -import static org.apache.jackrabbit.oak.segment.Segment.unpack; +import static org.apache.jackrabbit.oak.segment.Segment.RECORD_ID_BYTES; import static org.apache.jackrabbit.oak.spi.state.AbstractNodeState.checkValidName; import java.nio.ByteBuffer; @@ -91,7 +91,7 @@ public class SegmentNodeState extends Record implements NodeState { if (templateId == null) { // no problem if updated concurrently, // as each concurrent thread will just get the same value - templateId = getSegment().readRecordId(getOffset(0, 1)); + templateId = getSegment().readRecordId(getRecordNumber(), 0, 1); } return templateId; } @@ -107,7 +107,7 @@ public class SegmentNodeState extends Record implements NodeState { MapRecord getChildNodeMap() { Segment segment = getSegment(); - return reader.readMap(segment.readRecordId(getOffset(0, 2))); + return reader.readMap(segment.readRecordId(getRecordNumber(), 0, 2)); } /** @@ -121,7 +121,7 @@ public class SegmentNodeState extends Record implements NodeState { ByteBuffer buffer = ByteBuffer.wrap(getStableIdBytes()); long msb = buffer.getLong(); long lsb = buffer.getLong(); - int offset = unpack(buffer.getShort()); + int offset = buffer.getInt(); return new UUID(msb, lsb) + ":" + offset; } @@ -135,7 +135,7 @@ public class SegmentNodeState extends Record implements NodeState { */ byte[] getStableIdBytes() { // The first record id of this node points to the stable id. - RecordId id = getSegment().readRecordId(getOffset()); + RecordId id = getSegment().readRecordId(getRecordNumber()); if (id.equals(getRecordId())) { // If that id is equal to the record id of this node then the stable @@ -145,8 +145,8 @@ public class SegmentNodeState extends Record implements NodeState { } else { // Otherwise that id points to the serialised (msb, lsb, offset) // stable id. - byte[] buffer = new byte[18]; - id.getSegment().readBytes(id.getOffset(), buffer, 0, buffer.length); + byte[] buffer = new byte[RECORD_ID_BYTES]; + id.getSegment().readBytes(id.getRecordNumber(), buffer, 0, buffer.length); return buffer; } } @@ -213,9 +213,8 @@ public class SegmentNodeState extends Record implements NodeState { if (template.getChildName() != Template.ZERO_CHILD_NODES) { ids++; } - RecordId rid = segment.readRecordId(getOffset(0, ids)); - ListRecord pIds = new ListRecord(rid, - template.getPropertyTemplates().length); + RecordId rid = segment.readRecordId(getRecordNumber(), 0, ids); + ListRecord pIds = new ListRecord(rid, template.getPropertyTemplates().length); return pIds.getEntry(propertyTemplate.getIndex()); } @@ -243,9 +242,7 @@ public class SegmentNodeState extends Record implements NodeState { } if (propertyTemplates.length > 0) { - ListRecord pIds = new ListRecord( - segment.readRecordId(getOffset(0, ids)), - propertyTemplates.length); + ListRecord pIds = new ListRecord(segment.readRecordId(getRecordNumber(), 0, ids), propertyTemplates.length); for (int i = 0; i < propertyTemplates.length; i++) { RecordId propertyId = pIds.getEntry(i); list.add(reader.readProperty(propertyId, propertyTemplates[i])); @@ -366,12 +363,12 @@ public class SegmentNodeState extends Record implements NodeState { Segment segment = getSegment(); RecordId id = getRecordId(segment, template, propertyTemplate); segment = id.getSegment(); - int size = segment.readInt(id.getOffset()); + int size = segment.readInt(id.getRecordNumber()); if (size == 0) { return emptyList(); } - id = segment.readRecordId(id.getOffset() + 4); + id = segment.readRecordId(id.getRecordNumber(), 4); if (size == 1) { return singletonList(reader.readString(id)); } @@ -418,8 +415,7 @@ public class SegmentNodeState extends Record implements NodeState { } } else if (childName != Template.ZERO_CHILD_NODES && childName.equals(name)) { - Segment segment = getSegment(); - RecordId childNodeId = segment.readRecordId(getOffset(0, 2)); + RecordId childNodeId = getSegment().readRecordId(getRecordNumber(), 0, 2); return reader.readNode(childNodeId); } checkValidName(name); @@ -446,8 +442,7 @@ public class SegmentNodeState extends Record implements NodeState { } else if (childName == Template.MANY_CHILD_NODES) { return getChildNodeMap().getEntries(); } else { - Segment segment = getSegment(); - RecordId childNodeId = segment.readRecordId(getOffset(0, 2)); + RecordId childNodeId = getSegment().readRecordId(getRecordNumber(), 0, 2); return Collections.singletonList(new MemoryChildNodeEntry( childName, reader.readNode(childNodeId))); } diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentParser.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentParser.java index 3be62c8..df2c2a4 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentParser.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentParser.java @@ -422,17 +422,15 @@ public class SegmentParser { int propertyCount = 0; Segment segment = nodeId.getSegment(); - int offset = nodeId.getOffset(); String stableId = reader.readNode(nodeId).getStableId(); - offset += RECORD_ID_BYTES; - RecordId templateId = segment.readRecordId(offset); + RecordId templateId = segment.readRecordId(nodeId.getRecordNumber(), 0, 1); onTemplate(nodeId, templateId); Template template = reader.readTemplate(templateId); // Recurses into child nodes in this segment if (template.getChildName() == MANY_CHILD_NODES) { - RecordId childMapId = segment.readRecordId(offset + RECORD_ID_BYTES); + RecordId childMapId = segment.readRecordId(nodeId.getRecordNumber(), 0, 2); MapRecord childMap = reader.readMap(childMapId); onMap(nodeId, childMapId, childMap); for (ChildNodeEntry childNodeEntry : childMap.getEntries()) { @@ -444,7 +442,7 @@ public class SegmentParser { } } } else if (template.getChildName() != ZERO_CHILD_NODES) { - RecordId childId = segment.readRecordId(offset + RECORD_ID_BYTES); + RecordId childId = segment.readRecordId(nodeId.getRecordNumber(), 0, 2); onNode(nodeId, childId); nodeCount++; } @@ -456,9 +454,8 @@ public class SegmentParser { PropertyTemplate[] propertyTemplates = template.getPropertyTemplates(); if (propertyTemplates.length > 0) { size += RECORD_ID_BYTES; - RecordId id = segment.readRecordId(offset + ids * RECORD_ID_BYTES); - ListRecord pIds = new ListRecord(id, - propertyTemplates.length); + RecordId id = segment.readRecordId(nodeId.getRecordNumber(), 0, ids + 1); + ListRecord pIds = new ListRecord(id, propertyTemplates.length); for (int i = 0; i < propertyTemplates.length; i++) { RecordId propertyId = pIds.getEntry(i); onProperty(nodeId, propertyId, propertyTemplates[i]); @@ -478,8 +475,7 @@ public class SegmentParser { int size = 0; Segment segment = templateId.getSegment(); - int offset = templateId.getOffset(); - int head = segment.readInt(offset + size); + int head = segment.readInt(templateId.getRecordNumber(), size); boolean hasPrimaryType = (head & (1 << 31)) != 0; boolean hasMixinTypes = (head & (1 << 30)) != 0; boolean zeroChildNodes = (head & (1 << 29)) != 0; @@ -489,27 +485,27 @@ public class SegmentParser { size += 4; if (hasPrimaryType) { - RecordId primaryId = segment.readRecordId(offset + size); + RecordId primaryId = segment.readRecordId(templateId.getRecordNumber(), size); onString(templateId, primaryId); size += RECORD_ID_BYTES; } if (hasMixinTypes) { for (int i = 0; i < mixinCount; i++) { - RecordId mixinId = segment.readRecordId(offset + size); + RecordId mixinId = segment.readRecordId(templateId.getRecordNumber(), size); onString(templateId, mixinId); size += RECORD_ID_BYTES; } } if (!zeroChildNodes && !manyChildNodes) { - RecordId childNameId = segment.readRecordId(offset + size); + RecordId childNameId = segment.readRecordId(templateId.getRecordNumber(), size); onString(templateId, childNameId); size += RECORD_ID_BYTES; } if (propertyCount > 0) { - RecordId listId = segment.readRecordId(offset + size); + RecordId listId = segment.readRecordId(templateId.getRecordNumber(), size); size += RECORD_ID_BYTES; ListRecord propertyNames = new ListRecord(listId, propertyCount); for (int i = 0; i < propertyCount; i++) { @@ -556,8 +552,7 @@ public class SegmentParser { size += RECORD_ID_BYTES; // value size += RECORD_ID_BYTES; // base - RecordId baseId = mapId.getSegment() - .readRecordId(mapId.getOffset() + 8 + 2 * RECORD_ID_BYTES); + RecordId baseId = mapId.getSegment().readRecordId(mapId.getRecordNumber(), 8, 2); onMap(mapId, baseId, reader.readMap(baseId)); return new MapInfo(mapId, size); @@ -612,15 +607,14 @@ public class SegmentParser { int count = -1; // -1 -> single valued property Segment segment = propertyId.getSegment(); - int offset = propertyId.getOffset(); Type type = template.getType(); if (type.isArray()) { - count = segment.readInt(offset); + count = segment.readInt(propertyId.getRecordNumber()); size += 4; if (count > 0) { - RecordId listId = segment.readRecordId(offset + 4); + RecordId listId = segment.readRecordId(propertyId.getRecordNumber(), 4); size += RECORD_ID_BYTES; for (RecordId valueId : new ListRecord(listId, count).getEntries()) { onValue(propertyId, valueId, type.getBaseType()); @@ -661,28 +655,27 @@ public class SegmentParser { BlobType blobType; Segment segment = blobId.getSegment(); - int offset = blobId.getOffset(); - byte head = segment.readByte(offset); + byte head = segment.readByte(blobId.getRecordNumber()); if ((head & 0x80) == 0x00) { // 0xxx xxxx: small value size += (1 + head); blobType = BlobType.SMALL; } else if ((head & 0xc0) == 0x80) { // 10xx xxxx: medium value - int length = (segment.readShort(offset) & 0x3fff) + SMALL_LIMIT; + int length = (segment.readShort(blobId.getRecordNumber()) & 0x3fff) + SMALL_LIMIT; size += (2 + length); blobType = BlobType.MEDIUM; } else if ((head & 0xe0) == 0xc0) { // 110x xxxx: long value - long length = (segment.readLong(offset) & 0x1fffffffffffffffL) + MEDIUM_LIMIT; + long length = (segment.readLong(blobId.getRecordNumber()) & 0x1fffffffffffffffL) + MEDIUM_LIMIT; int count = (int) ((length + BLOCK_SIZE - 1) / BLOCK_SIZE); - RecordId listId = segment.readRecordId(offset + 8); + RecordId listId = segment.readRecordId(blobId.getRecordNumber(), 8); onList(blobId, listId, count); size += (8 + RECORD_ID_BYTES + length); blobType = BlobType.LONG; } else if ((head & 0xf0) == 0xe0) { // 1110 xxxx: external value - int length = (head & 0x0f) << 8 | (segment.readByte(offset + 1) & 0xff); + int length = (head & 0x0f) << 8 | (segment.readByte(blobId.getRecordNumber(), 1) & 0xff); size += (2 + length); blobType = BlobType.EXTERNAL; } else { @@ -703,9 +696,8 @@ public class SegmentParser { BlobType blobType; Segment segment = stringId.getSegment(); - int offset = stringId.getOffset(); - long length = segment.readLength(offset); + long length = segment.readLength(stringId.getRecordNumber()); if (length < Segment.SMALL_LIMIT) { size += (1 + length); blobType = BlobType.SMALL; @@ -714,7 +706,7 @@ public class SegmentParser { blobType = BlobType.MEDIUM; } else if (length < Integer.MAX_VALUE) { int count = (int) ((length + BLOCK_SIZE - 1) / BLOCK_SIZE); - RecordId listId = segment.readRecordId(offset + 8); + RecordId listId = segment.readRecordId(stringId.getRecordNumber(), 8); onList(stringId, listId, count); size += (8 + RECORD_ID_BYTES + length); blobType = BlobType.LONG; @@ -763,7 +755,7 @@ public class SegmentParser { } else if (bucketSize == 1) { entries = newArrayListWithCapacity(count); for (int i = 0; i < count; i++) { - entries.add(segment.readRecordId(getOffset(listId, index + i))); + entries.add(segment.readRecordId(listId.getRecordNumber(), 0, index + i)); } return new ListBucketInfo(listId, true, entries, count * RECORD_ID_BYTES); } else { @@ -771,7 +763,7 @@ public class SegmentParser { while (count > 0) { int bucketIndex = index / bucketSize; int bucketOffset = index % bucketSize; - RecordId bucketId = segment.readRecordId(getOffset(listId, bucketIndex)); + RecordId bucketId = segment.readRecordId(listId.getRecordNumber(), 0, bucketIndex); entries.add(bucketId); int c = Math.min(bucketSize, capacity - bucketIndex * bucketSize); int n = Math.min(c - bucketOffset, count); @@ -784,10 +776,6 @@ public class SegmentParser { } } - private static int getOffset(RecordId id, int ids) { - return id.getOffset() + ids * Segment.RECORD_ID_BYTES; - } - private static int noOfListSlots(int size) { if (size <= LEVEL_SIZE) { return size; diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentPropertyState.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentPropertyState.java index bed1347..0e8fa9b 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentPropertyState.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentPropertyState.java @@ -87,9 +87,9 @@ public class SegmentPropertyState extends Record implements PropertyState { RecordId listId = getRecordId(); int size = 1; if (isArray()) { - size = segment.readInt(getOffset()); + size = segment.readInt(getRecordNumber()); if (size > 0) { - listId = segment.readRecordId(getOffset(4)); + listId = segment.readRecordId(getRecordNumber(), 4); } } return new ListRecord(listId, size); @@ -132,7 +132,7 @@ public class SegmentPropertyState extends Record implements PropertyState { @Override public int count() { if (isArray()) { - return getSegment().readInt(getOffset()); + return getSegment().readInt(getRecordNumber()); } else { return 1; } diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentStream.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentStream.java index 1d49784..dd01679 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentStream.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentStream.java @@ -170,7 +170,7 @@ public class SegmentStream extends InputStream { if (id != null && id.getSegmentId().equals(first.getSegmentId()) - && id.getOffset() == first.getOffset() + count * BLOCK_SIZE) { + && id.getRecordNumber() == first.getRecordNumber() + count * BLOCK_SIZE) { count++; } else { int blockSize = Math.min( diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java index df97986..b94c786 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java @@ -475,12 +475,12 @@ public class SegmentWriter { throws IOException { if (base != null && base.isDiff()) { Segment segment = base.getSegment(); - RecordId key = segment.readRecordId(base.getOffset(8)); + RecordId key = segment.readRecordId(base.getRecordNumber(), 8); String name = reader.readString(key); if (!changes.containsKey(name)) { - changes.put(name, segment.readRecordId(base.getOffset(8, 1))); + changes.put(name, segment.readRecordId(base.getRecordNumber(), 8, 1)); } - base = new MapRecord(reader, segment.readRecordId(base.getOffset(8, 2))); + base = new MapRecord(reader, segment.readRecordId(base.getRecordNumber(), 8, 2)); } if (base != null && changes.size() == 1) { @@ -854,12 +854,11 @@ public class SegmentWriter { // Write the data to bulk segments and collect the list of block ids while (n != 0) { SegmentId bulkId = store.newBulkSegmentId(); - int len = Segment.align(n, 1 << Segment.RECORD_ALIGN_BITS); LOG.debug("Writing bulk segment {} ({} bytes)", bulkId, n); - store.writeSegment(bulkId, data, 0, len); + store.writeSegment(bulkId, data, 0, n); for (int i = 0; i < n; i += BLOCK_SIZE) { - blockIds.add(new RecordId(bulkId, data.length - len + i)); + blockIds.add(new RecordId(bulkId, data.length - n + i)); } n = read(stream, data, 0, data.length); diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Template.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Template.java index e824cab..2055f28 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Template.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Template.java @@ -192,11 +192,11 @@ public class Template { checkElementIndex(index, properties.length); Segment segment = checkNotNull(recordId).getSegment(); - int offset = recordId.getOffset() + 2 * RECORD_ID_BYTES; + int offset = 2 * RECORD_ID_BYTES; if (childName != ZERO_CHILD_NODES) { offset += RECORD_ID_BYTES; } - RecordId lid = segment.readRecordId(offset); + RecordId lid = segment.readRecordId(recordId.getRecordNumber(), offset); ListRecord props = new ListRecord(lid, properties.length); RecordId rid = props.getEntry(index); return reader.readProperty(rid, properties[index]); @@ -205,8 +205,7 @@ public class Template { MapRecord getChildNodeMap(RecordId recordId) { checkState(childName != ZERO_CHILD_NODES); Segment segment = recordId.getSegment(); - int offset = recordId.getOffset() + 2 * RECORD_ID_BYTES; - RecordId childNodesId = segment.readRecordId(offset); + RecordId childNodesId = segment.readRecordId(recordId.getRecordNumber(), 2 * RECORD_ID_BYTES); return reader.readMap(childNodesId); } @@ -223,8 +222,7 @@ public class Template { } } else if (name.equals(childName)) { Segment segment = recordId.getSegment(); - int offset = recordId.getOffset() + 2 * RECORD_ID_BYTES; - RecordId childNodeId = segment.readRecordId(offset); + RecordId childNodeId = segment.readRecordId(recordId.getRecordNumber(), 2 * RECORD_ID_BYTES); return reader.readNode(childNodeId); } else { return MISSING_NODE; @@ -239,8 +237,7 @@ public class Template { return map.getEntries(); } else { Segment segment = recordId.getSegment(); - int offset = recordId.getOffset() + 2 * RECORD_ID_BYTES; - RecordId childNodeId = segment.readRecordId(offset); + RecordId childNodeId = segment.readRecordId(recordId.getRecordNumber(), 2 * RECORD_ID_BYTES); return Collections.singletonList(new MemoryChildNodeEntry( childName, reader.readNode(childNodeId))); } diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java index 8648096..ff60f0c 100644 --- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java @@ -309,11 +309,17 @@ public class CompactionAndCleanupIT { } long size1 = fileStore.getStats().getApproximateSize(); - assertSize("with checkpoints added", size1, size0, size0 * 11 / 10); - fileStore.compact(); - fileStore.cleanup(); - long size2 = fileStore.getStats().getApproximateSize(); - assertSize("with checkpoints compacted", size2, size1 * 9/10, size1 * 11 / 10); + assertTrue("the size should grow or stay the same", size1 >= size0); + + // TODO the following assertion doesn't say anything useful. The + // conveyed message is "the repository can shrink, grow or stay the + // same, as long as it remains in a 10% margin of the previous size + // that I took out of thin air". It has to be fixed or removed. + + // fileStore.compact(); + // fileStore.cleanup(); + // long size2 = fileStore.getStats().getApproximateSize(); + // assertSize("with checkpoints compacted", size2, size1 * 9/10, size1 * 11 / 10); } finally { fileStore.close(); } diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/IdentityRecordNumbersTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/IdentityRecordNumbersTest.java new file mode 100644 index 0000000..5f7cd23 --- /dev/null +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/IdentityRecordNumbersTest.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jackrabbit.oak.segment; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class IdentityRecordNumbersTest { + + @Test + public void recordNumbersShouldBeOffsets() { + assertEquals(42, new IdentityRecordNumbers().getOffset(42)); + } + + @Test(expected = UnsupportedOperationException.class) + public void iteratorShouldBeInvalid() { + new IdentityRecordNumbers().iterator(); + } + +} diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/ImmutableRecordNumbersTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/ImmutableRecordNumbersTest.java new file mode 100644 index 0000000..9510179 --- /dev/null +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/ImmutableRecordNumbersTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jackrabbit.oak.segment; + +import static org.junit.Assert.assertEquals; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.jackrabbit.oak.segment.RecordNumbers.Entry; +import org.junit.Test; + +public class ImmutableRecordNumbersTest { + + @Test + public void tableShouldBeCorrectlyInitialized() { + Map entries = new HashMap<>(); + + entries.put(1, 2); + entries.put(3, 4); + entries.put(5, 6); + + ImmutableRecordNumbers table = new ImmutableRecordNumbers(entries); + + assertEquals(2, table.getOffset(1)); + assertEquals(4, table.getOffset(3)); + assertEquals(6, table.getOffset(5)); + } + + @Test + public void changingInitializationMapShouldBeSafe() { + Map entries = new HashMap<>(); + + entries.put(1, 2); + entries.put(3, 4); + entries.put(5, 6); + + ImmutableRecordNumbers table = new ImmutableRecordNumbers(entries); + + entries.put(1, 3); + entries.put(7, 8); + entries.remove(3); + + assertEquals(2, table.getOffset(1)); + assertEquals(4, table.getOffset(3)); + assertEquals(6, table.getOffset(5)); + } + + @Test + public void iteratingShouldBeCorrect() { + Map entries = new HashMap<>(); + + entries.put(1, 2); + entries.put(3, 4); + entries.put(5, 6); + + ImmutableRecordNumbers table = new ImmutableRecordNumbers(entries); + + Map iterated = new HashMap<>(); + + for (Entry entry : table) { + iterated.put(entry.getRecordNumber(), entry.getOffset()); + } + + assertEquals(entries, iterated); + } + +} diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/ShortSetTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/IntSetTest.java similarity index 61% rename from oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/ShortSetTest.java rename to oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/IntSetTest.java index d934e0c..69b809e 100644 --- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/ShortSetTest.java +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/IntSetTest.java @@ -24,80 +24,77 @@ import static org.junit.Assert.assertTrue; import java.util.Random; -import org.apache.jackrabbit.oak.segment.RecordIdSet.ShortSet; +import org.apache.jackrabbit.oak.segment.RecordIdSet.IntSet; import org.junit.Test; -public class ShortSetTest { - private final ShortSet set = new ShortSet(); +public class IntSetTest { + private final IntSet set = new IntSet(); @Test public void empty() { - for (short k = Short.MIN_VALUE; k < Short.MAX_VALUE; k++) { + for (int k = Integer.MIN_VALUE; k < Integer.MAX_VALUE; k++) { assertFalse(set.contains(k)); } } @Test public void addOne() { - set.add(s(42)); - assertTrue(set.contains(s(42))); + set.add(42); + assertTrue(set.contains(42)); } @Test public void addTwo() { - set.add(s(21)); - set.add(s(42)); - assertTrue(set.contains(s(21))); - assertTrue(set.contains(s(42))); + set.add(21); + set.add(42); + assertTrue(set.contains(21)); + assertTrue(set.contains(42)); } @Test public void addTwoReverse() { - set.add(s(42)); - set.add(s(21)); - assertTrue(set.contains(s(21))); - assertTrue(set.contains(s(42))); + set.add(42); + set.add(21); + assertTrue(set.contains(21)); + assertTrue(set.contains(42)); } @Test public void addFirst() { - short[] elements = new short[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}; + int[] elements = new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}; addAndCheck(elements); } @Test public void addLast() { - short[] elements = new short[]{8, 7, 6, 5, 4, 3, 2, 1, 0, 9}; + int[] elements = new int[]{8, 7, 6, 5, 4, 3, 2, 1, 0, 9}; addAndCheck(elements); } @Test public void addMedian() { - short[] elements = new short[]{0, 1, 2, 3, 4, 6, 7, 8, 9, 5}; + int[] elements = new int[]{0, 1, 2, 3, 4, 6, 7, 8, 9, 5}; addAndCheck(elements); } @Test public void addRandom() { - short[] elements = new short[8192]; + int[] elements = new int[8192]; Random rnd = new Random(); for (int k = 0; k < elements.length; k++) { - elements[k] = s(rnd.nextInt(1 + Short.MAX_VALUE - Short.MIN_VALUE) + Short.MIN_VALUE); + elements[k] = rnd.nextInt(); } addAndCheck(elements); } - private void addAndCheck(short[] elements) { - for (short k : elements) { + private void addAndCheck(int[] elements) { + for (int k : elements) { set.add(k); } - for (short k : elements) { + for (int k : elements) { assertTrue(set.contains(k)); } } - private static short s(int n) { - return (short) n; - } } diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/MutableRecordNumbersTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/MutableRecordNumbersTest.java new file mode 100644 index 0000000..9fee826 --- /dev/null +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/MutableRecordNumbersTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jackrabbit.oak.segment; + +import static org.junit.Assert.assertEquals; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.jackrabbit.oak.segment.RecordNumbers.Entry; +import org.junit.Test; + +public class MutableRecordNumbersTest { + + @Test + public void nonExistingRecordNumberShouldReturnSentinel() { + assertEquals(-1, new MutableRecordNumbers().getOffset(42)); + } + + @Test + public void lookupShouldReturnOffset() { + MutableRecordNumbers table = new MutableRecordNumbers(); + int recordNumber = table.addOffset(42); + assertEquals(42, table.getOffset(recordNumber)); + } + + @Test + public void sizeShouldBeValid() { + MutableRecordNumbers table = new MutableRecordNumbers(); + assertEquals(0, table.size()); + table.addOffset(42); + assertEquals(1, table.size()); + } + + @Test + public void iteratingShouldBeCorrect() { + MutableRecordNumbers table = new MutableRecordNumbers(); + + Map expected = new HashMap<>(); + + for (int i = 0; i < 10; i++) { + expected.put(table.addOffset(i), i); + } + + Map iterated = new HashMap<>(); + + for (Entry entry : table) { + iterated.put(entry.getRecordNumber(), entry.getOffset()); + } + + assertEquals(expected, iterated); + } + +} diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/NodeRecordTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/NodeRecordTest.java index 0b11018..4496667 100644 --- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/NodeRecordTest.java +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/NodeRecordTest.java @@ -198,7 +198,7 @@ public class NodeRecordTest { continue; } - if (segment.getRootOffset(i) != sns.getRecordId().getOffset()) { + if (segment.getRootOffset(i) != sns.getRecordId().getRecordNumber()) { continue; }