Index: src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyser.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyser.java (revision 1660633) +++ src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyser.java (working copy) @@ -32,6 +32,7 @@ import static org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.BLOCK_SIZE; import static org.apache.jackrabbit.oak.plugins.segment.Template.MANY_CHILD_NODES; import static org.apache.jackrabbit.oak.plugins.segment.Template.ZERO_CHILD_NODES; +import static org.apache.jackrabbit.oak.plugins.segment.SegmentVersion.V_11; import java.util.Formatter; import java.util.Map; @@ -227,10 +228,25 @@ int ids = template.getChildName() == ZERO_CHILD_NODES ? 1 : 2; nodeSize += ids * RECORD_ID_BYTES; PropertyTemplate[] propertyTemplates = template.getPropertyTemplates(); - for (PropertyTemplate propertyTemplate : propertyTemplates) { - nodeSize += RECORD_ID_BYTES; - RecordId propertyId = segment.readRecordId(offset + ids++ * RECORD_ID_BYTES); - analyseProperty(propertyId, propertyTemplate); + if (segment.getStorageFormatVersion().onOrAfter(V_11)) { + if (propertyTemplates.length > 0) { + nodeSize += RECORD_ID_BYTES; + RecordId id = segment.readRecordId(offset + ids * RECORD_ID_BYTES); + ListRecord pIds = new ListRecord(id, + propertyTemplates.length); + for (int i = 0; i < propertyTemplates.length; i++) { + RecordId propertyId = pIds.getEntry(i); + analyseProperty(propertyId, propertyTemplates[i]); + } + analyseList(id, propertyTemplates.length); + } + } else { + for (PropertyTemplate propertyTemplate : propertyTemplates) { + nodeSize += RECORD_ID_BYTES; + RecordId propertyId = segment.readRecordId(offset + ids++ + * RECORD_ID_BYTES); + analyseProperty(propertyId, propertyTemplate); + } } } } @@ -238,6 +254,7 @@ @Override public String toString() { StringBuilder sb = new StringBuilder(); + @SuppressWarnings("resource") Formatter formatter = new Formatter(sb); formatter.format( "%s in maps (%s leaf and branch records)%n", @@ -295,11 +312,26 @@ size += Segment.RECORD_ID_BYTES; } - for (int i = 0; i < propertyCount; i++) { - RecordId propertyNameId = segment.readRecordId(offset + size); - size += Segment.RECORD_ID_BYTES; - size++; // type - analyseString(propertyNameId); + if (segment.getStorageFormatVersion().onOrAfter(V_11)) { + if (propertyCount > 0) { + RecordId listId = segment.readRecordId(offset + size); + ListRecord propertyNames = new ListRecord(listId, + propertyCount); + for (int i = 0; i < propertyCount; i++) { + RecordId propertyNameId = propertyNames.getEntry(i); + size += Segment.RECORD_ID_BYTES; + size++; // type + analyseString(propertyNameId); + } + analyseList(listId, propertyCount); + } + } else { + for (int i = 0; i < propertyCount; i++) { + RecordId propertyNameId = segment.readRecordId(offset + size); + size += Segment.RECORD_ID_BYTES; + size++; // type + analyseString(propertyNameId); + } } templateSize += size; } Index: src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java (revision 1660633) +++ src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java (working copy) @@ -22,6 +22,7 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.Lists.newArrayListWithCapacity; import static com.google.common.collect.Maps.newConcurrentMap; +import static org.apache.jackrabbit.oak.plugins.segment.SegmentVersion.V_11; import static org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.BLOCK_SIZE; import java.io.IOException; @@ -52,14 +53,6 @@ public class Segment { /** - * Version of the segment storage format. - * - */ - public static final byte STORAGE_FORMAT_VERSION = 10; - - /** * Number of bytes used for storing a record identifier. One byte * is used for identifying the segment and two for the record offset * within that segment. @@ -117,6 +110,11 @@ private final ByteBuffer data; /** + * Version of the segment storage format. + */ + private final SegmentVersion version; + + /** * Referenced segment identifiers. Entries are initialized lazily in * {@link #getRefId(int)}. Set to {@code null} for bulk segments. */ @@ -137,19 +135,25 @@ private volatile long accessed = 0; public Segment(SegmentTracker tracker, SegmentId id, ByteBuffer data) { + this(tracker, id, data, V_11); + } + + public Segment(SegmentTracker tracker, SegmentId id, ByteBuffer data, SegmentVersion version) { this.tracker = checkNotNull(tracker); this.id = checkNotNull(id); this.data = checkNotNull(data); - if (id.isDataSegmentId()) { + byte segmentFormat = data.get(3); checkState(data.get(0) == '0' && data.get(1) == 'a' && data.get(2) == 'K' - && data.get(3) == STORAGE_FORMAT_VERSION); + && SegmentVersion.isValid(segmentFormat)); this.refids = new SegmentId[getRefCount()]; - refids[0] = id; + this.refids[0] = id; + this.version = SegmentVersion.fromByte(segmentFormat); } else { this.refids = null; + this.version = version; } } @@ -157,9 +161,9 @@ this.tracker = checkNotNull(tracker); this.id = tracker.newDataSegmentId(); this.data = ByteBuffer.wrap(checkNotNull(buffer)); - this.refids = new SegmentId[SEGMENT_REFERENCE_LIMIT + 1]; - refids[0] = id; + this.refids[0] = id; + this.version = SegmentVersion.fromByte(buffer[3]); } void access() { @@ -171,6 +175,10 @@ return accessed != 0; } + SegmentVersion getStorageFormatVersion() { + return version; + } + /** * Maps the given record offset to the respective position within the * internal {@link #data} array. The validity of a record with the given @@ -436,19 +444,41 @@ offset += Segment.RECORD_ID_BYTES; } - PropertyTemplate[] properties = - new PropertyTemplate[propertyCount]; - for (int i = 0; i < properties.length; i++) { + PropertyTemplate[] properties; + if (version.onOrAfter(V_11)) { + properties = readPropsV11(propertyCount, offset); + } else { + properties = readPropsV10(propertyCount, offset); + } + return new Template(primaryType, mixinTypes, properties, childName); + } + + private PropertyTemplate[] readPropsV10(int propertyCount, int offset) { + PropertyTemplate[] properties = new PropertyTemplate[propertyCount]; + for (int i = 0; i < propertyCount; i++) { RecordId propertyNameId = readRecordId(offset); offset += Segment.RECORD_ID_BYTES; byte type = readByte(offset++); - properties[i] = new PropertyTemplate( - i, readString(propertyNameId), + properties[i] = new PropertyTemplate(i, readString(propertyNameId), Type.fromTag(Math.abs(type), type < 0)); } + return properties; + } - return new Template( - primaryType, mixinTypes, properties, childName); + private PropertyTemplate[] readPropsV11(int propertyCount, int offset) { + PropertyTemplate[] properties = new PropertyTemplate[propertyCount]; + if (propertyCount > 0) { + RecordId id = readRecordId(offset); + ListRecord propertyNames = new ListRecord(id, properties.length); + offset += Segment.RECORD_ID_BYTES; + for (int i = 0; i < propertyCount; i++) { + byte type = readByte(offset++); + properties[i] = new PropertyTemplate(i, + readString(propertyNames.getEntry(i)), Type.fromTag( + Math.abs(type), type < 0)); + } + } + return properties; } long readLength(RecordId id) { Index: src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeState.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeState.java (revision 1660633) +++ src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeState.java (working copy) @@ -46,6 +46,7 @@ import static org.apache.jackrabbit.oak.api.Type.STRINGS; import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.MISSING_NODE; +import static org.apache.jackrabbit.oak.plugins.segment.SegmentVersion.V_11; import static org.apache.jackrabbit.oak.spi.state.AbstractNodeState.checkValidName; /** @@ -134,17 +135,39 @@ template.getPropertyTemplate(name); if (propertyTemplate != null) { Segment segment = getSegment(); - int ids = 1 + propertyTemplate.getIndex(); - if (template.getChildName() != Template.ZERO_CHILD_NODES) { - ids++; + RecordId id; + if (getSegment().getStorageFormatVersion().onOrAfter(V_11)) { + id = getRecordIdV11(segment, template, propertyTemplate); + } else { + id = getRecordIdV10(segment, template, propertyTemplate); } - return new SegmentPropertyState( - segment.readRecordId(getOffset(0, ids)), propertyTemplate); + return new SegmentPropertyState(id, propertyTemplate); } else { return null; } } + private RecordId getRecordIdV10(Segment segment, Template template, + PropertyTemplate propertyTemplate) { + int ids = 1 + propertyTemplate.getIndex(); + if (template.getChildName() != Template.ZERO_CHILD_NODES) { + ids++; + } + return segment.readRecordId(getOffset(0, ids)); + } + + private RecordId getRecordIdV11(Segment segment, Template template, + PropertyTemplate propertyTemplate) { + int ids = 1; + if (template.getChildName() != Template.ZERO_CHILD_NODES) { + ids++; + } + RecordId rid = segment.readRecordId(getOffset(0, ids)); + ListRecord pIds = new ListRecord(rid, + template.getPropertyTemplates().length); + return pIds.getEntry(propertyTemplate.getIndex()); + } + @Override @Nonnull public Iterable getProperties() { Template template = getTemplate(); @@ -167,10 +190,24 @@ if (template.getChildName() != Template.ZERO_CHILD_NODES) { ids++; } - for (int i = 0; i < propertyTemplates.length; i++) { - RecordId propertyId = segment.readRecordId(getOffset(0, ids++)); - list.add(new SegmentPropertyState( - propertyId, propertyTemplates[i])); + + if (segment.getStorageFormatVersion().onOrAfter(V_11)) { + if (propertyTemplates.length > 0) { + ListRecord pIds = new ListRecord( + segment.readRecordId(getOffset(0, ids)), + propertyTemplates.length); + for (int i = 0; i < propertyTemplates.length; i++) { + RecordId propertyId = pIds.getEntry(i); + list.add(new SegmentPropertyState(propertyId, + propertyTemplates[i])); + } + } + } else { + for (int i = 0; i < propertyTemplates.length; i++) { + RecordId propertyId = segment.readRecordId(getOffset(0, ids++)); + list.add(new SegmentPropertyState(propertyId, + propertyTemplates[i])); + } } return list; @@ -247,11 +284,13 @@ } Segment segment = getSegment(); - int ids = 1 + propertyTemplate.getIndex(); - if (template.getChildName() != Template.ZERO_CHILD_NODES) { - ids++; + RecordId id; + if (getSegment().getStorageFormatVersion().onOrAfter(V_11)) { + id = getRecordIdV11(segment, template, propertyTemplate); + } else { + id = getRecordIdV10(segment, template, propertyTemplate); } - return segment.readString(segment.readRecordId(getOffset(0, ids))); + return segment.readString(id); } /** @@ -288,12 +327,12 @@ } Segment segment = getSegment(); - int ids = 1 + propertyTemplate.getIndex(); - if (template.getChildName() != Template.ZERO_CHILD_NODES) { - ids++; + RecordId id; + if (getSegment().getStorageFormatVersion().onOrAfter(V_11)) { + id = getRecordIdV11(segment, template, propertyTemplate); + } else { + id = getRecordIdV10(segment, template, propertyTemplate); } - - RecordId id = segment.readRecordId(getOffset(0, ids)); segment = id.getSegment(); int size = segment.readInt(id.getOffset()); if (size == 0) { Index: src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java (revision 1660633) +++ src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java (working copy) @@ -93,20 +93,25 @@ private long currentSize = 0; - public SegmentTracker(SegmentStore store, int cacheSizeMB) { + public SegmentTracker(SegmentStore store, int cacheSizeMB, + SegmentVersion version) { for (int i = 0; i < tables.length; i++) { tables[i] = new SegmentIdTable(this); } this.store = store; - this.writer = new SegmentWriter(store, this); + this.writer = new SegmentWriter(store, this, version); this.cacheSize = cacheSizeMB * MB; this.compactionMap = new AtomicReference( new CompactionMap(1, this)); } + public SegmentTracker(SegmentStore store, SegmentVersion version) { + this(store, DEFAULT_MEMORY_CACHE_SIZE, version); + } + public SegmentTracker(SegmentStore store) { - this(store, DEFAULT_MEMORY_CACHE_SIZE); + this(store, DEFAULT_MEMORY_CACHE_SIZE, SegmentVersion.V_11); } public SegmentWriter getWriter() { Index: src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersion.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersion.java (revision 0) +++ src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersion.java (revision 0) @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.segment; + +/** + * Version of the segment storage format. + * + */ +public enum SegmentVersion { + + /** + * @deprecated Use latest version V11 + */ + @Deprecated + V_10, + + V_11; + + public boolean onOrAfter(SegmentVersion other) { + return compareTo(other) >= 0; + } + + public static byte asByte(SegmentVersion v) { + switch (v) { + case V_11: + return 11; + case V_10: + return 10; + } + throw new IllegalArgumentException("Unwnown version " + v); + } + + public static SegmentVersion fromByte(byte v) { + switch (v) { + case 11: + return V_11; + case 10: + return V_10; + } + throw new IllegalArgumentException("Unwnown version " + v); + } + + public static boolean isValid(byte v) { + return v == 10 || v == 11; + } + +} Index: src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java (revision 1660633) +++ src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java (working copy) @@ -38,7 +38,7 @@ import static org.apache.jackrabbit.oak.plugins.segment.Segment.MAX_SEGMENT_SIZE; import static org.apache.jackrabbit.oak.plugins.segment.Segment.RECORD_ID_BYTES; import static org.apache.jackrabbit.oak.plugins.segment.Segment.SEGMENT_REFERENCE_LIMIT; -import static org.apache.jackrabbit.oak.plugins.segment.Segment.STORAGE_FORMAT_VERSION; +import static org.apache.jackrabbit.oak.plugins.segment.SegmentVersion.V_11; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -90,12 +90,12 @@ static final int BLOCK_SIZE = 1 << 12; // 4kB - static byte[] createNewBuffer() { + static byte[] createNewBuffer(SegmentVersion v) { byte[] buffer = new byte[Segment.MAX_SEGMENT_SIZE]; buffer[0] = '0'; buffer[1] = 'a'; buffer[2] = 'K'; - buffer[3] = STORAGE_FORMAT_VERSION; + buffer[3] = SegmentVersion.asByte(v); buffer[4] = 0; // reserved buffer[5] = 0; // refcount return buffer; @@ -119,6 +119,7 @@ * avoid storing duplicates of frequently occurring data. * Should only be accessed from synchronized blocks to prevent corruption. */ + @SuppressWarnings("serial") private final Map records = new LinkedHashMap(15000, 0.75f, true) { @Override @@ -142,7 +143,7 @@ * The segment write buffer, filled from the end to the beginning * (see OAK-629). */ - private byte[] buffer = createNewBuffer(); + private byte[] buffer; /** * The number of bytes already written (or allocated). Counted from @@ -158,9 +159,16 @@ private Segment segment; - public SegmentWriter(SegmentStore store, SegmentTracker tracker) { + /** + * Version of the segment storage format. + */ + private final SegmentVersion version; + + public SegmentWriter(SegmentStore store, SegmentTracker tracker, SegmentVersion version) { this.store = store; this.tracker = tracker; + this.version = version; + this.buffer = createNewBuffer(version); this.segment = new Segment(tracker, buffer); segment.getSegmentId().setSegment(segment); } @@ -240,7 +248,7 @@ } tracker.setSegment(id, new Segment(tracker, id, data)); - buffer = createNewBuffer(); + buffer = createNewBuffer(version); roots.clear(); blobrefs.clear(); length = 0; @@ -444,7 +452,6 @@ } } - private synchronized RecordId writeListBucket(List bucket) { checkArgument(bucket.size() > 1); RecordId bucketId = prepare(RecordType.BUCKET, 0, bucket); @@ -968,7 +975,17 @@ propertyTypes[i] = (byte) type.tag(); } } - ids.addAll(Arrays.asList(propertyNames)); + + RecordId propNamesId = null; + if (segment.getStorageFormatVersion().onOrAfter(V_11)) { + if (propertyNames.length > 0) { + propNamesId = writeList(Arrays.asList(propertyNames)); + ids.add(propNamesId); + } + } else { + ids.addAll(Arrays.asList(propertyNames)); + } + checkState(propertyNames.length < (1 << 18)); head |= propertyNames.length; @@ -985,8 +1002,16 @@ if (childNameId != null) { writeRecordId(childNameId); } + if (segment.getStorageFormatVersion().onOrAfter(V_11)) { + if (propNamesId != null) { + writeRecordId(propNamesId); + } + } for (int i = 0; i < propertyNames.length; i++) { - writeRecordId(propertyNames[i]); + if (!segment.getStorageFormatVersion().onOrAfter(V_11)) { + // V10 only + writeRecordId(propertyNames[i]); + } buffer[position++] = propertyTypes[i]; } @@ -1087,36 +1112,46 @@ ids.add(writeNode(state.getChildNode(template.getChildName())).getRecordId()); } + List pIds = Lists.newArrayList(); for (PropertyTemplate pt : template.getPropertyTemplates()) { String name = pt.getName(); PropertyState property = state.getProperty(name); if (property instanceof SegmentPropertyState && store.containsSegment(((SegmentPropertyState) property).getRecordId().getSegmentId())) { - ids.add(((SegmentPropertyState) property).getRecordId()); + pIds.add(((SegmentPropertyState) property).getRecordId()); } else if (before == null || !store.containsSegment(before.getRecordId().getSegmentId())) { - ids.add(writeProperty(property)); + pIds.add(writeProperty(property)); } else { // reuse previously stored property, if possible PropertyTemplate bt = beforeTemplate.getPropertyTemplate(name); if (bt == null) { - ids.add(writeProperty(property)); // new property + RecordId p = writeProperty(property); + pIds.add(p); // new property } else { SegmentPropertyState bp = beforeTemplate.getProperty( before.getRecordId(), bt.getIndex()); if (property.equals(bp)) { - ids.add(bp.getRecordId()); // no changes + pIds.add(bp.getRecordId()); // no changes } else if (bp.isArray() && bp.getType() != BINARIES) { // reuse entries from the previous list - ids.add(writeProperty(property, bp.getValueRecords())); + pIds.add(writeProperty(property, bp.getValueRecords())); } else { - ids.add(writeProperty(property)); + pIds.add(writeProperty(property)); } } } } + if (!pIds.isEmpty()) { + if (segment.getStorageFormatVersion().onOrAfter(V_11)) { + ids.add(writeList(pIds)); + } else { + ids.addAll(pIds); + } + } + synchronized (this) { RecordId recordId = prepare(RecordType.NODE, 0, ids); for (RecordId id : ids) { Index: src/main/java/org/apache/jackrabbit/oak/plugins/segment/Template.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/segment/Template.java (revision 1660633) +++ src/main/java/org/apache/jackrabbit/oak/plugins/segment/Template.java (working copy) @@ -22,6 +22,7 @@ import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.MISSING_NODE; import static org.apache.jackrabbit.oak.plugins.segment.Record.fastEquals; import static org.apache.jackrabbit.oak.plugins.segment.Segment.RECORD_ID_BYTES; +import static org.apache.jackrabbit.oak.plugins.segment.SegmentVersion.V_11; import java.util.Arrays; import java.util.Collections; @@ -173,9 +174,16 @@ if (childName != ZERO_CHILD_NODES) { offset += RECORD_ID_BYTES; } - offset += index * RECORD_ID_BYTES; - return new SegmentPropertyState( - segment.readRecordId(offset), properties[index]); + RecordId rid = null; + if (segment.getStorageFormatVersion().onOrAfter(V_11)) { + RecordId lid = segment.readRecordId(offset); + ListRecord props = new ListRecord(lid, properties.length); + rid = props.getEntry(index); + } else { + offset += index * RECORD_ID_BYTES; + rid = segment.readRecordId(offset); + } + return new SegmentPropertyState(rid, properties[index]); } MapRecord getChildNodeMap(RecordId recordId) { Index: src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (revision 1660633) +++ src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (working copy) @@ -52,6 +52,7 @@ import com.google.common.base.Stopwatch; import com.google.common.collect.Maps; + import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob; import org.apache.jackrabbit.oak.plugins.segment.CompactionMap; @@ -64,6 +65,7 @@ import org.apache.jackrabbit.oak.plugins.segment.SegmentNotFoundException; import org.apache.jackrabbit.oak.plugins.segment.SegmentStore; import org.apache.jackrabbit.oak.plugins.segment.SegmentTracker; +import org.apache.jackrabbit.oak.plugins.segment.SegmentVersion; import org.apache.jackrabbit.oak.plugins.segment.SegmentWriter; import org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy; import org.apache.jackrabbit.oak.spi.blob.BlobStore; @@ -151,6 +153,11 @@ */ private final LinkedList toBeRemoved = newLinkedList(); + /** + * Version of the segment storage format. + */ + private final SegmentVersion version = SegmentVersion.V_11; + public FileStore(BlobStore blobStore, File directory, int maxFileSizeMB, boolean memoryMapping) throws IOException { this(blobStore, directory, EMPTY_NODE, maxFileSizeMB, 0, memoryMapping); @@ -188,11 +195,11 @@ throws IOException { checkNotNull(directory).mkdirs(); if (cacheSizeMB < 0) { - this.tracker = new SegmentTracker(this, 0); + this.tracker = new SegmentTracker(this, 0, getVersion()); } else if (cacheSizeMB > 0) { - this.tracker = new SegmentTracker(this, cacheSizeMB); + this.tracker = new SegmentTracker(this, cacheSizeMB, getVersion()); } else { - this.tracker = new SegmentTracker(this); + this.tracker = new SegmentTracker(this, getVersion()); } this.blobStore = blobStore; this.directory = directory; @@ -537,7 +544,7 @@ log.info("TarMK compaction running, strategy={}", compactionStrategy); long start = System.currentTimeMillis(); - SegmentWriter writer = new SegmentWriter(this, tracker); + SegmentWriter writer = new SegmentWriter(this, tracker, getVersion()); final Compactor compactor = new Compactor(writer, compactionStrategy.cloneBinaries()); SegmentNodeState before = getHead(); long existing = before.getChildNode(SegmentNodeStore.CHECKPOINTS) @@ -613,6 +620,7 @@ flush(); writer.close(); + tracker.getWriter().dropCache(); List list = readers; readers = newArrayList(); @@ -885,4 +893,8 @@ } } } + + public SegmentVersion getVersion() { + return version; + } } Index: src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyserTest.java =================================================================== --- src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyserTest.java (revision 1660633) +++ src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyserTest.java (working copy) @@ -29,6 +29,7 @@ import static org.apache.jackrabbit.oak.plugins.segment.ListRecord.LEVEL_SIZE; import static org.apache.jackrabbit.oak.plugins.segment.Segment.MEDIUM_LIMIT; import static org.apache.jackrabbit.oak.plugins.segment.Segment.SMALL_LIMIT; +import static org.apache.jackrabbit.oak.plugins.segment.SegmentVersion.V_11; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -52,7 +53,7 @@ store = mock(SegmentStore.class); SegmentTracker tracker = new SegmentTracker(store); when(store.getTracker()).thenReturn(tracker); - writer = new SegmentWriter(store, store.getTracker()); + writer = new SegmentWriter(store, store.getTracker(), V_11); analyser = new RecordUsageAnalyser(); } @@ -270,7 +271,7 @@ SegmentNodeState node = writer.writeNode(builder.getNodeState()); analyser.analyseNode(node.getRecordId()); - assertCounts(analyser, 1, 3, 6, 1, 1, 1, 0, 10, 1, 1, 2, 3); + assertCounts(analyser, 1, 5, 6, 1, 1, 1, 0, 10, 1, 1, 2, 3); } private static Blob createRandomBlob(int size) { Index: src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentIdFactoryTest.java =================================================================== --- src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentIdFactoryTest.java (revision 1660633) +++ src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentIdFactoryTest.java (working copy) @@ -90,7 +90,7 @@ @Test(expected = IllegalStateException.class) public void dataAIOOBE() { SegmentId id = factory.newDataSegmentId(); - byte[] buffer = SegmentWriter.createNewBuffer(); + byte[] buffer = SegmentWriter.createNewBuffer(SegmentVersion.V_11); ByteBuffer data = ByteBuffer.allocate(Segment.MAX_SEGMENT_SIZE); data.put(buffer); data.rewind(); Index: src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java =================================================================== --- src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java (revision 1660633) +++ src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java (working copy) @@ -51,8 +51,8 @@ builder = EMPTY_NODE.builder(); builder.setProperty("foo", "bar"); builder.setProperty("baz", 123); - assertEquals(64, getSize(builder)); - assertEquals(12, getAmortizedSize(builder)); + assertEquals(80, getSize(builder)); + assertEquals(16, getAmortizedSize(builder)); builder = EMPTY_NODE.builder(); builder.child("foo"); @@ -127,15 +127,15 @@ deny.setProperty(PropertyStates.createProperty( "rep:privileges", ImmutableList.of("jcr:read"), Type.NAMES)); assertEquals(176, getSize(builder)); - assertEquals(28, getAmortizedSize(builder)); + assertEquals(32, getAmortizedSize(builder)); NodeBuilder allow = builder.child("allow"); allow.setProperty("jcr:primaryType", "rep:GrantACE"); allow.setProperty("rep:principalName", "administrators"); allow.setProperty(PropertyStates.createProperty( "rep:privileges", ImmutableList.of("jcr:all"), Type.NAMES)); - assertEquals(288, getSize(builder)); - assertEquals(72, getAmortizedSize(builder)); + assertEquals(320, getSize(builder)); + assertEquals(84, getAmortizedSize(builder)); NodeBuilder deny0 = builder.child("deny0"); deny0.setProperty("jcr:primaryType", "rep:DenyACE", Type.NAME); @@ -143,16 +143,16 @@ deny0.setProperty("rep:glob", "*/activities/*"); builder.setProperty(PropertyStates.createProperty( "rep:privileges", ImmutableList.of("jcr:read"), Type.NAMES)); - assertEquals(384, getSize(builder)); - assertEquals(108, getAmortizedSize(builder)); + assertEquals(416, getSize(builder)); + assertEquals(124, getAmortizedSize(builder)); NodeBuilder allow0 = builder.child("allow0"); allow0.setProperty("jcr:primaryType", "rep:GrantACE"); allow0.setProperty("rep:principalName", "user-administrators"); allow0.setProperty(PropertyStates.createProperty( "rep:privileges", ImmutableList.of("jcr:all"), Type.NAMES)); - assertEquals(432, getSize(builder)); - assertEquals(136, getAmortizedSize(builder)); + assertEquals(480, getSize(builder)); + assertEquals(160, getAmortizedSize(builder)); } @Test Index: src/test/java/org/apache/jackrabbit/oak/plugins/segment/VersionTest.java =================================================================== --- src/test/java/org/apache/jackrabbit/oak/plugins/segment/VersionTest.java (revision 0) +++ src/test/java/org/apache/jackrabbit/oak/plugins/segment/VersionTest.java (revision 0) @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.segment; + +import static org.apache.commons.io.FileUtils.deleteDirectory; +import static org.apache.jackrabbit.oak.api.Type.LONG; +import static org.apache.jackrabbit.oak.api.Type.LONGS; +import static org.apache.jackrabbit.oak.api.Type.STRING; +import static org.apache.jackrabbit.oak.api.Type.STRINGS; +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.io.IOException; + +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.plugins.segment.file.FileStore; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EmptyHook; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +public class VersionTest { + + private static final Logger log = LoggerFactory + .getLogger(VersionTest.class); + + private File directory; + + @Before + public void setUp() throws IOException { + directory = File.createTempFile("VersionTest", "dir", + new File("target")); + directory.delete(); + directory.mkdir(); + } + + @After + public void cleanDir() { + try { + deleteDirectory(directory); + } catch (IOException e) { + log.error("Error cleaning directory", e); + } + } + + @Test + public void readOldVersions() throws Exception { + FileStore fileStoreV10 = new FileStore(directory, 1) { + + @SuppressWarnings("deprecation") + @Override + public SegmentVersion getVersion() { + return SegmentVersion.V_10; + } + }; + try { + addTestContent(fileStoreV10); + } finally { + fileStoreV10.close(); + } + + FileStore fileStoreV11 = new FileStore(directory, 1); + try { + verifyContent(fileStoreV11); + } finally { + fileStoreV11.close(); + } + + } + + @SuppressWarnings("deprecation") + private static void addTestContent(FileStore fs) + throws CommitFailedException { + NodeStore store = new SegmentNodeStore(fs); + NodeBuilder builder = store.getRoot().builder(); + + NodeBuilder content = builder.child("content"); + content.setProperty("a", 1); + content.setProperty("aM", ImmutableList.of(1L, 2L, 3L, 4L), LONGS); + + content.setProperty("b", "azerty"); + content.setProperty("bM", + ImmutableList.of("a", "z", "e", "r", "t", "y"), STRINGS); + + // add blobs? + + store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY); + assertEquals(SegmentVersion.V_10, fs.getHead().getSegment() + .getStorageFormatVersion()); + } + + private static void verifyContent(FileStore fs) + throws CommitFailedException { + NodeStore store = new SegmentNodeStore(fs); + SegmentNodeState content = (SegmentNodeState) store.getRoot() + .getChildNode("content"); + + assertEquals(new Long(1), content.getProperty("a").getValue(LONG)); + assertEquals(ImmutableList.of(1L, 2L, 3L, 4L), + Lists.newArrayList(content.getProperty("aM").getValue(LONGS))); + + assertEquals("azerty", content.getProperty("b").getValue(STRING)); + assertEquals("azerty", content.getString("b")); + + assertEquals(ImmutableList.of("a", "z", "e", "r", "t", "y"), + Lists.newArrayList(content.getProperty("bM").getValue(STRINGS))); + assertEquals(ImmutableList.of("a", "z", "e", "r", "t", "y"), + Lists.newArrayList(content.getStrings("bM"))); + } +} Index: src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreTest.java =================================================================== --- src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreTest.java (revision 1660633) +++ src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreTest.java (working copy) @@ -21,6 +21,7 @@ import static junit.framework.Assert.assertEquals; import static junit.framework.Assert.assertFalse; import static junit.framework.Assert.assertTrue; +import static org.apache.jackrabbit.oak.plugins.segment.SegmentVersion.V_11; import java.io.ByteArrayInputStream; import java.io.File; @@ -139,7 +140,7 @@ store = new FileStore(directory, 1, false); head = store.getHead(); assertTrue(store.size() > largeBinarySize); - writer = new SegmentWriter(store, store.getTracker()); + writer = new SegmentWriter(store, store.getTracker(), V_11); compactor = new Compactor(writer); compacted = compactor.compact(EmptyNodeState.EMPTY_NODE, head); builder = head.builder(); Index: src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/LargeNumberOfPropertiesTestIT.java =================================================================== --- src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/LargeNumberOfPropertiesTestIT.java (revision 0) +++ src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/LargeNumberOfPropertiesTestIT.java (revision 0) @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.segment.file; + +import java.io.File; +import java.io.IOException; + +import org.apache.commons.io.FileUtils; +import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EmptyHook; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LargeNumberOfPropertiesTestIT { + + private static final Logger LOG = LoggerFactory + .getLogger(SegmentReferenceLimitTestIT.class); + + private File directory; + + @Before + public void setUp() throws IOException { + directory = File.createTempFile("LargeNumberOfPropertiesTestIT", "dir", + new File("target")); + directory.delete(); + directory.mkdir(); + } + + @After + public void cleanDir() { + try { + FileUtils.deleteDirectory(directory); + } catch (IOException e) { + LOG.error("Error cleaning directory", e); + } + } + + /** + * OAK-2481 IllegalStateException in TarMk with large number of properties + * + * TODO Test is currently ignored because of how memory intensive it is + * + * @see OAK-2481 + */ + @Test + @Ignore + public void corruption() throws Exception { + FileStore fileStore = new FileStore(directory, 5, 0, false); + SegmentNodeStore nodeStore = new SegmentNodeStore(fileStore); + + NodeBuilder root = nodeStore.getRoot().builder(); + + try { + NodeBuilder c = root.child("c" + System.currentTimeMillis()); + // i=26 hits the hard limit for the number of properties a node can + // have (262144) + for (int i = 0; i < 25; i++) { + for (int j = 0; j < 10000; j++) { + c.setProperty("int-" + i + "-" + j, i); + } + } + nodeStore.merge(root, EmptyHook.INSTANCE, CommitInfo.EMPTY); + } finally { + fileStore.close(); + } + } + +} Index: src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/SegmentReferenceLimitTestIT.java =================================================================== --- src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/SegmentReferenceLimitTestIT.java (revision 1660633) +++ src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/SegmentReferenceLimitTestIT.java (working copy) @@ -19,11 +19,10 @@ package org.apache.jackrabbit.oak.plugins.segment.file; -import static java.lang.String.valueOf; -import static org.junit.Assert.assertTrue; - import java.io.File; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.FutureTask; @@ -31,7 +30,6 @@ import org.apache.commons.io.FileUtils; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore; -import org.apache.jackrabbit.oak.plugins.segment.SegmentOverflowException; import org.apache.jackrabbit.oak.spi.commit.CommitInfo; import org.apache.jackrabbit.oak.spi.commit.EmptyHook; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; @@ -43,13 +41,14 @@ import org.slf4j.LoggerFactory; public class SegmentReferenceLimitTestIT { + private static final Logger LOG = LoggerFactory.getLogger(SegmentReferenceLimitTestIT.class); private File directory; @Before public void setUp() throws IOException { - directory = File.createTempFile("FileStoreTest", "dir", new File("target")); + directory = File.createTempFile("SegmentReferenceLimitTestIT", "dir", new File("target")); directory.delete(); directory.mkdir(); } @@ -63,6 +62,12 @@ } } + /** + * OAK-2294 Corrupt repository after concurrent version operations + * + * @see OAK-2294 + */ @Test public void corruption() throws IOException, CommitFailedException, ExecutionException, InterruptedException { FileStore fileStore = new FileStore(directory, 1, 0, false); @@ -72,19 +77,15 @@ root.setChildNode("test"); nodeStore.merge(root, EmptyHook.INSTANCE, CommitInfo.EMPTY); - FutureTask w1 = run(new Worker(nodeStore, "w1")); - FutureTask w2 = run(new Worker(nodeStore, "w2")); - FutureTask w3 = run(new Worker(nodeStore, "w3")); - FutureTask w4 = run(new Worker(nodeStore, "w4")); + List> l = new ArrayList>(); + for (int i = 0; i < 10; i++) { + l.add(run(new Worker(nodeStore, "w" + i))); + } try { - w1.get(); - w2.get(); - w3.get(); - w4.get(); - } catch (ExecutionException e) { - assertTrue(valueOf(e.getCause()), e.getCause() instanceof CommitFailedException); - assertTrue(valueOf(e.getCause()), e.getCause().getCause() instanceof SegmentOverflowException); + for (FutureTask w : l) { + w.get(); + } } finally { fileStore.close(); } @@ -107,11 +108,12 @@ @Override public Void call() throws Exception { - for (int k = 0; ; k++) { + for (int k = 0; k < 400; k++) { NodeBuilder root = nodeStore.getRoot().builder(); root.getChildNode("test").setProperty(name + ' ' + k, name + " value " + k); nodeStore.merge(root, EmptyHook.INSTANCE, CommitInfo.EMPTY); } + return null; } }