Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java (revision 55d41c97689ad790681ca071fbf90fb370944c81) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java (revision ) @@ -83,7 +83,6 @@ * split: new segments are automatically created if and when needed). */ public class SegmentWriter { - /** Logger instance */ private static final Logger log = LoggerFactory.getLogger(SegmentWriter.class); @@ -119,6 +118,7 @@ * avoid storing duplicates of frequently occurring data. * Should only be accessed from synchronized blocks to prevent corruption. */ + @SuppressWarnings("serial") private final Map records = new LinkedHashMap(15000, 0.75f, true) { @Override @@ -968,7 +968,14 @@ propertyTypes[i] = (byte) type.tag(); } } - ids.addAll(Arrays.asList(propertyNames)); + + // TODO + RecordId propNamesId = null; + if (propertyNames.length > 0) { + propNamesId = writeList(Arrays.asList(propertyNames)); + ids.add(propNamesId); + } + checkState(propertyNames.length < (1 << 18)); head |= propertyNames.length; @@ -985,8 +992,10 @@ if (childNameId != null) { writeRecordId(childNameId); } + if (propNamesId != null) { + writeRecordId(propNamesId); + } for (int i = 0; i < propertyNames.length; i++) { - writeRecordId(propertyNames[i]); buffer[position++] = propertyTypes[i]; } @@ -1087,33 +1096,44 @@ ids.add(writeNode(state.getChildNode(template.getChildName())).getRecordId()); } + List pIds = Lists.newArrayList(); for (PropertyTemplate pt : template.getPropertyTemplates()) { String name = pt.getName(); PropertyState property = state.getProperty(name); if (property instanceof SegmentPropertyState && store.containsSegment(((SegmentPropertyState) property).getRecordId().getSegmentId())) { - ids.add(((SegmentPropertyState) property).getRecordId()); + pIds.add(((SegmentPropertyState) property).getRecordId()); } else if (before == null || !store.containsSegment(before.getRecordId().getSegmentId())) { - ids.add(writeProperty(property)); + pIds.add(writeProperty(property)); } else { // reuse previously stored property, if possible PropertyTemplate bt = beforeTemplate.getPropertyTemplate(name); if (bt == null) { - ids.add(writeProperty(property)); // new property + RecordId p = writeProperty(property); + pIds.add(p); // new property } else { SegmentPropertyState bp = beforeTemplate.getProperty( before.getRecordId(), bt.getIndex()); if (property.equals(bp)) { - ids.add(bp.getRecordId()); // no changes + pIds.add(bp.getRecordId()); // no changes } else if (bp.isArray() && bp.getType() != BINARIES) { // reuse entries from the previous list - ids.add(writeProperty(property, bp.getValueRecords())); + pIds.add(writeProperty(property, bp.getValueRecords())); } else { - ids.add(writeProperty(property)); + pIds.add(writeProperty(property)); } } + } + } + + if (!pIds.isEmpty()) { + //TODO + if (segment.getStorageFormatVersion() == Segment.STORAGE_FORMAT_VERSION) { + ids.add(writeList(pIds)); + } else { + ids.addAll(pIds); } } Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeState.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeState.java (revision 55d41c97689ad790681ca071fbf90fb370944c81) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeState.java (revision ) @@ -16,21 +16,6 @@ */ package org.apache.jackrabbit.oak.plugins.segment; -import java.util.Collections; -import java.util.List; - -import javax.annotation.CheckForNull; -import javax.annotation.Nonnull; - -import org.apache.jackrabbit.oak.api.PropertyState; -import org.apache.jackrabbit.oak.api.Type; -import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState; -import org.apache.jackrabbit.oak.plugins.memory.MemoryChildNodeEntry; -import org.apache.jackrabbit.oak.spi.state.AbstractNodeState; -import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; -import org.apache.jackrabbit.oak.spi.state.NodeState; -import org.apache.jackrabbit.oak.spi.state.NodeStateDiff; - import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.collect.Lists.newArrayListWithCapacity; @@ -48,6 +33,21 @@ import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.MISSING_NODE; import static org.apache.jackrabbit.oak.spi.state.AbstractNodeState.checkValidName; +import java.util.Collections; +import java.util.List; + +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; + +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState; +import org.apache.jackrabbit.oak.plugins.memory.MemoryChildNodeEntry; +import org.apache.jackrabbit.oak.spi.state.AbstractNodeState; +import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStateDiff; + /** * A record of type "NODE". This class can read a node record from a segment. It * currently doesn't cache data (but the template is fully loaded). @@ -130,21 +130,47 @@ return property; } - PropertyTemplate propertyTemplate = - template.getPropertyTemplate(name); + PropertyTemplate propertyTemplate = template.getPropertyTemplate(name); if (propertyTemplate != null) { + // TODO + if (getSegment().getStorageFormatVersion() == Segment.STORAGE_FORMAT_VERSION) { + return readPropV1(template, propertyTemplate); + } else { + return readPropV0(template, propertyTemplate); + } + } else { + return null; + } + } + + private SegmentPropertyState readPropV0(Template template, PropertyTemplate propertyTemplate) { - Segment segment = getSegment(); - int ids = 1 + propertyTemplate.getIndex(); - if (template.getChildName() != Template.ZERO_CHILD_NODES) { - ids++; - } + Segment segment = getSegment(); + int ids = 1 + propertyTemplate.getIndex(); + if (template.getChildName() != Template.ZERO_CHILD_NODES) { + ids++; + } - return new SegmentPropertyState( - segment.readRecordId(getOffset(0, ids)), propertyTemplate); - } else { - return null; + RecordId rid = segment.readRecordId(getOffset(0, ids)); + SegmentPropertyState sps = new SegmentPropertyState(rid, propertyTemplate); + return sps; - } + } + + private SegmentPropertyState readPropV1(Template template, PropertyTemplate propertyTemplate) { + Segment segment = getSegment(); + int ids = 1; + if (template.getChildName() != Template.ZERO_CHILD_NODES) { + ids++; - } + } + RecordId rid = segment.readRecordId(getOffset(0, ids)); + ListRecord pIds = new ListRecord(rid, + template.getPropertyTemplates().length); + RecordId pid = pIds.getEntry(propertyTemplate.getIndex()); + + SegmentPropertyState sps = new SegmentPropertyState(pid, + propertyTemplate); + return sps; + } + @Override @Nonnull public Iterable getProperties() { Template template = getTemplate(); @@ -167,11 +193,28 @@ if (template.getChildName() != Template.ZERO_CHILD_NODES) { ids++; } + + //TODO + if (segment.getStorageFormatVersion() == Segment.STORAGE_FORMAT_VERSION) { + // -- V1 -- + if (propertyTemplates.length > 0) { + ListRecord pIds = new ListRecord( + segment.readRecordId(getOffset(0, ids)), + propertyTemplates.length); - for (int i = 0; i < propertyTemplates.length; i++) { + for (int i = 0; i < propertyTemplates.length; i++) { + RecordId propertyId = pIds.getEntry(i); + list.add(new SegmentPropertyState(propertyId, + propertyTemplates[i])); + } + } + } else { + // -- V0 -- + for (int i = 0; i < propertyTemplates.length; i++) { - RecordId propertyId = segment.readRecordId(getOffset(0, ids++)); + RecordId propertyId = segment.readRecordId(getOffset(0, ids++)); - list.add(new SegmentPropertyState( - propertyId, propertyTemplates[i])); + list.add(new SegmentPropertyState(propertyId, + propertyTemplates[i])); - } + } + } return list; } @@ -246,6 +289,8 @@ return null; } + //TODO update to latest version + System.err.println("#getValueAsString " + name); Segment segment = getSegment(); int ids = 1 + propertyTemplate.getIndex(); if (template.getChildName() != Template.ZERO_CHILD_NODES) { @@ -287,6 +332,8 @@ return emptyList(); } + //TODO update to latest version + System.err.println("#getValuesAsString " + name); Segment segment = getSegment(); int ids = 1 + propertyTemplate.getIndex(); if (template.getChildName() != Template.ZERO_CHILD_NODES) { Index: oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/SegmentReferenceLimitTestIT.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/SegmentReferenceLimitTestIT.java (revision 55d41c97689ad790681ca071fbf90fb370944c81) +++ oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/SegmentReferenceLimitTestIT.java (revision ) @@ -108,9 +108,14 @@ @Override public Void call() throws Exception { for (int k = 0; ; k++) { + try { - NodeBuilder root = nodeStore.getRoot().builder(); - root.getChildNode("test").setProperty(name + ' ' + k, name + " value " + k); - nodeStore.merge(root, EmptyHook.INSTANCE, CommitInfo.EMPTY); + NodeBuilder root = nodeStore.getRoot().builder(); + root.getChildNode("test").setProperty(name + ' ' + k, name + " value " + k); + nodeStore.merge(root, EmptyHook.INSTANCE, CommitInfo.EMPTY); + } catch (Exception e) { + e.printStackTrace(); + throw e; + } } } } Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Template.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Template.java (revision 55d41c97689ad790681ca071fbf90fb370944c81) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Template.java (revision ) @@ -173,9 +173,18 @@ if (childName != ZERO_CHILD_NODES) { offset += RECORD_ID_BYTES; } + + RecordId rid = null; + if (segment.getStorageFormatVersion() == Segment.STORAGE_FORMAT_VERSION) { + RecordId lid = segment.readRecordId(offset); + ListRecord props = new ListRecord(lid, properties.length); + + rid = props.getEntry(index); + } else { - offset += index * RECORD_ID_BYTES; + offset += index * RECORD_ID_BYTES; - return new SegmentPropertyState( - segment.readRecordId(offset), properties[index]); + rid = segment.readRecordId(offset); + } + return new SegmentPropertyState(rid, properties[index]); } MapRecord getChildNodeMap(RecordId recordId) { Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java (revision 55d41c97689ad790681ca071fbf90fb370944c81) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java (revision ) @@ -54,10 +54,11 @@ /** * Version of the segment storage format. *
    - *
  • 10 = all Oak versions released so far
    • + *
  • 10 = all Oak versions previous to 11
    • + *
  • 11 = all Oak versions starting from TODO
    • *
*/ - public static final byte STORAGE_FORMAT_VERSION = 10; + public static final byte STORAGE_FORMAT_VERSION = 11; /** * Number of bytes used for storing a record identifier. One byte @@ -116,6 +117,8 @@ private final ByteBuffer data; + private final byte segmentFormat; + /** * Referenced segment identifiers. Entries are initialized lazily in * {@link #getRefId(int)}. Set to {@code null} for bulk segments. @@ -142,13 +145,16 @@ this.data = checkNotNull(data); if (id.isDataSegmentId()) { + segmentFormat = data.get(3); checkState(data.get(0) == '0' && data.get(1) == 'a' && data.get(2) == 'K' - && data.get(3) == STORAGE_FORMAT_VERSION); + && (segmentFormat == STORAGE_FORMAT_VERSION - 1 // backward compatible with previous format + || segmentFormat == STORAGE_FORMAT_VERSION)); this.refids = new SegmentId[getRefCount()]; refids[0] = id; } else { + this.segmentFormat = STORAGE_FORMAT_VERSION; this.refids = null; } } @@ -159,6 +165,7 @@ this.data = ByteBuffer.wrap(checkNotNull(buffer)); this.refids = new SegmentId[SEGMENT_REFERENCE_LIMIT + 1]; + this.segmentFormat = STORAGE_FORMAT_VERSION; refids[0] = id; } @@ -171,6 +178,10 @@ return accessed != 0; } + byte getStorageFormatVersion() { + return segmentFormat; + } + /** * Maps the given record offset to the respective position within the * internal {@link #data} array. The validity of a record with the given @@ -192,8 +203,13 @@ } int getRefCount() { - return (data.get(REF_COUNT_OFFSET) & 0xff) + 1; + int refcount = (data.get(REF_COUNT_OFFSET) & 0xff); + if (refcount + 1 > SEGMENT_REFERENCE_LIMIT) { + throw new SegmentOverflowException( + "Segment cannot have more than 255 references " + id); - } + } + return refcount + 1; + } public int getRootCount() { return data.getShort(ROOT_COUNT_OFFSET) & 0xffff; @@ -436,19 +452,41 @@ offset += Segment.RECORD_ID_BYTES; } - PropertyTemplate[] properties = - new PropertyTemplate[propertyCount]; - for (int i = 0; i < properties.length; i++) { + PropertyTemplate[] properties; + if (segmentFormat == STORAGE_FORMAT_VERSION) { + properties = readPropsV1(propertyCount, offset); + } else { + properties = readPropsV0(propertyCount, offset); + } + return new Template(primaryType, mixinTypes, properties, childName); + } + + private PropertyTemplate[] readPropsV0(int propertyCount, int offset) { + PropertyTemplate[] properties = new PropertyTemplate[propertyCount]; + for (int i = 0; i < propertyCount; i++) { RecordId propertyNameId = readRecordId(offset); offset += Segment.RECORD_ID_BYTES; byte type = readByte(offset++); - properties[i] = new PropertyTemplate( - i, readString(propertyNameId), + properties[i] = new PropertyTemplate(i, readString(propertyNameId), Type.fromTag(Math.abs(type), type < 0)); } + return properties; + } - return new Template( - primaryType, mixinTypes, properties, childName); + private PropertyTemplate[] readPropsV1(int propertyCount, int offset) { + PropertyTemplate[] properties = new PropertyTemplate[propertyCount]; + if (propertyCount > 0) { + RecordId id = readRecordId(offset); + ListRecord propertyNames = new ListRecord(id, properties.length); + offset += Segment.RECORD_ID_BYTES; + for (int i = 0; i < propertyCount; i++) { + byte type = readByte(offset++); + properties[i] = new PropertyTemplate(i, + readString(propertyNames.getEntry(i)), Type.fromTag( + Math.abs(type), type < 0)); + } + } + return properties; } long readLength(RecordId id) {