diff --git a/oak-upgrade/pom.xml b/oak-upgrade/pom.xml index 1b0d3c4332..cf5f95af76 100644 --- a/oak-upgrade/pom.xml +++ b/oak-upgrade/pom.xml @@ -212,6 +212,12 @@ ${h2.version} test + + pl.pragmatists + JUnitParams + 1.1.1 + test + diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/LoopbackBlobStore.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/LoopbackBlobStore.java new file mode 100644 index 0000000000..95e7b8f571 --- /dev/null +++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/LoopbackBlobStore.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.upgrade.cli.blob; + +import org.apache.jackrabbit.oak.spi.blob.BlobOptions; +import org.apache.jackrabbit.oak.spi.blob.BlobStore; + +import javax.annotation.Nonnull; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +import static com.google.common.base.Preconditions.checkNotNull; + +/** + * Utility BlobStore implementation to be used in tooling that can work with a + * FileStore without the need of the DataStore being present locally. + * + * Additionally instead of failing it tries to mimic and return blob reference + * passed in by caller by passing it back as a binary. + * + * Example: requesting blobId = e7c22b994c59d9 it will return the + * e7c22b994c59d9 text as a UTF-8 encoded binary file. + */ +public class LoopbackBlobStore implements BlobStore { + + @Override + public String writeBlob(InputStream in) { + throw new UnsupportedOperationException(); + } + + @Override + public String writeBlob(InputStream in, BlobOptions options) throws IOException { + return writeBlob(in); + } + + @Override + public int readBlob(String blobId, long pos, byte[] buff, int off, + int length) { + // Only a part of binary can be requested! + final int binaryLength = blobId.length(); + checkBinaryOffsetInRange(pos, binaryLength); + final int effectiveSrcPos = Math.toIntExact(pos); + final int effectiveBlobLengthToBeRead = Math.min( + binaryLength - effectiveSrcPos, length); + checkForBufferOverflow(buff, off, effectiveBlobLengthToBeRead); + final byte[] blobIdBytes = getBlobIdStringAsByteArray(blobId); + System.arraycopy(blobIdBytes, effectiveSrcPos, buff, off, + effectiveBlobLengthToBeRead); + return effectiveBlobLengthToBeRead; + } + + private void checkForBufferOverflow(final byte[] buff, final int off, + final int effectiveBlobLengthToBeRead) { + if (buff.length < effectiveBlobLengthToBeRead + off) { + // We cannot recover if buffer used to write is too small + throw new UnsupportedOperationException("Edge case: cannot fit " + + "blobId in a buffer (buffer too small)"); + } + } + + private void checkBinaryOffsetInRange(final long pos, final int binaryLength) { + if (pos > binaryLength) { + throw new IllegalArgumentException( + String.format("Offset %d out of range of %d", pos, + binaryLength)); + } + } + + private byte[] getBlobIdStringAsByteArray(final String blobId) { + return blobId.getBytes(StandardCharsets.UTF_8); + } + + @Override + public long getBlobLength(String blobId) throws IOException { + return blobId.length(); + } + + @Override + public InputStream getInputStream(String blobId) throws IOException { + checkNotNull(blobId); + return new ByteArrayInputStream(getBlobIdStringAsByteArray(blobId)); + } + + @Override + public String getBlobId(@Nonnull String reference) { + return checkNotNull(reference); + } + + @Override + public String getReference(@Nonnull String blobId) { + return checkNotNull(blobId); + } +} diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/LoopbackBlobStoreFactory.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/LoopbackBlobStoreFactory.java new file mode 100644 index 0000000000..88e68154cf --- /dev/null +++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/LoopbackBlobStoreFactory.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.upgrade.cli.blob; + +import com.google.common.io.Closer; +import org.apache.jackrabbit.oak.spi.blob.BlobStore; + +import static com.google.common.base.Preconditions.checkNotNull; + +public class LoopbackBlobStoreFactory implements BlobStoreFactory { + + @Override + public BlobStore create(Closer closer) { + checkNotNull(closer, "Closer object cannot be null"); + return new LoopbackBlobStore(); + } + + @Override + public String toString() { + return "LoopbackBlobStore"; + } +} diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/DatastoreArguments.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/DatastoreArguments.java index 5f6dea0f16..2035e19ee0 100644 --- a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/DatastoreArguments.java +++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/DatastoreArguments.java @@ -23,7 +23,7 @@ import org.apache.jackrabbit.oak.upgrade.cli.blob.ConstantBlobStoreFactory; import org.apache.jackrabbit.oak.upgrade.cli.blob.DummyBlobStoreFactory; import org.apache.jackrabbit.oak.upgrade.cli.blob.FileBlobStoreFactory; import org.apache.jackrabbit.oak.upgrade.cli.blob.FileDataStoreFactory; -import org.apache.jackrabbit.oak.upgrade.cli.blob.MissingBlobStoreFactory; +import org.apache.jackrabbit.oak.upgrade.cli.blob.LoopbackBlobStoreFactory; import org.apache.jackrabbit.oak.upgrade.cli.blob.S3DataStoreFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -110,7 +110,7 @@ public class DatastoreArguments { if (options.isSrcBlobStoreDefined()) { result = definedSrcBlob; } else if (blobMigrationCase == BlobMigrationCase.COPY_REFERENCES) { - result = new MissingBlobStoreFactory(); + result = new LoopbackBlobStoreFactory(); } else { result = new DummyBlobStoreFactory(); // embedded } @@ -125,7 +125,7 @@ public class DatastoreArguments { } else if (blobMigrationCase == BlobMigrationCase.COPY_REFERENCES && (options.isSrcBlobStoreDefined() || storeArguments.getSrcType() == JCR2_DIR_XML)) { result = new ConstantBlobStoreFactory(srcBlobStore); } else if (blobMigrationCase == BlobMigrationCase.COPY_REFERENCES) { - result = new MissingBlobStoreFactory(); + result = new LoopbackBlobStoreFactory(); } else { result = new DummyBlobStoreFactory(); // embedded } diff --git a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/CopyCheckpointsTest.java b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/CopyCheckpointsTest.java index ba3dbc4c43..45fe3d108d 100644 --- a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/CopyCheckpointsTest.java +++ b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/CopyCheckpointsTest.java @@ -63,11 +63,11 @@ public class CopyCheckpointsTest extends AbstractOak2OakTest { BlobStoreContainer blob = new FileDataStoreContainer(); params.add(new Object[]{ - "Fails on missing blobstore", + "Without data store defined it always copies checkpoints", new SegmentNodeStoreContainer(blob), new SegmentNodeStoreContainer(blob), asList(), - Result.EXCEPTION + Result.CHECKPOINTS_COPIED }); params.add(new Object[]{ "Suppress the warning", diff --git a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/LoopbackBlobStoreFactoryTest.java b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/LoopbackBlobStoreFactoryTest.java new file mode 100644 index 0000000000..8ed9899691 --- /dev/null +++ b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/LoopbackBlobStoreFactoryTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.upgrade.cli.blob; + +import com.google.common.io.Closer; +import org.apache.jackrabbit.oak.spi.blob.BlobStore; +import org.junit.Test; + +import java.io.IOException; + +import static junit.framework.TestCase.assertEquals; +import static org.junit.Assert.assertNotNull; + +@SuppressWarnings("UnusedLabel") +public class LoopbackBlobStoreFactoryTest { + + @Test(expected = NullPointerException.class) + public void cannotCreateLoopbackBlobStoreFactoryWithNullCloser() { + when: { + final LoopbackBlobStoreFactory factory = new LoopbackBlobStoreFactory(); + factory.create(null); + } + } + + @Test + public void canCreateLoopbackBlobStoreFactory() throws IOException { + when: { + final LoopbackBlobStoreFactory factory = new LoopbackBlobStoreFactory(); + final Closer closer = Closer.create(); + final BlobStore blobStore = factory.create(closer); + + then: { + assertNotNull(blobStore); + } + and: { + closer.close(); + } + } + } + + @Test + public void canGetNameFromLoopbackBlobStoreFactory() { + when: { + final LoopbackBlobStoreFactory factory = new LoopbackBlobStoreFactory(); + + then: { + assertEquals("LoopbackBlobStore", factory.toString()); + } + } + } + +} diff --git a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/LoopbackBlobStoreTest.java b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/LoopbackBlobStoreTest.java new file mode 100644 index 0000000000..bb1a075f49 --- /dev/null +++ b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/LoopbackBlobStoreTest.java @@ -0,0 +1,356 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.upgrade.cli.blob; + +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; +import org.apache.commons.io.IOUtils; +import org.apache.jackrabbit.oak.spi.blob.BlobOptions; +import org.apache.jackrabbit.oak.spi.blob.BlobStore; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.io.InputStream; + +import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertNotNull; + +@SuppressWarnings("UnusedLabel") +@RunWith(JUnitParamsRunner.class) +public class LoopbackBlobStoreTest { + + @Test(expected = UnsupportedOperationException.class) + public void writingBinariesIsNotSupported() throws IOException { + given: + { + final BlobStore blobStore = new LoopbackBlobStore(); + + when: + { + final String test = "Test"; + blobStore.writeBlob(adaptToUtf8InputStream(test)); + } + } + } + + @Test(expected = UnsupportedOperationException.class) + public void writingBinariesWithBlobOptsIsNotSupported() throws IOException { + given: + { + final BlobStore blobStore = new LoopbackBlobStore(); + final BlobOptions blobOptions = new BlobOptions(); + + when: + { + blobStore.writeBlob(adaptToUtf8InputStream("Test"), + blobOptions); + } + } + } + + + @Test + @Parameters(method = "blobIds") + public void getBlobIdShouldReturnTheSameValuePassedExceptOfNull( + final String blobId) { + given: + { + final BlobStore blobStore = new LoopbackBlobStore(); + expect: + { + assertEquals(blobId, blobStore.getBlobId(blobId)); + } + } + } + + + @SuppressWarnings("ConstantConditions") + @Test(expected = NullPointerException.class) + public void getBlobIdShouldThrowAnExceptionWhenNullIsPassed() { + given: + { + final BlobStore blobStore = new LoopbackBlobStore(); + when: + { + blobStore.getBlobId(null); + } + } + } + + + @Test + @Parameters(method = "blobIds") + public void getReferenceShouldReturnTheSameValuePassedExceptOfNull( + final String blobId) { + given: + { + final BlobStore blobStore = new LoopbackBlobStore(); + where: + { + expect: + { + assertEquals(blobId, blobStore.getReference(blobId)); + } + } + } + } + + @SuppressWarnings("ConstantConditions") + @Test(expected = NullPointerException.class) + public void getReferenceShouldThrowAnExceptionWhenNullIsPassed() { + given: + { + final BlobStore blobStore = new LoopbackBlobStore(); + when: + { + blobStore.getReference(null); + } + } + } + + @Test + @Parameters(method = "blobIds") + public void getBlobLengthShouldAlwaysReturnRealLengthOfBlobThatWillBeReturned( + final String blobId) throws IOException { + given: + { + final BlobStore store = new LoopbackBlobStore(); + expect: + { + assertEquals(blobId.getBytes().length, store.getBlobLength(blobId)); + } + } + } + + @Test(expected = NullPointerException.class) + public void getBlobLengthShouldAlwaysThrowAnExceptionWhenNullBlobIdIsPassed() + throws IOException { + given: + { + final BlobStore store = new LoopbackBlobStore(); + when: + { + store.getBlobLength(null); + } + } + } + + @Test(expected = NullPointerException.class) + public void getInputStreamShouldAlwaysThrowAnExceptionWhenNullBlobIdIsPassed() + throws IOException { + given: + { + final BlobStore store = new LoopbackBlobStore(); + when: + { + store.getInputStream(null); + } + } + } + + @Test + @Parameters(method = "blobIds") + public void shouldAlwaysReturnStreamOfRequestedBlobIdUtf8BinRepresentation( + final String blobId) throws IOException { + given: + { + final String encoding = "UTF-8"; + final BlobStore store = new LoopbackBlobStore(); + when: + { + final InputStream inputStream = store.getInputStream(blobId); + then: + { + assertNotNull(inputStream); + } + and: + { + final String actualInputStreamAsString = IOUtils.toString( + inputStream, encoding); + then: + { + assertEquals(actualInputStreamAsString, blobId); + } + } + } + } + } + + @Test + @Parameters(method = "blobIdsReads") + public void shouldAlwaysFillBufferWithRequestedBlobIdUtf8BinRepresentation( + final String blobId, + int offsetToRead, + int bufSize, + int bufOffset, + int lengthToRead, + final String expectedBufferContent, + final int expectedNumberOfBytesRead) throws IOException { + given: + { + final String encoding = "UTF-8"; + final BlobStore blobStore = new LoopbackBlobStore(); + final byte[] buffer = new byte[bufSize]; + when: + { + final int numberOfBytesRead = blobStore.readBlob( + blobId, offsetToRead, buffer, bufOffset, lengthToRead); + and: + { + final String actualInputStreamAsString = IOUtils.toString( + buffer, encoding); + then: + { + assertEquals(numberOfBytesRead, + expectedNumberOfBytesRead); + assertEquals(expectedBufferContent, + actualInputStreamAsString); + } + } + } + } + } + + @Test(expected = UnsupportedOperationException.class) + @Parameters(method = "blobIdsFailedBufferReadsCases") + public void getInputStreamShouldAlwaysReturnExceptionIfBufferTooSmall( + final String blobId, + int offsetToRead, + int bufSize, + int bufOffset, + int lengthToRead) throws IOException { + given: + { + final BlobStore store = new LoopbackBlobStore(); + final byte[] buffer = new byte[bufSize]; + when: + { + store.readBlob( + blobId, offsetToRead, buffer, bufOffset, lengthToRead); + } + } + } + + @Test(expected = IllegalArgumentException.class) + @Parameters(method = "blobIdsFailedOffsetReadsCases") + public void getInputStreamShouldAlwaysReturnExceptionIfBinaryOffsetIsBad( + final String blobId, + int offsetToRead, + int bufSize, + int bufOffset, + int lengthToRead) throws IOException { + given: + { + final BlobStore store = new LoopbackBlobStore(); + final byte[] buffer = new byte[bufSize]; + when: + { + store.readBlob( + blobId, offsetToRead, buffer, bufOffset, lengthToRead); + } + } + } + + @SuppressWarnings("unused") + private Object blobIdsReads() { + return new Object[]{ + //blobId, offsetToRead, bufSize, bufOffset, lengthToRead, expectedBufferContent, expectedNumOfBytesRead + new Object[]{ + "", 0, 0, 0, 0, "", 0}, + new Object[]{ + "", 0, 0, 0, 1, "", 0}, + new Object[]{ + "IDX1", 0, 4, 0, 4, "IDX1", 4}, + new Object[]{ + "IDX1", 4, 0, 0, 4, "", 0}, + new Object[]{ + "IDX1", 4, 4, 0, 4, "\0\0\0\0", 0}, + new Object[]{ + "IDX1", 0, 5, 0, 4, "IDX1\0", 4}, + new Object[]{ + "IDX1", 1, 4, 0, 3, "DX1\0", 3}, + new Object[]{ + "IDX1", 1, 4, 0, 4, "DX1\0", 3}, + new Object[]{ + "ID2XXXXXXXXXXXYYZYZYYXYZYZYXYZQ", 10, 20, 3, 10, "\0\0\0XXXXYYZYZY\0\0\0\0\0\0\0", 10}, + new Object[]{ + "ID2XXXXXXXXXXXYYZY", 10, 20, 3, 10, "\0\0\0XXXXYYZY\0\0\0\0\0\0\0\0\0", 8}, + new Object[]{ + "ID2XXXXXXXXXXXYYZY", 10, 20, 3, 10, "\0\0\0XXXXYYZY\0\0\0\0\0\0\0\0\0", 8}, + new Object[]{ + "ID2XXXXXXXXXXXYYZY", 10, 11, 3, 10, "\0\0\0XXXXYYZY", 8}, + new Object[]{ + "ID2XXXXXXXXXXXYYZY", 10, 11, 2, 10, "\0\0XXXXYYZY\0", 8}, + new Object[]{ + "ID2XXXXXXXXXXXYYZY", 10, 11, 1, 10, "\0XXXXYYZY\0\0", 8}, + }; + } + + @SuppressWarnings("unused") + private Object blobIdsFailedBufferReadsCases() { + return new Object[]{ + //blobId, offsetToRead, bufferSize, bufferOffset, lengthToRead + new Object[]{ + " ", 0, 0, 0, 1}, + new Object[]{ + "IDX1", 0, 3, 0, 4}, + new Object[]{ + "IDX1", 1, 3, 2, 3}, + new Object[]{ + "IDX1", 1, 2, 0, 3}, + new Object[]{ + "ID2XXXXXXXXXXXYYZY", 10, 0, 30, 10}, + }; + } + + @SuppressWarnings("unused") + private Object blobIdsFailedOffsetReadsCases() { + return new Object[]{ + //blobId, offsetToRead, bufferSize, bufferOffset, lengthToRead + new Object[]{ + "", 1, 50, 0, 0}, + new Object[]{ + "IDX1", 5, 50, 0, 3}, + new Object[]{ + "IDX1", 6, 50, 0, 4}, + new Object[]{ + "ID2XXXXXXXXXXXYYZY", 30, 50, 1, 10}, + }; + } + + + @SuppressWarnings("unused") + private Object blobIds() { + return new Object[]{ + new Object[]{""}, + new Object[]{"IDX1"}, + new Object[]{"ID2XXXXXXXXXXXYYZYZYYXYZYZYXYZQ"}, + new Object[]{"ABCQ"} + }; + } + + private InputStream adaptToUtf8InputStream(final String string) + throws IOException { + return IOUtils.toInputStream(string, + "UTF-8"); + } + +} +