.../org/apache/jackrabbit/aws/ext/S3Constants.java | 15 ++++ .../jackrabbit/aws/ext/S3RequestDecorator.java | 87 ++++++++++++++++++ .../apache/jackrabbit/aws/ext/ds/S3Backend.java | 33 +++---- .../org/apache/jackrabbit/aws/ext/TestAll.java | 6 +- .../jackrabbit/aws/ext/ds/TestS3DSWithSSES3.java | 100 +++++++++++++++++++++ 5 files changed, 224 insertions(+), 17 deletions(-) diff --git a/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/S3Constants.java b/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/S3Constants.java index 02cdd5e..7c85473 100644 --- a/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/S3Constants.java +++ b/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/S3Constants.java @@ -83,6 +83,21 @@ public final class S3Constants { public static final String S3_WRITE_THREADS = "writeThreads"; /** + * Constant to enable encrption in S3. + */ + public static final String S3_ENCRYPTION = "s3Encrption"; + + /** + * Constant for no encryption. it is default. + */ + public static final String S3_ENCRYPTION_NONE = "NONE"; + + /** + * Constant to set SSE_S3 encryption. + */ + public static final String S3_ENCRYPTION_SSE_S3 = "SSE_S3"; + + /** * private constructor so that class cannot initialized from outside. */ private S3Constants() { diff --git a/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/S3RequestDecorator.java b/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/S3RequestDecorator.java new file mode 100644 index 0000000..2fcc244 --- /dev/null +++ b/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/S3RequestDecorator.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jackrabbit.aws.ext; + +import java.util.Properties; + +import com.amazonaws.services.s3.model.CopyObjectRequest; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PutObjectRequest; + +/** + * This class to sets encrption mode in S3 request. + * + */ +public class S3RequestDecorator { + DataEncryption dataEncryption = DataEncryption.NONE; + + public S3RequestDecorator(Properties props) { + if (props.getProperty(S3Constants.S3_ENCRYPTION) != null) { + this.dataEncryption = dataEncryption.valueOf(props.getProperty(S3Constants.S3_ENCRYPTION)); + } + } + + /** + * Set encryption in {@link PutObjectRequest} + */ + public PutObjectRequest decorate(PutObjectRequest request) { + switch (getDataEncryption()) { + case SSE_S3: + ObjectMetadata metadata = request.getMetadata() == null + ? new ObjectMetadata() + : request.getMetadata(); + metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); + request.setMetadata(metadata); + break; + case NONE: + break; + } + return request; + } + + /** + * Set encryption in {@link CopyObjectRequest} + */ + public CopyObjectRequest decorate(CopyObjectRequest request) { + switch (getDataEncryption()) { + case SSE_S3: + ObjectMetadata metadata = request.getNewObjectMetadata() == null + ? new ObjectMetadata() + : request.getNewObjectMetadata(); + metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); + request.setNewObjectMetadata(metadata); + break; + case NONE: + break; + } + return request; + } + + private DataEncryption getDataEncryption() { + return this.dataEncryption; + } + + /** + * Enum to indicate S3 encryption mode + * + */ + private enum DataEncryption { + SSE_S3, NONE; + } + +} diff --git a/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/ds/S3Backend.java b/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/ds/S3Backend.java index 804e7d2..d097b8b 100644 --- a/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/ds/S3Backend.java +++ b/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/ds/S3Backend.java @@ -34,6 +34,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.jackrabbit.aws.ext.S3Constants; +import org.apache.jackrabbit.aws.ext.S3RequestDecorator; import org.apache.jackrabbit.aws.ext.Utils; import org.apache.jackrabbit.core.data.AsyncTouchCallback; import org.apache.jackrabbit.core.data.AsyncTouchResult; @@ -91,6 +92,8 @@ public class S3Backend implements Backend { private ThreadPoolExecutor asyncWriteExecuter; + private S3RequestDecorator s3ReqDecorator; + /** * Initialize S3Backend. It creates AmazonS3Client and TransferManager from * aws.properties. It creates S3 bucket if it doesn't pre-exist in S3. @@ -128,6 +131,8 @@ public class S3Backend implements Backend { getClass().getClassLoader()); LOG.debug("init"); this.store = store; + s3ReqDecorator = new S3RequestDecorator(prop); + s3service = Utils.openService(prop); if (bucket == null || "".equals(bucket.trim())) { bucket = prop.getProperty(S3Constants.S3_BUCKET); @@ -264,7 +269,7 @@ public class S3Backend implements Backend { CopyObjectRequest copReq = new CopyObjectRequest(bucket, key, bucket, key); copReq.setNewObjectMetadata(objectMetaData); - Copy copy = tmx.copy(copReq); + Copy copy = tmx.copy(s3ReqDecorator.decorate(copReq)); copy.waitForCopyResult(); LOG.debug("[{}] touched took [{}] ms. ", identifier, (System.currentTimeMillis() - start)); @@ -346,7 +351,7 @@ public class S3Backend implements Backend { CopyObjectRequest copReq = new CopyObjectRequest(bucket, key, bucket, key); copReq.setNewObjectMetadata(new ObjectMetadata()); - Copy copy = tmx.copy(copReq); + Copy copy = tmx.copy(s3ReqDecorator.decorate(copReq)); copy.waitForCompletion(); LOG.debug("[{}] touched. time taken [{}] ms ", new Object[] { identifier, (System.currentTimeMillis() - start) }); @@ -621,7 +626,7 @@ public class S3Backend implements Backend { CopyObjectRequest copReq = new CopyObjectRequest(bucket, key, bucket, key); copReq.setNewObjectMetadata(objectMetaData); - Copy copy = tmx.copy(copReq); + Copy copy = tmx.copy(s3ReqDecorator.decorate(copReq)); try { copy.waitForCopyResult(); LOG.debug("lastModified of [{}] updated successfully.", identifier); @@ -641,8 +646,8 @@ public class S3Backend implements Backend { if (objectMetaData == null) { try { // start multipart parallel upload using amazon sdk - Upload up = tmx.upload(new PutObjectRequest(bucket, key, - file)); + Upload up = tmx.upload(s3ReqDecorator.decorate(new PutObjectRequest( + bucket, key, file))); // wait for upload to finish if (asyncUpload) { up.addProgressListener(new S3UploadProgressListener(up, @@ -691,8 +696,7 @@ public class S3Backend implements Backend { try { Thread.currentThread().setContextClassLoader( getClass().getClassLoader()); - ObjectListing prevObjectListing = s3service.listObjects(bucket, - KEY_PREFIX); + ObjectListing prevObjectListing = s3service.listObjects(bucket); List deleteList = new ArrayList(); int nThreads = Integer.parseInt(properties.getProperty("maxConnections")); ExecutorService executor = Executors.newFixedThreadPool(nThreads, @@ -703,8 +707,11 @@ public class S3Backend implements Backend { executor.execute(new KeyRenameThread(s3ObjSumm.getKey())); taskAdded = true; count++; - deleteList.add(new DeleteObjectsRequest.KeyVersion( - s3ObjSumm.getKey())); + // delete the object if it follows old key name format + if( s3ObjSumm.getKey().startsWith(KEY_PREFIX)) { + deleteList.add(new DeleteObjectsRequest.KeyVersion( + s3ObjSumm.getKey())); + } } if (!prevObjectListing.isTruncated()) break; prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing); @@ -763,8 +770,7 @@ public class S3Backend implements Backend { private static String convertKey(String oldKey) throws IllegalArgumentException { if (!oldKey.startsWith(KEY_PREFIX)) { - throw new IllegalArgumentException("[" + oldKey - + "] doesn't start with prefix [" + KEY_PREFIX + "]"); + return oldKey; } String key = oldKey.substring(KEY_PREFIX.length()); return key.substring(0, 4) + Utils.DASH + key.substring(4); @@ -804,7 +810,7 @@ public class S3Backend implements Backend { String newS3Key = convertKey(oldKey); CopyObjectRequest copReq = new CopyObjectRequest(bucket, oldKey, bucket, newS3Key); - Copy copy = tmx.copy(copReq); + Copy copy = tmx.copy(s3ReqDecorator.decorate(copReq)); try { copy.waitForCopyResult(); LOG.debug("[{}] renamed to [{}] ", oldKey, newS3Key); @@ -812,7 +818,6 @@ public class S3Backend implements Backend { LOG.error(" Exception in renaming [{}] to [{}] ", new Object[] { ie, oldKey, newS3Key }); } - } finally { if (contextClassLoader != null) { Thread.currentThread().setContextClassLoader( @@ -902,6 +907,4 @@ public class S3Backend implements Backend { } } - - } diff --git a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/TestAll.java b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/TestAll.java index 9292314..e1e810a 100644 --- a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/TestAll.java +++ b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/TestAll.java @@ -21,10 +21,11 @@ import junit.framework.Test; import junit.framework.TestCase; import junit.framework.TestSuite; -import org.apache.jackrabbit.aws.ext.ds.TestS3Ds; import org.apache.jackrabbit.aws.ext.ds.TestS3DSAsyncTouch; -import org.apache.jackrabbit.aws.ext.ds.TestS3DsCacheOff; +import org.apache.jackrabbit.aws.ext.ds.TestS3DSWithSSES3; import org.apache.jackrabbit.aws.ext.ds.TestS3DSWithSmallCache; +import org.apache.jackrabbit.aws.ext.ds.TestS3Ds; +import org.apache.jackrabbit.aws.ext.ds.TestS3DsCacheOff; import org.apache.jackrabbit.core.data.TestCaseBase; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,6 +52,7 @@ public class TestAll extends TestCase { suite.addTestSuite(TestS3DSAsyncTouch.class); suite.addTestSuite(TestS3DSWithSmallCache.class); suite.addTestSuite(TestS3DsCacheOff.class); + suite.addTestSuite(TestS3DSWithSSES3.class); } return suite; } diff --git a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSWithSSES3.java b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSWithSSES3.java new file mode 100644 index 0000000..4102a00 --- /dev/null +++ b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSWithSSES3.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jackrabbit.aws.ext.ds; + +import java.io.ByteArrayInputStream; +import java.io.IOException; + +import javax.jcr.RepositoryException; + +import org.apache.jackrabbit.aws.ext.S3Constants; +import org.apache.jackrabbit.core.data.CachingDataStore; +import org.apache.jackrabbit.core.data.DataRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Test S3DataStore operation with SSE_S3 encryption. + */ +public class TestS3DSWithSSES3 extends TestS3Ds { + + protected static final Logger LOG = LoggerFactory.getLogger(TestS3DSWithSSES3.class); + + public TestS3DSWithSSES3() throws IOException { + config = System.getProperty(CONFIG); + memoryBackend = false; + noCache = true; + } + + protected CachingDataStore createDataStore() throws RepositoryException { + props.setProperty( + S3Constants.S3_BUCKET, + String.valueOf(randomGen.nextInt(9999)) + "-" + + String.valueOf(randomGen.nextInt(9999)) + "-test"); + props.setProperty(S3Constants.S3_ENCRYPTION, + S3Constants.S3_ENCRYPTION_SSE_S3); + ds = new S3TestDataStore(props); + ds.setConfig(config); + ds.init(dataStoreDir); + return ds; + } + + /** + * Test data migration enabling SSE_S3 encryption. + */ + public void testDataMigration() { + try { + String bucket = String.valueOf(randomGen.nextInt(9999)) + "-" + + String.valueOf(randomGen.nextInt(9999)) + "-test"; + props.setProperty(S3Constants.S3_BUCKET, bucket); + ds = new S3TestDataStore(props); + ds.setConfig(config); + ds.setCacheSize(0); + ds.init(dataStoreDir); + byte[] data = new byte[dataLength]; + randomGen.nextBytes(data); + DataRecord rec = ds.addRecord(new ByteArrayInputStream(data)); + assertEquals(data.length, rec.getLength()); + assertRecord(data, rec); + ds.close(); + + // turn encryption now. + props.setProperty(S3Constants.S3_BUCKET, bucket); + props.setProperty(S3Constants.S3_ENCRYPTION, + S3Constants.S3_ENCRYPTION_SSE_S3); + props.setProperty(S3Constants.S3_RENAME_KEYS, "true"); + ds = new S3TestDataStore(props); + ds.setConfig(config); + ds.setCacheSize(0); + ds.init(dataStoreDir); + + rec = ds.getRecord(rec.getIdentifier()); + assertEquals(data.length, rec.getLength()); + assertRecord(data, rec); + + randomGen.nextBytes(data); + rec = ds.addRecord(new ByteArrayInputStream(data)); + ds.close(); + + } catch (Exception e) { + LOG.error("error:", e); + fail(e.getMessage()); + } + } + +}