jackrabbit-aws-ext/pom.xml | 4 +- .../org/apache/jackrabbit/aws/ext/S3Constants.java | 15 ++++ .../jackrabbit/aws/ext/S3RequestDecorator.java | 87 +++++++++++++++++++ .../java/org/apache/jackrabbit/aws/ext/Utils.java | 69 ++++++++++++--- .../apache/jackrabbit/aws/ext/ds/S3Backend.java | 78 +++++++---------- .../org/apache/jackrabbit/aws/ext/TestAll.java | 5 ++ .../jackrabbit/aws/ext/ds/S3TestDataStore.java | 13 +-- .../jackrabbit/aws/ext/ds/TestS3DSAsyncTouch.java | 9 +- .../jackrabbit/aws/ext/ds/TestS3DSWithSSES3.java | 98 ++++++++++++++++++++++ .../aws/ext/ds/TestS3DSWithSmallCache.java | 8 +- .../org/apache/jackrabbit/aws/ext/ds/TestS3Ds.java | 10 ++- .../jackrabbit/aws/ext/ds/TestS3DsCacheOff.java | 4 +- .../src/test/resources/log4j.properties | 4 +- 13 files changed, 325 insertions(+), 79 deletions(-) diff --git a/jackrabbit-aws-ext/pom.xml b/jackrabbit-aws-ext/pom.xml index cb763c6..8b486f3 100644 --- a/jackrabbit-aws-ext/pom.xml +++ b/jackrabbit-aws-ext/pom.xml @@ -43,8 +43,8 @@ com.amazonaws - aws-java-sdk - 1.9.11 + aws-java-sdk-s3 + 1.9.17 org.apache.jackrabbit diff --git a/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/S3Constants.java b/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/S3Constants.java index 02cdd5e..cd30275 100644 --- a/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/S3Constants.java +++ b/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/S3Constants.java @@ -83,6 +83,21 @@ public final class S3Constants { public static final String S3_WRITE_THREADS = "writeThreads"; /** + * Constant to enable encrption in S3. + */ + public static final String S3_ENCRYPTION = "s3Encrption"; + + /** + * Constant for no encryption. it is default. + */ + public static final String S3_ENCRYPTION_NONE = "NONE"; + + /** + * Constant to set SSE_S3 encryption. + */ + public static final String S3_ENCRYPTION_SSE_S3 = "SSE_S3"; + + /** * private constructor so that class cannot initialized from outside. */ private S3Constants() { diff --git a/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/S3RequestDecorator.java b/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/S3RequestDecorator.java new file mode 100644 index 0000000..2fcc244 --- /dev/null +++ b/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/S3RequestDecorator.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jackrabbit.aws.ext; + +import java.util.Properties; + +import com.amazonaws.services.s3.model.CopyObjectRequest; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PutObjectRequest; + +/** + * This class to sets encrption mode in S3 request. + * + */ +public class S3RequestDecorator { + DataEncryption dataEncryption = DataEncryption.NONE; + + public S3RequestDecorator(Properties props) { + if (props.getProperty(S3Constants.S3_ENCRYPTION) != null) { + this.dataEncryption = dataEncryption.valueOf(props.getProperty(S3Constants.S3_ENCRYPTION)); + } + } + + /** + * Set encryption in {@link PutObjectRequest} + */ + public PutObjectRequest decorate(PutObjectRequest request) { + switch (getDataEncryption()) { + case SSE_S3: + ObjectMetadata metadata = request.getMetadata() == null + ? new ObjectMetadata() + : request.getMetadata(); + metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); + request.setMetadata(metadata); + break; + case NONE: + break; + } + return request; + } + + /** + * Set encryption in {@link CopyObjectRequest} + */ + public CopyObjectRequest decorate(CopyObjectRequest request) { + switch (getDataEncryption()) { + case SSE_S3: + ObjectMetadata metadata = request.getNewObjectMetadata() == null + ? new ObjectMetadata() + : request.getNewObjectMetadata(); + metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); + request.setNewObjectMetadata(metadata); + break; + case NONE: + break; + } + return request; + } + + private DataEncryption getDataEncryption() { + return this.dataEncryption; + } + + /** + * Enum to indicate S3 encryption mode + * + */ + private enum DataEncryption { + SSE_S3, NONE; + } + +} diff --git a/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/Utils.java b/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/Utils.java index e564de7..3754bc5 100644 --- a/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/Utils.java +++ b/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/Utils.java @@ -30,6 +30,7 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.model.Region; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.model.ObjectListing; @@ -45,6 +46,22 @@ public final class Utils { public static final String DEFAULT_CONFIG_FILE = "aws.properties"; private static final String DELETE_CONFIG_SUFFIX = ";burn"; + + /** + * The default value AWS bucket region. + */ + public static final String DEFAULT_AWS_BUCKET_REGION = "us-standard"; + + /** + * constants to define endpoint to various AWS region + */ + public static final String AWSDOTCOM = "amazonaws.com"; + + public static final String S3 = "s3"; + + public static final String DOT = "."; + + public static final String DASH = "-"; /** * private constructor so that class cannot initialized from outside. @@ -63,20 +80,29 @@ public final class Utils { AWSCredentials credentials = new BasicAWSCredentials( prop.getProperty(S3Constants.ACCESS_KEY), prop.getProperty(S3Constants.SECRET_KEY)); - int connectionTimeOut = Integer.parseInt(prop.getProperty(S3Constants.S3_CONN_TIMEOUT)); - int socketTimeOut = Integer.parseInt(prop.getProperty(S3Constants.S3_SOCK_TIMEOUT)); - int maxConnections = Integer.parseInt(prop.getProperty(S3Constants.S3_MAX_CONNS)); - int maxErrorRetry = Integer.parseInt(prop.getProperty(S3Constants.S3_MAX_ERR_RETRY)); - ClientConfiguration cc = new ClientConfiguration(); - String protocol = prop.getProperty(S3Constants.S3_CONN_PROTOCOL); - if ( protocol != null && protocol.equalsIgnoreCase("http")) { - cc.setProtocol(Protocol.HTTP); + AmazonS3Client s3service = new AmazonS3Client(credentials, getClientConfiguration(prop)); + String region = prop.getProperty(S3Constants.S3_REGION); + String endpoint = null; + String propEndPoint = prop.getProperty(S3Constants.S3_END_POINT); + if ((propEndPoint != null) & !"".equals(propEndPoint)) { + endpoint = propEndPoint; + } else { + if (DEFAULT_AWS_BUCKET_REGION.equals(region)) { + endpoint = S3 + DOT + AWSDOTCOM; + } else if (Region.EU_Ireland.toString().equals(region)) { + endpoint = "s3-eu-west-1" + DOT + AWSDOTCOM; + } else { + endpoint = S3 + DASH + region + DOT + AWSDOTCOM; + } } - cc.setConnectionTimeout(connectionTimeOut); - cc.setSocketTimeout(socketTimeOut); - cc.setMaxConnections(maxConnections); - cc.setMaxErrorRetry(maxErrorRetry); - return new AmazonS3Client(credentials, cc); + /* + * setting endpoint to remove latency of redirection. If endpoint is + * not set, invocation first goes us standard region, which + * redirects it to correct location. + */ + s3service.setEndpoint(endpoint); + LOG.info("S3 service endpoint [{}] ", endpoint); + return s3service; } /** @@ -141,5 +167,22 @@ public final class Utils { LOG.warn("Could not delete " + file.getAbsolutePath()); } } + + private static ClientConfiguration getClientConfiguration(Properties prop) { + int connectionTimeOut = Integer.parseInt(prop.getProperty(S3Constants.S3_CONN_TIMEOUT)); + int socketTimeOut = Integer.parseInt(prop.getProperty(S3Constants.S3_SOCK_TIMEOUT)); + int maxConnections = Integer.parseInt(prop.getProperty(S3Constants.S3_MAX_CONNS)); + int maxErrorRetry = Integer.parseInt(prop.getProperty(S3Constants.S3_MAX_ERR_RETRY)); + ClientConfiguration cc = new ClientConfiguration(); + String protocol = prop.getProperty(S3Constants.S3_CONN_PROTOCOL); + if (protocol != null && protocol.equalsIgnoreCase("http")) { + cc.setProtocol(Protocol.HTTP); + } + cc.setConnectionTimeout(connectionTimeOut); + cc.setSocketTimeout(socketTimeOut); + cc.setMaxConnections(maxConnections); + cc.setMaxErrorRetry(maxErrorRetry); + return cc; + } } diff --git a/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/ds/S3Backend.java b/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/ds/S3Backend.java index df9a5d6..c7b2e6e 100644 --- a/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/ds/S3Backend.java +++ b/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/ds/S3Backend.java @@ -34,6 +34,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.jackrabbit.aws.ext.S3Constants; +import org.apache.jackrabbit.aws.ext.S3RequestDecorator; import org.apache.jackrabbit.aws.ext.Utils; import org.apache.jackrabbit.core.data.AsyncTouchCallback; import org.apache.jackrabbit.core.data.AsyncTouchResult; @@ -77,21 +78,6 @@ public class S3Backend implements Backend { private static final String KEY_PREFIX = "dataStore_"; - /** - * The default value AWS bucket region. - */ - private static final String DEFAULT_AWS_BUCKET_REGION = "us-standard"; - - /** - * constants to define endpoint to various AWS region - */ - private static final String AWSDOTCOM = "amazonaws.com"; - - private static final String S3 = "s3"; - - private static final String DOT = "."; - - private static final String DASH = "-"; private AmazonS3Client s3service; @@ -106,6 +92,8 @@ public class S3Backend implements Backend { private Date startTime; private ThreadPoolExecutor asyncWriteExecuter; + + private S3RequestDecorator s3ReqDecorator; /** * Initialize S3Backend. It creates AmazonS3Client and TransferManager from @@ -144,39 +132,28 @@ public class S3Backend implements Backend { getClass().getClassLoader()); LOG.debug("init"); this.store = store; + s3ReqDecorator = new S3RequestDecorator(prop); + s3service = Utils.openService(prop); if (bucket == null || "".equals(bucket.trim())) { bucket = prop.getProperty(S3Constants.S3_BUCKET); } String region = prop.getProperty(S3Constants.S3_REGION); Region s3Region = null; - String endpoint = null; - if (DEFAULT_AWS_BUCKET_REGION.equals(region)) { + + if (Utils.DEFAULT_AWS_BUCKET_REGION.equals(region)) { s3Region = Region.US_Standard; - endpoint = S3 + DOT + AWSDOTCOM; } else if (Region.EU_Ireland.toString().equals(region)) { s3Region = Region.EU_Ireland; - endpoint = "s3-eu-west-1" + DOT + AWSDOTCOM; } else { s3Region = Region.fromValue(region); - endpoint = S3 + DASH + region + DOT + AWSDOTCOM; } - String propEndPoint = prop.getProperty(S3Constants.S3_END_POINT); - if ((propEndPoint != null) & !"".equals(propEndPoint)) { - endpoint = propEndPoint; - } - /* - * setting endpoint to remove latency of redirection. If endpoint is - * not set, invocation first goes us standard region, which - * redirects it to correct location. - */ - s3service.setEndpoint(endpoint); - LOG.info("S3 service endpoint [{}] ", endpoint); - if (!s3service.doesBucketExist(bucket)) { s3service.createBucket(bucket, s3Region); LOG.info("Created bucket [{}] in [{}] ", bucket, region); + } else { + LOG.info("Using bucket [{}] in [{}] ", bucket, region); } int writeThreads = 10; @@ -200,8 +177,9 @@ public class S3Backend implements Backend { asyncWritePoolSize, new NamedThreadFactory("s3-write-worker")); String renameKeyProp = prop.getProperty(S3Constants.S3_RENAME_KEYS); boolean renameKeyBool = (renameKeyProp == null || "".equals(renameKeyProp)) - ? true + ? false : Boolean.parseBoolean(renameKeyProp); + LOG.info("Rename keys [{}]", renameKeyBool); if (renameKeyBool) { renameKeys(); } @@ -293,7 +271,7 @@ public class S3Backend implements Backend { CopyObjectRequest copReq = new CopyObjectRequest(bucket, key, bucket, key); copReq.setNewObjectMetadata(objectMetaData); - Copy copy = tmx.copy(copReq); + Copy copy = tmx.copy(s3ReqDecorator.decorate(copReq)); copy.waitForCopyResult(); LOG.debug("[{}] touched took [{}] ms. ", identifier, (System.currentTimeMillis() - start)); @@ -375,7 +353,7 @@ public class S3Backend implements Backend { CopyObjectRequest copReq = new CopyObjectRequest(bucket, key, bucket, key); copReq.setNewObjectMetadata(new ObjectMetadata()); - Copy copy = tmx.copy(copReq); + Copy copy = tmx.copy(s3ReqDecorator.decorate(copReq)); copy.waitForCompletion(); LOG.debug("[{}] touched. time taken [{}] ms ", new Object[] { identifier, (System.currentTimeMillis() - start) }); @@ -650,7 +628,7 @@ public class S3Backend implements Backend { CopyObjectRequest copReq = new CopyObjectRequest(bucket, key, bucket, key); copReq.setNewObjectMetadata(objectMetaData); - Copy copy = tmx.copy(copReq); + Copy copy = tmx.copy(s3ReqDecorator.decorate(copReq)); try { copy.waitForCopyResult(); LOG.debug("lastModified of [{}] updated successfully.", identifier); @@ -670,8 +648,8 @@ public class S3Backend implements Backend { if (objectMetaData == null) { try { // start multipart parallel upload using amazon sdk - Upload up = tmx.upload(new PutObjectRequest(bucket, key, - file)); + Upload up = tmx.upload(s3ReqDecorator.decorate(new PutObjectRequest( + bucket, key, file))); // wait for upload to finish if (asyncUpload) { up.addProgressListener(new S3UploadProgressListener(up, @@ -720,8 +698,7 @@ public class S3Backend implements Backend { try { Thread.currentThread().setContextClassLoader( getClass().getClassLoader()); - ObjectListing prevObjectListing = s3service.listObjects(bucket, - KEY_PREFIX); + ObjectListing prevObjectListing = s3service.listObjects(bucket); List deleteList = new ArrayList(); int nThreads = Integer.parseInt(properties.getProperty("maxConnections")); ExecutorService executor = Executors.newFixedThreadPool(nThreads, @@ -732,8 +709,12 @@ public class S3Backend implements Backend { executor.execute(new KeyRenameThread(s3ObjSumm.getKey())); taskAdded = true; count++; - deleteList.add(new DeleteObjectsRequest.KeyVersion( - s3ObjSumm.getKey())); + // delete the object if it follows old key name format + if( s3ObjSumm.getKey().startsWith(KEY_PREFIX)) { + deleteList.add(new DeleteObjectsRequest.KeyVersion( + s3ObjSumm.getKey())); + } + } if (!prevObjectListing.isTruncated()) break; prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing); @@ -792,11 +773,10 @@ public class S3Backend implements Backend { private static String convertKey(String oldKey) throws IllegalArgumentException { if (!oldKey.startsWith(KEY_PREFIX)) { - throw new IllegalArgumentException("[" + oldKey - + "] doesn't start with prefix [" + KEY_PREFIX + "]"); + return oldKey; } String key = oldKey.substring(KEY_PREFIX.length()); - return key.substring(0, 4) + DASH + key.substring(4); + return key.substring(0, 4) + Utils.DASH + key.substring(4); } /** @@ -804,14 +784,14 @@ public class S3Backend implements Backend { */ private static String getKeyName(DataIdentifier identifier) { String key = identifier.toString(); - return key.substring(0, 4) + DASH + key.substring(4); + return key.substring(0, 4) + Utils.DASH + key.substring(4); } /** * Get data identifier from key. */ private static String getIdentifierName(String key) { - if (!key.contains(DASH)) { + if (!key.contains(Utils.DASH)) { return null; } return key.substring(0, 4) + key.substring(5); @@ -833,7 +813,7 @@ public class S3Backend implements Backend { String newS3Key = convertKey(oldKey); CopyObjectRequest copReq = new CopyObjectRequest(bucket, oldKey, bucket, newS3Key); - Copy copy = tmx.copy(copReq); + Copy copy = tmx.copy(s3ReqDecorator.decorate(copReq)); try { copy.waitForCopyResult(); LOG.debug("[{}] renamed to [{}] ", oldKey, newS3Key); @@ -841,7 +821,6 @@ public class S3Backend implements Backend { LOG.error(" Exception in renaming [{}] to [{}] ", new Object[] { ie, oldKey, newS3Key }); } - } finally { if (contextClassLoader != null) { Thread.currentThread().setContextClassLoader( @@ -931,6 +910,5 @@ public class S3Backend implements Backend { } } - } diff --git a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/TestAll.java b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/TestAll.java index 9292314..2ebe400 100644 --- a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/TestAll.java +++ b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/TestAll.java @@ -21,6 +21,7 @@ import junit.framework.Test; import junit.framework.TestCase; import junit.framework.TestSuite; +import org.apache.jackrabbit.aws.ext.ds.TestS3DSWithSSES3; import org.apache.jackrabbit.aws.ext.ds.TestS3Ds; import org.apache.jackrabbit.aws.ext.ds.TestS3DSAsyncTouch; import org.apache.jackrabbit.aws.ext.ds.TestS3DsCacheOff; @@ -44,6 +45,9 @@ public class TestAll extends TestCase { */ public static Test suite() { TestSuite suite = new TestSuite("S3 tests"); + System.setProperty( + TestCaseBase.CONFIG, + "C:/src/apache/jackrabbit-svn/jackrabbit/jackrabbit-aws-ext/src/test/resources/aws.properties"); String config = System.getProperty(TestCaseBase.CONFIG); LOG.info("config= " + config); if (config != null && !"".equals(config.trim())) { @@ -51,6 +55,7 @@ public class TestAll extends TestCase { suite.addTestSuite(TestS3DSAsyncTouch.class); suite.addTestSuite(TestS3DSWithSmallCache.class); suite.addTestSuite(TestS3DsCacheOff.class); + suite.addTestSuite(TestS3DSWithSSES3.class); } return suite; } diff --git a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/S3TestDataStore.java b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/S3TestDataStore.java index 34206d3..dfbfc7c 100644 --- a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/S3TestDataStore.java +++ b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/S3TestDataStore.java @@ -16,6 +16,8 @@ */ package org.apache.jackrabbit.aws.ext.ds; +import java.util.Properties; + import org.apache.jackrabbit.core.data.Backend; /** @@ -25,20 +27,21 @@ import org.apache.jackrabbit.core.data.Backend; * bucket are not immediately reflected in the next test case. */ public class S3TestDataStore extends S3DataStore { - String bucket; + + Properties props; public S3TestDataStore() { super(); } - - public S3TestDataStore(String bucket) { + + public S3TestDataStore(Properties props) { super(); - this.bucket = bucket; + this.props = props; } protected Backend createBackend() { Backend backend = new S3Backend(); - ((S3Backend) backend).setBucket(bucket); + ((S3Backend) backend).setProperties(props); return backend; } } diff --git a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSAsyncTouch.java b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSAsyncTouch.java index 957ed7a..6cf5bda 100644 --- a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSAsyncTouch.java +++ b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSAsyncTouch.java @@ -16,8 +16,11 @@ */ package org.apache.jackrabbit.aws.ext.ds; +import java.io.IOException; + import javax.jcr.RepositoryException; +import org.apache.jackrabbit.aws.ext.S3Constants; import org.apache.jackrabbit.core.data.CachingDataStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,14 +34,16 @@ import org.slf4j.LoggerFactory; */ public class TestS3DSAsyncTouch extends TestS3Ds { - public TestS3DSAsyncTouch() { + protected static final Logger LOG = LoggerFactory.getLogger(TestS3DSAsyncTouch.class); + public TestS3DSAsyncTouch() throws IOException { config = System.getProperty(CONFIG); memoryBackend = false; noCache = false; } protected CachingDataStore createDataStore() throws RepositoryException { - ds = new S3TestDataStore(String.valueOf(randomGen.nextInt(9999)) + "-test"); + props.setProperty(S3Constants.S3_BUCKET, String.valueOf(randomGen.nextInt(9999)) + "-test"); + ds = new S3TestDataStore(props); ds.setConfig(config); ds.init(dataStoreDir); ds.setTouchAsync(true); diff --git a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSWithSSES3.java b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSWithSSES3.java new file mode 100644 index 0000000..f9814cc --- /dev/null +++ b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSWithSSES3.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jackrabbit.aws.ext.ds; + +import java.io.ByteArrayInputStream; +import java.io.IOException; + +import javax.jcr.RepositoryException; + +import org.apache.jackrabbit.aws.ext.S3Constants; +import org.apache.jackrabbit.core.data.CachingDataStore; +import org.apache.jackrabbit.core.data.DataRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Test S3DataStore operation with SSE_S3 encryption. + */ +public class TestS3DSWithSSES3 extends TestS3Ds { + + protected static final Logger LOG = LoggerFactory.getLogger(TestS3DSWithSSES3.class); + + public TestS3DSWithSSES3() throws IOException { + config = System.getProperty(CONFIG); + memoryBackend = false; + noCache = true; + } + + protected CachingDataStore createDataStore() throws RepositoryException { + + props.setProperty(S3Constants.S3_BUCKET, + String.valueOf(randomGen.nextInt(9999)) + "-test"); + props.setProperty(S3Constants.S3_ENCRYPTION, + S3Constants.S3_ENCRYPTION_SSE_S3); + ds = new S3TestDataStore(props); + ds.setConfig(config); + ds.init(dataStoreDir); + return ds; + } + + /** + * Test data migration enabling SSE_S3 encryption. + */ + public void testDataMigration() { + try { + String bucket = String.valueOf(randomGen.nextInt(9999)) + "-test"; + props.setProperty(S3Constants.S3_BUCKET, bucket); + ds = new S3TestDataStore(props); + ds.setConfig(config); + ds.setCacheSize(0); + ds.init(dataStoreDir); + byte[] data = new byte[dataLength]; + randomGen.nextBytes(data); + DataRecord rec = ds.addRecord(new ByteArrayInputStream(data)); + assertEquals(data.length, rec.getLength()); + assertRecord(data, rec); + ds.close(); + + // turn encryption now. + props.setProperty(S3Constants.S3_BUCKET, bucket); + props.setProperty(S3Constants.S3_ENCRYPTION, + S3Constants.S3_ENCRYPTION_SSE_S3); + props.setProperty(S3Constants.S3_RENAME_KEYS, "true"); + ds = new S3TestDataStore(props); + ds.setConfig(config); + ds.setCacheSize(0); + ds.init(dataStoreDir); + + rec = ds.getRecord(rec.getIdentifier()); + assertEquals(data.length, rec.getLength()); + assertRecord(data, rec); + + randomGen.nextBytes(data); + rec = ds.addRecord(new ByteArrayInputStream(data)); + ds.close(); + + } catch (Exception e) { + LOG.error("error:", e); + fail(e.getMessage()); + } + } + +} diff --git a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSWithSmallCache.java b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSWithSmallCache.java index 705c202..90cfa33 100644 --- a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSWithSmallCache.java +++ b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSWithSmallCache.java @@ -16,8 +16,11 @@ */ package org.apache.jackrabbit.aws.ext.ds; +import java.io.IOException; + import javax.jcr.RepositoryException; +import org.apache.jackrabbit.aws.ext.S3Constants; import org.apache.jackrabbit.core.data.CachingDataStore; import org.apache.jackrabbit.core.data.LocalCache; import org.slf4j.Logger; @@ -31,14 +34,15 @@ import org.slf4j.LoggerFactory; */ public class TestS3DSWithSmallCache extends TestS3Ds { - public TestS3DSWithSmallCache() { + public TestS3DSWithSmallCache() throws IOException { config = System.getProperty(CONFIG); memoryBackend = false; noCache = false; } protected CachingDataStore createDataStore() throws RepositoryException { - ds = new S3TestDataStore(String.valueOf(randomGen.nextInt(9999)) + "-test"); + props.setProperty(S3Constants.S3_BUCKET, String.valueOf(randomGen.nextInt(9999)) + "-test"); + ds = new S3TestDataStore(props); ds.setConfig(config); ds.setCacheSize(dataLength * 10); ds.setCachePurgeTrigFactor(0.5d); diff --git a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3Ds.java b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3Ds.java index b3c57bf..0f408fb 100644 --- a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3Ds.java +++ b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3Ds.java @@ -16,6 +16,7 @@ */ package org.apache.jackrabbit.aws.ext.ds; +import java.io.IOException; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -23,6 +24,7 @@ import java.util.Properties; import javax.jcr.RepositoryException; +import org.apache.jackrabbit.aws.ext.S3Constants; import org.apache.jackrabbit.aws.ext.Utils; import org.apache.jackrabbit.core.data.Backend; import org.apache.jackrabbit.core.data.CachingDataStore; @@ -47,11 +49,14 @@ public class TestS3Ds extends TestCaseBase { protected static final Logger LOG = LoggerFactory.getLogger(TestS3Ds.class); private Date startTime = null; + + protected Properties props; - public TestS3Ds() { + public TestS3Ds() throws IOException { config = System.getProperty(CONFIG); memoryBackend = false; noCache = false; + props = Utils.readConfig(config); } protected void setUp() throws Exception { @@ -64,7 +69,8 @@ public class TestS3Ds extends TestCaseBase { } protected CachingDataStore createDataStore() throws RepositoryException { - ds = new S3TestDataStore(String.valueOf(randomGen.nextInt(9999)) + "-test"); + props.setProperty(S3Constants.S3_BUCKET, String.valueOf(randomGen.nextInt(9999)) + "-test"); + ds = new S3TestDataStore(props); ds.setConfig(config); if (noCache) { ds.setCacheSize(0); diff --git a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DsCacheOff.java b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DsCacheOff.java index 63e6547..beee6ca 100644 --- a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DsCacheOff.java +++ b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DsCacheOff.java @@ -16,6 +16,8 @@ */ package org.apache.jackrabbit.aws.ext.ds; +import java.io.IOException; + import org.apache.jackrabbit.core.data.CachingDataStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,7 +32,7 @@ public class TestS3DsCacheOff extends TestS3Ds { protected static final Logger LOG = LoggerFactory.getLogger(TestS3DsCacheOff.class); - public TestS3DsCacheOff() { + public TestS3DsCacheOff() throws IOException { config = System.getProperty(CONFIG); memoryBackend = false; noCache = true; diff --git a/jackrabbit-aws-ext/src/test/resources/log4j.properties b/jackrabbit-aws-ext/src/test/resources/log4j.properties index 5dca166..f815177 100644 --- a/jackrabbit-aws-ext/src/test/resources/log4j.properties +++ b/jackrabbit-aws-ext/src/test/resources/log4j.properties @@ -17,8 +17,8 @@ # this is the log4j configuration for the JCR API tests log4j.rootLogger=INFO, file -log4j.logger.org.apache.jackrabbit.core.data=DEBUG -log4j.logger.org.apache.jackrabbit.aws.ext.ds=DEBUG +log4j.logger.org.apache.jackrabbit.core.data=INFO +log4j.logger.org.apache.jackrabbit.aws.ext.ds=INFO #log4j.logger.org.apache.jackrabbit.test=DEBUG