jackrabbit-aws-ext/pom.xml | 4 +- .../java/org/apache/jackrabbit/aws/ext/Utils.java | 69 ++++++++++++++++++---- .../apache/jackrabbit/aws/ext/ds/S3Backend.java | 45 +++----------- .../jackrabbit/aws/ext/ds/S3TestDataStore.java | 11 ++-- .../jackrabbit/aws/ext/ds/TestS3DSAsyncTouch.java | 12 +++- .../aws/ext/ds/TestS3DSWithSmallCache.java | 13 +++- .../org/apache/jackrabbit/aws/ext/ds/TestS3Ds.java | 13 +++- .../jackrabbit/aws/ext/ds/TestS3DsCacheOff.java | 4 +- .../src/test/resources/log4j.properties | 4 +- .../apache/jackrabbit/core/data/LocalCache.java | 12 ++-- .../apache/jackrabbit/core/data/TestCaseBase.java | 11 +--- 11 files changed, 118 insertions(+), 80 deletions(-) diff --git a/jackrabbit-aws-ext/pom.xml b/jackrabbit-aws-ext/pom.xml index cb763c6..8bb98e2 100644 --- a/jackrabbit-aws-ext/pom.xml +++ b/jackrabbit-aws-ext/pom.xml @@ -43,8 +43,8 @@ com.amazonaws - aws-java-sdk - 1.9.11 + aws-java-sdk-s3 + 1.9.22 org.apache.jackrabbit diff --git a/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/Utils.java b/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/Utils.java index e564de7..d5d0ba1 100644 --- a/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/Utils.java +++ b/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/Utils.java @@ -30,6 +30,7 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.model.Region; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.model.ObjectListing; @@ -47,6 +48,22 @@ public final class Utils { private static final String DELETE_CONFIG_SUFFIX = ";burn"; /** + * The default value AWS bucket region. + */ + public static final String DEFAULT_AWS_BUCKET_REGION = "us-standard"; + + /** + * constants to define endpoint to various AWS region + */ + public static final String AWSDOTCOM = "amazonaws.com"; + + public static final String S3 = "s3"; + + public static final String DOT = "."; + + public static final String DASH = "-"; + + /** * private constructor so that class cannot initialized from outside. */ private Utils() { @@ -63,20 +80,29 @@ public final class Utils { AWSCredentials credentials = new BasicAWSCredentials( prop.getProperty(S3Constants.ACCESS_KEY), prop.getProperty(S3Constants.SECRET_KEY)); - int connectionTimeOut = Integer.parseInt(prop.getProperty(S3Constants.S3_CONN_TIMEOUT)); - int socketTimeOut = Integer.parseInt(prop.getProperty(S3Constants.S3_SOCK_TIMEOUT)); - int maxConnections = Integer.parseInt(prop.getProperty(S3Constants.S3_MAX_CONNS)); - int maxErrorRetry = Integer.parseInt(prop.getProperty(S3Constants.S3_MAX_ERR_RETRY)); - ClientConfiguration cc = new ClientConfiguration(); - String protocol = prop.getProperty(S3Constants.S3_CONN_PROTOCOL); - if ( protocol != null && protocol.equalsIgnoreCase("http")) { - cc.setProtocol(Protocol.HTTP); + AmazonS3Client s3service = new AmazonS3Client(credentials, getClientConfiguration(prop)); + String region = prop.getProperty(S3Constants.S3_REGION); + String endpoint = null; + String propEndPoint = prop.getProperty(S3Constants.S3_END_POINT); + if ((propEndPoint != null) & !"".equals(propEndPoint)) { + endpoint = propEndPoint; + } else { + if (DEFAULT_AWS_BUCKET_REGION.equals(region)) { + endpoint = S3 + DOT + AWSDOTCOM; + } else if (Region.EU_Ireland.toString().equals(region)) { + endpoint = "s3-eu-west-1" + DOT + AWSDOTCOM; + } else { + endpoint = S3 + DASH + region + DOT + AWSDOTCOM; + } } - cc.setConnectionTimeout(connectionTimeOut); - cc.setSocketTimeout(socketTimeOut); - cc.setMaxConnections(maxConnections); - cc.setMaxErrorRetry(maxErrorRetry); - return new AmazonS3Client(credentials, cc); + /* + * setting endpoint to remove latency of redirection. If endpoint is + * not set, invocation first goes us standard region, which + * redirects it to correct location. + */ + s3service.setEndpoint(endpoint); + LOG.info("S3 service endpoint [{}] ", endpoint); + return s3service; } /** @@ -142,4 +168,21 @@ public final class Utils { } } + private static ClientConfiguration getClientConfiguration(Properties prop) { + int connectionTimeOut = Integer.parseInt(prop.getProperty(S3Constants.S3_CONN_TIMEOUT)); + int socketTimeOut = Integer.parseInt(prop.getProperty(S3Constants.S3_SOCK_TIMEOUT)); + int maxConnections = Integer.parseInt(prop.getProperty(S3Constants.S3_MAX_CONNS)); + int maxErrorRetry = Integer.parseInt(prop.getProperty(S3Constants.S3_MAX_ERR_RETRY)); + ClientConfiguration cc = new ClientConfiguration(); + String protocol = prop.getProperty(S3Constants.S3_CONN_PROTOCOL); + if (protocol != null && protocol.equalsIgnoreCase("http")) { + cc.setProtocol(Protocol.HTTP); + } + cc.setConnectionTimeout(connectionTimeOut); + cc.setSocketTimeout(socketTimeOut); + cc.setMaxConnections(maxConnections); + cc.setMaxErrorRetry(maxErrorRetry); + return cc; + } + } diff --git a/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/ds/S3Backend.java b/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/ds/S3Backend.java index df9a5d6..804e7d2 100644 --- a/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/ds/S3Backend.java +++ b/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/ds/S3Backend.java @@ -77,22 +77,6 @@ public class S3Backend implements Backend { private static final String KEY_PREFIX = "dataStore_"; - /** - * The default value AWS bucket region. - */ - private static final String DEFAULT_AWS_BUCKET_REGION = "us-standard"; - - /** - * constants to define endpoint to various AWS region - */ - private static final String AWSDOTCOM = "amazonaws.com"; - - private static final String S3 = "s3"; - - private static final String DOT = "."; - - private static final String DASH = "-"; - private AmazonS3Client s3service; private String bucket; @@ -150,33 +134,19 @@ public class S3Backend implements Backend { } String region = prop.getProperty(S3Constants.S3_REGION); Region s3Region = null; - String endpoint = null; - if (DEFAULT_AWS_BUCKET_REGION.equals(region)) { + if (Utils.DEFAULT_AWS_BUCKET_REGION.equals(region)) { s3Region = Region.US_Standard; - endpoint = S3 + DOT + AWSDOTCOM; } else if (Region.EU_Ireland.toString().equals(region)) { s3Region = Region.EU_Ireland; - endpoint = "s3-eu-west-1" + DOT + AWSDOTCOM; } else { s3Region = Region.fromValue(region); - endpoint = S3 + DASH + region + DOT + AWSDOTCOM; } - String propEndPoint = prop.getProperty(S3Constants.S3_END_POINT); - if ((propEndPoint != null) & !"".equals(propEndPoint)) { - endpoint = propEndPoint; - } - /* - * setting endpoint to remove latency of redirection. If endpoint is - * not set, invocation first goes us standard region, which - * redirects it to correct location. - */ - s3service.setEndpoint(endpoint); - LOG.info("S3 service endpoint [{}] ", endpoint); - if (!s3service.doesBucketExist(bucket)) { s3service.createBucket(bucket, s3Region); LOG.info("Created bucket [{}] in [{}] ", bucket, region); + } else { + LOG.info("Using bucket [{}] in [{}] ", bucket, region); } int writeThreads = 10; @@ -200,8 +170,9 @@ public class S3Backend implements Backend { asyncWritePoolSize, new NamedThreadFactory("s3-write-worker")); String renameKeyProp = prop.getProperty(S3Constants.S3_RENAME_KEYS); boolean renameKeyBool = (renameKeyProp == null || "".equals(renameKeyProp)) - ? true + ? false : Boolean.parseBoolean(renameKeyProp); + LOG.info("Rename keys [{}]", renameKeyBool); if (renameKeyBool) { renameKeys(); } @@ -796,7 +767,7 @@ public class S3Backend implements Backend { + "] doesn't start with prefix [" + KEY_PREFIX + "]"); } String key = oldKey.substring(KEY_PREFIX.length()); - return key.substring(0, 4) + DASH + key.substring(4); + return key.substring(0, 4) + Utils.DASH + key.substring(4); } /** @@ -804,14 +775,14 @@ public class S3Backend implements Backend { */ private static String getKeyName(DataIdentifier identifier) { String key = identifier.toString(); - return key.substring(0, 4) + DASH + key.substring(4); + return key.substring(0, 4) + Utils.DASH + key.substring(4); } /** * Get data identifier from key. */ private static String getIdentifierName(String key) { - if (!key.contains(DASH)) { + if (!key.contains(Utils.DASH)) { return null; } return key.substring(0, 4) + key.substring(5); diff --git a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/S3TestDataStore.java b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/S3TestDataStore.java index 34206d3..33dbba4 100644 --- a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/S3TestDataStore.java +++ b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/S3TestDataStore.java @@ -16,6 +16,8 @@ */ package org.apache.jackrabbit.aws.ext.ds; +import java.util.Properties; + import org.apache.jackrabbit.core.data.Backend; /** @@ -25,20 +27,21 @@ import org.apache.jackrabbit.core.data.Backend; * bucket are not immediately reflected in the next test case. */ public class S3TestDataStore extends S3DataStore { - String bucket; + + Properties props; public S3TestDataStore() { super(); } - public S3TestDataStore(String bucket) { + public S3TestDataStore(Properties props) { super(); - this.bucket = bucket; + this.props = props; } protected Backend createBackend() { Backend backend = new S3Backend(); - ((S3Backend) backend).setBucket(bucket); + ((S3Backend) backend).setProperties(props); return backend; } } diff --git a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSAsyncTouch.java b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSAsyncTouch.java index 957ed7a..b4e9a18 100644 --- a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSAsyncTouch.java +++ b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSAsyncTouch.java @@ -16,8 +16,11 @@ */ package org.apache.jackrabbit.aws.ext.ds; +import java.io.IOException; + import javax.jcr.RepositoryException; +import org.apache.jackrabbit.aws.ext.S3Constants; import org.apache.jackrabbit.core.data.CachingDataStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,14 +34,19 @@ import org.slf4j.LoggerFactory; */ public class TestS3DSAsyncTouch extends TestS3Ds { - public TestS3DSAsyncTouch() { + protected static final Logger LOG = LoggerFactory.getLogger(TestS3DSAsyncTouch.class); + public TestS3DSAsyncTouch() throws IOException { config = System.getProperty(CONFIG); memoryBackend = false; noCache = false; } protected CachingDataStore createDataStore() throws RepositoryException { - ds = new S3TestDataStore(String.valueOf(randomGen.nextInt(9999)) + "-test"); + props.setProperty( + S3Constants.S3_BUCKET, + String.valueOf(randomGen.nextInt(9999)) + "-" + + String.valueOf(randomGen.nextInt(9999)) + "-test"); + ds = new S3TestDataStore(props); ds.setConfig(config); ds.init(dataStoreDir); ds.setTouchAsync(true); diff --git a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSWithSmallCache.java b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSWithSmallCache.java index 705c202..79d3de3 100644 --- a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSWithSmallCache.java +++ b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSWithSmallCache.java @@ -16,8 +16,11 @@ */ package org.apache.jackrabbit.aws.ext.ds; +import java.io.IOException; + import javax.jcr.RepositoryException; +import org.apache.jackrabbit.aws.ext.S3Constants; import org.apache.jackrabbit.core.data.CachingDataStore; import org.apache.jackrabbit.core.data.LocalCache; import org.slf4j.Logger; @@ -31,16 +34,20 @@ import org.slf4j.LoggerFactory; */ public class TestS3DSWithSmallCache extends TestS3Ds { - public TestS3DSWithSmallCache() { + public TestS3DSWithSmallCache() throws IOException { config = System.getProperty(CONFIG); memoryBackend = false; noCache = false; } protected CachingDataStore createDataStore() throws RepositoryException { - ds = new S3TestDataStore(String.valueOf(randomGen.nextInt(9999)) + "-test"); + props.setProperty( + S3Constants.S3_BUCKET, + String.valueOf(randomGen.nextInt(9999)) + "-" + + String.valueOf(randomGen.nextInt(9999)) + "-test"); + ds = new S3TestDataStore(props); ds.setConfig(config); - ds.setCacheSize(dataLength * 10); + ds.setCacheSize(dataLength * 20); ds.setCachePurgeTrigFactor(0.5d); ds.setCachePurgeResizeFactor(0.4d); ds.init(dataStoreDir); diff --git a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3Ds.java b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3Ds.java index b3c57bf..5a67f17 100644 --- a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3Ds.java +++ b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3Ds.java @@ -16,6 +16,7 @@ */ package org.apache.jackrabbit.aws.ext.ds; +import java.io.IOException; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -23,6 +24,7 @@ import java.util.Properties; import javax.jcr.RepositoryException; +import org.apache.jackrabbit.aws.ext.S3Constants; import org.apache.jackrabbit.aws.ext.Utils; import org.apache.jackrabbit.core.data.Backend; import org.apache.jackrabbit.core.data.CachingDataStore; @@ -48,10 +50,13 @@ public class TestS3Ds extends TestCaseBase { private Date startTime = null; - public TestS3Ds() { + protected Properties props; + + public TestS3Ds() throws IOException { config = System.getProperty(CONFIG); memoryBackend = false; noCache = false; + props = Utils.readConfig(config); } protected void setUp() throws Exception { @@ -64,7 +69,11 @@ public class TestS3Ds extends TestCaseBase { } protected CachingDataStore createDataStore() throws RepositoryException { - ds = new S3TestDataStore(String.valueOf(randomGen.nextInt(9999)) + "-test"); + props.setProperty( + S3Constants.S3_BUCKET, + String.valueOf(randomGen.nextInt(9999)) + "-" + + String.valueOf(randomGen.nextInt(9999)) + "-test"); + ds = new S3TestDataStore(props); ds.setConfig(config); if (noCache) { ds.setCacheSize(0); diff --git a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DsCacheOff.java b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DsCacheOff.java index 63e6547..beee6ca 100644 --- a/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DsCacheOff.java +++ b/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DsCacheOff.java @@ -16,6 +16,8 @@ */ package org.apache.jackrabbit.aws.ext.ds; +import java.io.IOException; + import org.apache.jackrabbit.core.data.CachingDataStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,7 +32,7 @@ public class TestS3DsCacheOff extends TestS3Ds { protected static final Logger LOG = LoggerFactory.getLogger(TestS3DsCacheOff.class); - public TestS3DsCacheOff() { + public TestS3DsCacheOff() throws IOException { config = System.getProperty(CONFIG); memoryBackend = false; noCache = true; diff --git a/jackrabbit-aws-ext/src/test/resources/log4j.properties b/jackrabbit-aws-ext/src/test/resources/log4j.properties index 5dca166..f815177 100644 --- a/jackrabbit-aws-ext/src/test/resources/log4j.properties +++ b/jackrabbit-aws-ext/src/test/resources/log4j.properties @@ -17,8 +17,8 @@ # this is the log4j configuration for the JCR API tests log4j.rootLogger=INFO, file -log4j.logger.org.apache.jackrabbit.core.data=DEBUG -log4j.logger.org.apache.jackrabbit.aws.ext.ds=DEBUG +log4j.logger.org.apache.jackrabbit.core.data=INFO +log4j.logger.org.apache.jackrabbit.aws.ext.ds=INFO #log4j.logger.org.apache.jackrabbit.test=DEBUG diff --git a/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/LocalCache.java b/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/LocalCache.java index ef9fd1a..f922215 100644 --- a/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/LocalCache.java +++ b/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/LocalCache.java @@ -24,12 +24,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.Set; @@ -218,8 +215,11 @@ public class LocalCache { if (destExists) { dest.setLastModified(System.currentTimeMillis()); } - LOG.debug("file [{}] exists= [{}] and adding to local cache.", - fileName, destExists); + LOG.debug( + "file [{}] exists= [{}] added to local cache, isLastModified [{}]", + new Object[] { dest.getAbsolutePath(), dest.exists(), + destExists }); + cache.put(fileName, dest.length()); result.setFile(dest); if (tryForAsyncUpload) { @@ -251,7 +251,7 @@ public class LocalCache { if (!f.exists() || (isInPurgeMode() && !asyncUploadCache.hasEntry(fileName, false))) { LOG.debug( "getFileIfStored returned: purgeMode=[{}], file=[{}] exists=[{}]", - new Object[] { isInPurgeMode(), fileName, f.exists() }); + new Object[] { isInPurgeMode(), f.getAbsolutePath(), f.exists() }); return null; } else { // touch entry in LRU caches diff --git a/jackrabbit-data/src/test/java/org/apache/jackrabbit/core/data/TestCaseBase.java b/jackrabbit-data/src/test/java/org/apache/jackrabbit/core/data/TestCaseBase.java index 4fd71d7..8889e4c 100644 --- a/jackrabbit-data/src/test/java/org/apache/jackrabbit/core/data/TestCaseBase.java +++ b/jackrabbit-data/src/test/java/org/apache/jackrabbit/core/data/TestCaseBase.java @@ -94,14 +94,9 @@ public abstract class TestCaseBase extends TestCase { @Override protected void setUp() throws Exception { dataStoreDir = TEST_DIR + "-" + + String.valueOf(randomGen.nextInt(dataLength)) + "-" + String.valueOf(randomGen.nextInt(dataLength)); - } - - /** - * Delete temporary directory. - */ - @Override - protected void tearDown() throws Exception { + // delete director if it exists boolean delSuccessFul = FileUtils.deleteQuietly(new File(dataStoreDir)); int retry = 2, count = 0; while (!delSuccessFul && count <= retry) { @@ -109,7 +104,7 @@ public abstract class TestCaseBase extends TestCase { delSuccessFul = FileUtils.deleteQuietly(new File(dataStoreDir)); count++; } - LOG.info("tearDown : directory [" + dataStoreDir + "] deleted [" + LOG.info("setup : directory [" + dataStoreDir + "] deleted [" + delSuccessFul + "]"); }