Index: oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3Backend.java =================================================================== --- oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3Backend.java (revision 1752390) +++ oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3Backend.java (working copy) @@ -18,6 +18,7 @@ package org.apache.jackrabbit.oak.blob.cloud.aws.s3; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; @@ -41,6 +42,8 @@ import com.google.common.collect.AbstractIterator; import com.google.common.collect.Lists; import com.google.common.collect.Maps; + +import org.apache.commons.io.IOUtils; import org.apache.jackrabbit.core.data.AsyncTouchCallback; import org.apache.jackrabbit.core.data.AsyncTouchResult; import org.apache.jackrabbit.core.data.AsyncUploadCallback; @@ -612,13 +615,26 @@ public void addMetadataRecord(final InputStream input, final String name) throws DataStoreException { ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); + File temp = null; try { Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); + // spool record to a temporary file so that the size of the data is known + // and the stream doesn't need to be buffered in memory. this makes uploading + // potentially large records (like the list of blobs marked by the GC) + // much more efficient (or even possible). + temp = File.createTempFile(getClass().getSimpleName(), "MetadataRecord"); + try (FileOutputStream os = new FileOutputStream(temp)) { + IOUtils.copyLarge(input, os); + } Upload upload = tmx.upload(s3ReqDecorator - .decorate(new PutObjectRequest(bucket, addMetaKeyPrefix(name), input, new ObjectMetadata()))); + .decorate(new PutObjectRequest(bucket, addMetaKeyPrefix(name), temp))); upload.waitForUploadResult(); + } catch (IOException e) { + LOG.error("Exception while writing temp file for metadata record [{}], {}", + new Object[] {name, e}); + throw new DataStoreException("Error in uploading", e); } catch (InterruptedException e) { LOG.error("Error in uploading", e); throw new DataStoreException("Error in uploading", e); @@ -626,6 +642,9 @@ if (contextClassLoader != null) { Thread.currentThread().setContextClassLoader(contextClassLoader); } + if (temp != null) { + temp.delete(); + } } }