Index: oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java (revision 14181bae5142bd1139a3a4a12aa69b9e409aed09) +++ oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java (revision ba46bd3be9ef24a2ea384741878dc482eb701772) @@ -99,6 +99,7 @@ private static final String META_KEY_PREFIX = META_DIR_NAME + "/"; private static final String REF_KEY = "reference.key"; + private static final String LAST_MODIFIED_KEY = "lastModified"; private static final long BUFFERED_STREAM_THRESHHOLD = 1024 * 1024; static final long MIN_MULTIPART_UPLOAD_PART_SIZE = 1024 * 1024 * 10; // 10MB @@ -266,6 +267,8 @@ LOG.debug("Blob write started. identifier={} length={}", key, len); CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(key); if (!blob.exists()) { + addLastModified(blob); + BlobRequestOptions options = new BlobRequestOptions(); options.setConcurrentRequestCount(concurrentRequestCount); boolean useBufferedStream = len < BUFFERED_STREAM_THRESHHOLD; @@ -289,16 +292,13 @@ " new length=" + len + " old length=" + blob.getProperties().getLength()); } - LOG.trace("Blob already exists. identifier={} lastModified={}", key, blob.getProperties().getLastModified().getTime()); - blob.startCopy(blob); - //TODO: better way of updating lastModified (use custom metadata?) - if (!waitForCopy(blob)) { - throw new DataStoreException( - String.format("Cannot update lastModified for blob. identifier=%s status=%s", - key, blob.getCopyState().getStatusDescription())); - } + + LOG.trace("Blob already exists. identifier={} lastModified={}", key, getLastModified(blob)); + addLastModified(blob); + blob.uploadMetadata(); + LOG.debug("Blob updated. identifier={} lastModified={} duration={}", key, - blob.getProperties().getLastModified().getTime(), (System.currentTimeMillis() - start)); + getLastModified(blob), (System.currentTimeMillis() - start)); } catch (StorageException e) { LOG.info("Error writing blob. identifier={}", key, e); @@ -307,9 +307,6 @@ catch (URISyntaxException | IOException e) { LOG.debug("Error writing blob. identifier={}", key, e); throw new DataStoreException(String.format("Cannot write blob. identifier=%s", key), e); - } catch (InterruptedException e) { - LOG.debug("Error writing blob. identifier={}", key, e); - throw new DataStoreException(String.format("Cannot copy blob. identifier=%s", key), e); } finally { if (null != contextClassLoader) { Thread.currentThread().setContextClassLoader(contextClassLoader); @@ -387,7 +384,7 @@ connectionString, containerName, new DataIdentifier(getIdentifierName(blob.getName())), - blob.getProperties().getLastModified().getTime(), + getLastModified(blob), blob.getProperties().getLength()); LOG.debug("Data record read for blob. identifier={} duration={} record={}", key, (System.currentTimeMillis() - start), record); @@ -553,6 +550,7 @@ try { CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(META_DIR_NAME); CloudBlockBlob blob = metaDir.getBlockBlobReference(name); + addLastModified(blob); blob.upload(input, recordLength); } catch (StorageException e) { @@ -578,7 +576,7 @@ return null; } blob.downloadAttributes(); - long lastModified = blob.getProperties().getLastModified().getTime(); + long lastModified = getLastModified(blob); long length = blob.getProperties().getLength(); AzureBlobStoreDataRecord record = new AzureBlobStoreDataRecord(this, connectionString, @@ -617,12 +615,13 @@ for (ListBlobItem item : metaDir.listBlobs(prefix)) { if (item instanceof CloudBlob) { CloudBlob blob = (CloudBlob) item; + blob.downloadAttributes(); records.add(new AzureBlobStoreDataRecord( this, connectionString, containerName, new DataIdentifier(stripMetaKeyPrefix(blob.getName())), - blob.getProperties().getLastModified().getTime(), + getLastModified(blob), blob.getProperties().getLength(), true)); } @@ -762,6 +761,17 @@ return name; } + private static void addLastModified(CloudBlockBlob blob) { + blob.getMetadata().put(LAST_MODIFIED_KEY, String.valueOf(System.currentTimeMillis())); + } + + private static long getLastModified(CloudBlob blob) { + if (blob.getMetadata().containsKey(LAST_MODIFIED_KEY)) { + return Long.parseLong(blob.getMetadata().get(LAST_MODIFIED_KEY)); + } + return blob.getProperties().getLastModified().getTime(); + } + void setHttpDownloadURIExpirySeconds(int seconds) { httpDownloadURIExpirySeconds = seconds; } @@ -787,7 +797,7 @@ // When running unit test from Maven, it doesn't always honor the @NotNull decorators if (null == identifier) throw new NullPointerException("identifier"); if (null == downloadOptions) throw new NullPointerException("downloadOptions"); - + if (httpDownloadURIExpirySeconds > 0) { String domain = getDirectDownloadBlobStorageDomain(downloadOptions.isDomainOverrideIgnored()); @@ -1012,6 +1022,7 @@ AccessCondition.generateEmptyCondition(), null, null); + addLastModified(blob); blob.commitBlockList(blocks); long size = 0L; for (BlockEntry block : blocks) { @@ -1022,7 +1033,7 @@ connectionString, containerName, blobId, - blob.getProperties().getLastModified().getTime(), + getLastModified(blob), size); } else { @@ -1182,9 +1193,10 @@ return length; } - public static AzureBlobInfo fromCloudBlob(CloudBlob cloudBlob) { + public static AzureBlobInfo fromCloudBlob(CloudBlob cloudBlob) throws StorageException { + cloudBlob.downloadAttributes(); return new AzureBlobInfo(cloudBlob.getName(), - cloudBlob.getProperties().getLastModified().getTime(), + AzureBlobStoreBackend.getLastModified(cloudBlob), cloudBlob.getProperties().getLength()); } } Index: oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDS.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDS.java (revision 14181bae5142bd1139a3a4a12aa69b9e409aed09) +++ oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDS.java (revision ba46bd3be9ef24a2ea384741878dc482eb701772) @@ -42,7 +42,7 @@ public class TestAzureDS extends AbstractDataStoreTest { protected static final Logger LOG = LoggerFactory.getLogger(TestAzureDS.class); - protected Properties props; + protected Properties props = new Properties(); protected String container; @BeforeClass @@ -53,7 +53,7 @@ @Override @Before public void setUp() throws Exception { - props = AzureDataStoreUtils.getAzureConfig(); + props.putAll(AzureDataStoreUtils.getAzureConfig()); container = String.valueOf(randomGen.nextInt(9999)) + "-" + String.valueOf(randomGen.nextInt(9999)) + "-test"; props.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, container); Index: oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDSWithSmallCache.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDSWithSmallCache.java (revision 14181bae5142bd1139a3a4a12aa69b9e409aed09) +++ oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDSWithSmallCache.java (revision ba46bd3be9ef24a2ea384741878dc482eb701772) @@ -38,7 +38,7 @@ @Override @Before public void setUp() throws Exception { - super.setUp(); props.setProperty("cacheSize", String.valueOf(dataLength * 10)); + super.setUp(); } } Index: oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDsCacheOff.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDsCacheOff.java (revision 14181bae5142bd1139a3a4a12aa69b9e409aed09) +++ oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDsCacheOff.java (revision ba46bd3be9ef24a2ea384741878dc482eb701772) @@ -36,7 +36,7 @@ @Override @Before public void setUp() throws Exception { - super.setUp(); props.setProperty("cacheSize", "0"); + super.setUp(); } } Index: oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java (revision 14181bae5142bd1139a3a4a12aa69b9e409aed09) +++ oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java (revision ba46bd3be9ef24a2ea384741878dc482eb701772) @@ -119,6 +119,25 @@ } } + /** + * Testcase to validate {@link DataStore#addRecord(InputStream)} API. + */ + @Test + public void testAddDuplicateRecord() { + try { + long start = System.currentTimeMillis(); + LOG.info("Testcase: " + this.getClass().getName() + + "#testAddDuplicateRecord, testDir=" + dataStoreDir); + doAddDuplicateRecordTest(); + LOG.info("Testcase: " + this.getClass().getName() + + "#testAddDuplicateRecord finished, time taken = [" + + (System.currentTimeMillis() - start) + "]ms"); + } catch (Exception e) { + LOG.error("error:", e); + fail(e.getMessage()); + } + } + /** * Testcase to validate {@link DataStore#getRecord(DataIdentifier)} API. */ @@ -308,6 +327,24 @@ assertRecord(data, rec); } + /** + * Test {@link DataStore#addRecord(InputStream)} and assert length & last modified of copied + * record. + */ + protected void doAddDuplicateRecordTest() throws Exception { + byte[] data = new byte[dataLength]; + randomGen.nextBytes(data); + DataRecord rec = ds.addRecord(new ByteArrayInputStream(data)); + Assert.assertEquals(data.length, rec.getLength()); + assertRecord(data, rec); + + DataRecord rec2 = ds.addRecord(new ByteArrayInputStream(data)); + Assert.assertEquals(data.length, rec2.getLength()); + assertRecord(data, rec2); + + assertTrue("Copied record last modified not greater", rec.getLastModified() <= rec2.getLastModified()); + } + /** * Test {@link DataStore#getRecord(DataIdentifier)} and assert length and * inputstream. Index: oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandMetadataTest.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandMetadataTest.java (revision 14181bae5142bd1139a3a4a12aa69b9e409aed09) +++ oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandMetadataTest.java (revision ba46bd3be9ef24a2ea384741878dc482eb701772) @@ -165,12 +165,12 @@ REFERENCES.getNameFromIdPrefix(rep2Id, sessionId)); List expectations = Lists.newArrayList(); - expectations.add(Joiner.on("|").join(rep2Id, MILLISECONDS.toSeconds(expectAuxMetadataRecord.getLastModified()), - MILLISECONDS.toSeconds(expectAuxMarkerMetadataRecord.getLastModified()), "-")); - expectations.add(Joiner.on("|").join(repoId, MILLISECONDS.toSeconds(expectMainMetadataRecord.getLastModified()), - MILLISECONDS.toSeconds(expectMainMarkerMetadataRecord.getLastModified()), "*")); - expectations.add(Joiner.on("|").join(rep3Id, MILLISECONDS.toSeconds(expectAux2MetadataRecord.getLastModified()), - MILLISECONDS.toSeconds(expectAux2MarkerMetadataRecord.getLastModified()), "-")); + expectations.add(Joiner.on("|").join(rep2Id, MILLISECONDS.toSeconds(expectAuxMarkerMetadataRecord.getLastModified()), + MILLISECONDS.toSeconds(expectAuxMetadataRecord.getLastModified()), "-")); + expectations.add(Joiner.on("|").join(repoId, MILLISECONDS.toSeconds(expectMainMarkerMetadataRecord.getLastModified()), + MILLISECONDS.toSeconds(expectMainMetadataRecord.getLastModified()), "*")); + expectations.add(Joiner.on("|").join(rep3Id, MILLISECONDS.toSeconds(expectAux2MarkerMetadataRecord.getLastModified()), + MILLISECONDS.toSeconds(expectAux2MetadataRecord.getLastModified()), "-")); storeFixture.close(); return expectations;