diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureRepositoryLock.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureRepositoryLock.java index 16a911bc70..60d7d0f3f3 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureRepositoryLock.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureRepositoryLock.java @@ -17,6 +17,7 @@ package org.apache.jackrabbit.oak.segment.azure; import com.microsoft.azure.storage.AccessCondition; +import com.microsoft.azure.storage.StorageErrorCodeStrings; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.blob.CloudBlockBlob; import org.apache.jackrabbit.oak.segment.spi.persistence.RepositoryLock; @@ -103,11 +104,15 @@ public class AzureRepositoryLock implements RepositoryLock { lastUpdate = System.currentTimeMillis(); } } catch (StorageException e) { + if (e.getErrorCode().equals(StorageErrorCodeStrings.OPERATION_TIMED_OUT)) { + log.warn("Could not renew the lease due to the operation timeout. Retry in progress ...", e); + } else { log.error("Can't renew the lease", e); shutdownHook.run(); doUpdate = false; return; } + } try { Thread.sleep(100); } catch (InterruptedException e) { diff --git a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureRepositoryLockTest.java b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureRepositoryLockTest.java index 5a0f7a5bb4..0991febe29 100644 --- a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureRepositoryLockTest.java +++ b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureRepositoryLockTest.java @@ -18,13 +18,16 @@ */ package org.apache.jackrabbit.oak.segment.azure; +import com.microsoft.azure.storage.StorageErrorCodeStrings; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.blob.CloudBlobContainer; import com.microsoft.azure.storage.blob.CloudBlockBlob; import org.apache.jackrabbit.oak.segment.spi.persistence.RepositoryLock; import org.junit.Before; import org.junit.ClassRule; +import org.junit.Ignore; import org.junit.Test; +import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +36,7 @@ import java.net.URISyntaxException; import java.rmi.server.ExportException; import java.security.InvalidKeyException; import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeoutException; import static org.junit.Assert.fail; @@ -81,4 +85,34 @@ public class AzureRepositoryLockTest { new AzureRepositoryLock(blob, () -> {}, 10).lock(); } + @Ignore + @Test + public void testLeaseRefreshUnsuccessful() throws URISyntaxException, StorageException, IOException, InterruptedException { + CloudBlockBlob blob = container.getBlockBlobReference("oak/repo.lock"); + + CloudBlockBlob blobMocked = Mockito.spy(blob); + + // instrument the mock to throw the exception twice when renewing the lease + StorageException storageException = + new StorageException(StorageErrorCodeStrings.OPERATION_TIMED_OUT, "operation timeout", new TimeoutException()); + Mockito.doThrow(storageException) + .doThrow(storageException) + .doCallRealMethod() + .when(blobMocked).renewLease(Mockito.any()); + + new AzureRepositoryLock(blobMocked, () -> {}, 0).lock(); + + // wait till lease expires + Thread.sleep(70000); + + // reset the mock to default behaviour + Mockito.doCallRealMethod().when(blobMocked).renewLease(Mockito.any()); + + try { + new AzureRepositoryLock(blobMocked, () -> {}, 0).lock(); + fail("The second lock should fail."); + } catch (IOException e) { + // it's fine + } + } } diff --git a/oak-segment-azure/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/oak-segment-azure/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 0000000000..ca6ee9cea8 --- /dev/null +++ b/oak-segment-azure/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline \ No newline at end of file