diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java index 47363c02b4..e2d7363203 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java @@ -86,32 +86,8 @@ public final class AzureUtilities { List blobList = new ArrayList<>(); ResultContinuation token = null; do { - ResultSegment result = null; - IOException lastException = null; - for (int i = 0; i < 10; i++) { - try { - result = directory.listBlobsSegmented( - null, - false, - EnumSet.of(BlobListingDetails.METADATA), - 2500, - token, - null, - null); - } catch (StorageException | URISyntaxException e) { - lastException = new IOException(e); - try { - Thread.sleep(1000); - } catch (InterruptedException ex) { - log.warn("Interrupted", e); - } - continue; - } - } - if (result == null) { - throw lastException; - } - for (ListBlobItem b : result.getResults()) { + ResultSegment result = listBlobsInSegments(directory, token); //get the blobs in pages of 5000 + for (ListBlobItem b : result.getResults()) { //add resultant blobs to list if (b instanceof CloudBlob) { CloudBlob cloudBlob = (CloudBlob) b; blobList.add(cloudBlob); @@ -119,7 +95,6 @@ public final class AzureUtilities { } token = result.getContinuationToken(); } while (token != null); - return blobList; } @@ -159,6 +134,38 @@ public final class AzureUtilities { return container.getDirectoryReference(dir); } + private static ResultSegment listBlobsInSegments(CloudBlobDirectory directory, + ResultContinuation token) throws IOException { + ResultSegment result = null; + IOException lastException = null; + for (int sleep = 10; sleep <= 10000; sleep *= 10) { //increment the sleep time in steps. + try { + result = directory.listBlobsSegmented( + null, + false, + EnumSet.of(BlobListingDetails.METADATA), + 5000, + token, + null, + null); + break; //we have the results, no need to retry + } catch (StorageException | URISyntaxException e) { + lastException = new IOException(e); + try { + Thread.sleep(sleep); //Sleep and retry + } catch (InterruptedException ex) { + log.warn("Interrupted", e); + } + continue; + } + } + if (result == null) { + throw lastException; + } else { + return result; + } + } + private static class ByteBufferOutputStream extends OutputStream { @NotNull @@ -178,6 +185,7 @@ public final class AzureUtilities { buffer.put(bytes, offset, length); } } + }