diff --git oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java index f82783f2df..0dcfeae036 100644 --- oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java +++ oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java @@ -24,6 +24,7 @@ import joptsimple.OptionParser; import joptsimple.OptionSet; import joptsimple.OptionSpec; import org.apache.jackrabbit.oak.run.commons.Command; +import org.apache.jackrabbit.oak.segment.azure.tool.AzureCompact; import org.apache.jackrabbit.oak.segment.tool.Compact; class CompactCommand implements Command { @@ -36,13 +37,13 @@ class CompactCommand implements Command { public void execute(String... args) throws Exception { OptionParser parser = new OptionParser(); OptionSpec directoryArg = parser.nonOptions( - "Path to segment store (required)").ofType(String.class); + "Path/URI to TAR/remote segment store (required)").ofType(String.class); OptionSpec mmapArg = parser.accepts("mmap", "Use memory mapped access if true, use file access if false. " + "If not specified, memory mapped access is used on 64 bit " + - "systems and file access is used on 32 bit systems. On " + - "Windows, regular file access is always enforced and this " + - "option is ignored.") + "systems and file access is used on 32 bit systems. For " + + "remote segment stores and on Windows, regular file access " + + "is always enforced and this option is ignored.") .withOptionalArg() .ofType(Boolean.class); OptionSpec forceArg = parser.accepts("force", @@ -61,15 +62,27 @@ class CompactCommand implements Command { System.exit(-1); } - int code = Compact.builder() - .withPath(new File(path)) - .withForce(isTrue(forceArg.value(options))) - .withMmap(mmapArg.value(options)) - .withOs(StandardSystemProperty.OS_NAME.value()) - .withSegmentCacheSize(Integer.getInteger("cache", 256)) - .withGCLogInterval(Long.getLong("compaction-progress-log", 150000)) - .build() - .run(); + int code = 0; + + if (path.startsWith("az:")) { + code = AzureCompact.builder() + .withPath(path.substring(3)) + .withForce(isTrue(forceArg.value(options))) + .withSegmentCacheSize(Integer.getInteger("cache", 256)) + .withGCLogInterval(Long.getLong("compaction-progress-log", 150000)) + .build() + .run(); + } else { + code = Compact.builder() + .withPath(new File(path)) + .withForce(isTrue(forceArg.value(options))) + .withMmap(mmapArg.value(options)) + .withOs(StandardSystemProperty.OS_NAME.value()) + .withSegmentCacheSize(Integer.getInteger("cache", 256)) + .withGCLogInterval(Long.getLong("compaction-progress-log", 150000)) + .build() + .run(); + } System.exit(code); } diff --git oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java index 468fd14eba..5368567040 100644 --- oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java +++ oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java @@ -16,17 +16,23 @@ */ package org.apache.jackrabbit.oak.segment.azure; +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.StorageCredentials; import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.StorageUri; import com.microsoft.azure.storage.blob.BlobListingDetails; import com.microsoft.azure.storage.blob.CloudBlob; +import com.microsoft.azure.storage.blob.CloudBlobContainer; import com.microsoft.azure.storage.blob.CloudBlobDirectory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.net.URI; import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.nio.file.Paths; +import java.security.InvalidKeyException; import java.util.EnumSet; import java.util.UUID; import java.util.stream.Stream; @@ -89,4 +95,21 @@ public final class AzureUtilities { } }); } + + public static CloudBlobDirectory cloudBlobDirectoryFrom(StorageCredentials credentials, + String uri, String dir) throws URISyntaxException, StorageException { + StorageUri storageUri = new StorageUri(new URI(uri)); + CloudBlobContainer container = new CloudBlobContainer(storageUri, credentials); + + return container.getDirectoryReference(dir); + } + + public static CloudBlobDirectory cloudBlobDirectoryFrom(String connection, String containerName, + String dir) throws InvalidKeyException, URISyntaxException, StorageException { + CloudStorageAccount cloud = CloudStorageAccount.parse(connection.toString()); + CloudBlobContainer container = cloud.createCloudBlobClient().getContainerReference(containerName); + container.createIfNotExists(); + + return container.getDirectoryReference(dir); + } } diff --git oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java new file mode 100644 index 0000000000..b79bc7ed87 --- /dev/null +++ oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jackrabbit.oak.segment.azure.tool; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.jackrabbit.oak.segment.SegmentCache.DEFAULT_SEGMENT_CACHE_MB; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_ACCOUNT_NAME; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_DIR; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_STORAGE_URI; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.parseAzureConfigurationFromUri; +import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.defaultGCOptions; + +import java.io.IOException; +import java.io.PrintStream; +import java.net.URISyntaxException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.jackrabbit.oak.segment.SegmentCache; +import org.apache.jackrabbit.oak.segment.azure.AzureJournalFile; +import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; +import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; +import org.apache.jackrabbit.oak.segment.file.FileStore; +import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; +import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; +import org.apache.jackrabbit.oak.segment.file.JournalReader; +import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitorAdapter; +import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitorAdapter; +import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile; +import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFileWriter; +import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveManager; +import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence; +import org.apache.jackrabbit.oak.segment.tool.Compact; + +import com.google.common.base.Stopwatch; +import com.google.common.io.Files; +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageCredentialsAccountAndKey; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.CloudBlobDirectory; + +/** + * Perform an offline compaction of an existing Azure Segment Store. + */ +public class AzureCompact { + + /** + * Create a builder for the {@link Compact} command. + * + * @return an instance of {@link Builder}. + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Collect options for the {@link Compact} command. + */ + public static class Builder { + + private String path; + + private boolean force; + + private long gcLogInterval = 150000; + + private int segmentCacheSize = DEFAULT_SEGMENT_CACHE_MB; + + private Builder() { + // Prevent external instantiation. + } + + /** + * The path (URI) to an existing segment store. This parameter is required. + * + * @param path + * the path to an existing segment store. + * @return this builder. + */ + public Builder withPath(String path) { + this.path = checkNotNull(path); + return this; + } + + /** + * Whether to fail if run on an older version of the store of force upgrading + * its format. + * + * @param force + * upgrade iff {@code true} + * @return this builder. + */ + public Builder withForce(boolean force) { + this.force = force; + return this; + } + + /** + * The size of the segment cache in MB. The default of + * {@link SegmentCache#DEFAULT_SEGMENT_CACHE_MB} when this method is not + * invoked. + * + * @param segmentCacheSize + * cache size in MB + * @return this builder + * @throws IllegalArgumentException + * if {@code segmentCacheSize} is not a positive integer. + */ + public Builder withSegmentCacheSize(int segmentCacheSize) { + checkArgument(segmentCacheSize > 0, "segmentCacheSize must be strictly positive"); + this.segmentCacheSize = segmentCacheSize; + return this; + } + + /** + * The number of nodes after which an update about the compaction process is + * logged. Set to a negative number to disable progress logging. If not + * specified, it defaults to 150,000 nodes. + * + * @param gcLogInterval + * The log interval. + * @return this builder. + */ + public Builder withGCLogInterval(long gcLogInterval) { + this.gcLogInterval = gcLogInterval; + return this; + } + + /** + * Create an executable version of the {@link Compact} command. + * + * @return an instance of {@link Runnable}. + */ + public AzureCompact build() { + checkNotNull(path); + return new AzureCompact(this); + } + + } + + private static String printableStopwatch(Stopwatch s) { + return String.format("%s (%ds)", s, s.elapsed(TimeUnit.SECONDS)); + } + + private final String path; + + private final int segmentCacheSize; + + private final boolean strictVersionCheck; + + private final long gcLogInterval; + + private AzureCompact(Builder builder) { + this.path = builder.path; + this.segmentCacheSize = builder.segmentCacheSize; + this.strictVersionCheck = !builder.force; + this.gcLogInterval = builder.gcLogInterval; + } + + public int run() { + Stopwatch watch = Stopwatch.createStarted(); + CloudBlobDirectory cloudBlobDirectory = null; + try { + cloudBlobDirectory = createCloudBlobDirectory(); + } catch (URISyntaxException | StorageException e1) { + throw new IllegalArgumentException( + "Could not connect to the Azure Storage. Please verify the path provided!"); + } + + SegmentNodeStorePersistence persistence = new AzurePersistence(cloudBlobDirectory); + SegmentArchiveManager archiveManager = null; + try { + archiveManager = persistence.createArchiveManager(false, new IOMonitorAdapter(), + new FileStoreMonitorAdapter()); + } catch (IOException e) { + throw new IllegalArgumentException( + "Could not access the Azure Storage. Please verify the path provided!"); + } + + System.out.printf("Compacting %s\n", path); + System.out.printf(" before\n"); + List beforeArchives = Collections.emptyList(); + try { + beforeArchives = archiveManager.listArchives(); + } catch (IOException e) { + System.err.println(e); + } + + printArchives(System.out, beforeArchives); + System.out.printf(" -> compacting\n"); + + try (FileStore store = newFileStore(persistence)) { + if (!store.compactFull()) { + System.out.printf("Compaction cancelled after %s.\n", printableStopwatch(watch)); + return 1; + } + System.out.printf(" -> cleaning up\n"); + store.cleanup(); + JournalFile journal = new AzureJournalFile(cloudBlobDirectory, "journal.log"); + String head; + try (JournalReader journalReader = new JournalReader(journal)) { + head = String.format("%s root %s\n", journalReader.next().getRevision(), System.currentTimeMillis()); + } + + try (JournalFileWriter journalWriter = journal.openJournalWriter()) { + System.out.printf(" -> writing new %s: %s\n", journal.getName(), head); + journalWriter.truncate(); + journalWriter.writeLine(head); + } + } catch (Exception e) { + watch.stop(); + e.printStackTrace(System.err); + System.out.printf("Compaction failed after %s.\n", printableStopwatch(watch)); + return 1; + } + + watch.stop(); + System.out.printf(" after\n"); + List afterArchives = Collections.emptyList(); + try { + afterArchives = archiveManager.listArchives(); + } catch (IOException e) { + System.err.println(e); + } + printArchives(System.out, afterArchives); + System.out.printf("Compaction succeeded in %s.\n", printableStopwatch(watch)); + return 0; + } + + private static void printArchives(PrintStream s, List archives) { + for (String a : archives) { + s.printf(" %s\n", a); + } + } + + private FileStore newFileStore(SegmentNodeStorePersistence persistence) + throws IOException, InvalidFileStoreVersionException, URISyntaxException, StorageException { + FileStoreBuilder builder = FileStoreBuilder.fileStoreBuilder(Files.createTempDir()) + .withCustomPersistence(persistence).withMemoryMapping(false).withStrictVersionCheck(strictVersionCheck) + .withSegmentCacheSize(segmentCacheSize) + .withGCOptions(defaultGCOptions().setOffline().setGCLogInterval(gcLogInterval)); + + return builder.build(); + } + + private CloudBlobDirectory createCloudBlobDirectory() throws URISyntaxException, StorageException { + Map config = parseAzureConfigurationFromUri(path); + + String accountName = config.get(KEY_ACCOUNT_NAME); + String key = System.getenv("AZURE_SECRET_KEY"); + StorageCredentials credentials = new StorageCredentialsAccountAndKey(accountName, key); + + String uri = config.get(KEY_STORAGE_URI); + String dir = config.get(KEY_DIR); + + return AzureUtilities.cloudBlobDirectoryFrom(credentials, uri, dir); + } +} diff --git oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/AzureParserUtils.java oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureConfigurationParserUtils.java similarity index 95% rename from oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/AzureParserUtils.java rename to oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureConfigurationParserUtils.java index efa50ada37..e09e7998ae 100644 --- oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/AzureParserUtils.java +++ oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureConfigurationParserUtils.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.jackrabbit.oak.upgrade.cli.parser; +package org.apache.jackrabbit.oak.segment.azure.util; -import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.AzureConnectionKey.*; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.AzureConnectionKey.*; import java.net.URI; import java.net.URISyntaxException; @@ -28,7 +28,7 @@ import java.util.Map; * string, container name, uri, etc.) from custom encoded String or Azure * standard URI. */ -public class AzureParserUtils { +public class AzureConfigurationParserUtils { public enum AzureConnectionKey { DEFAULT_ENDPOINTS_PROTOCOL("DefaultEndpointsProtocol"), ACCOUNT_NAME("AccountName"), @@ -54,7 +54,7 @@ public class AzureParserUtils { public static final String KEY_STORAGE_URI = "storageUri"; public static final String KEY_DIR = "directory"; - private AzureParserUtils() { + private AzureConfigurationParserUtils() { // prevent instantiation } @@ -152,3 +152,4 @@ public class AzureParserUtils { return config; } } + diff --git oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/parser/AzureParserUtilsTest.java oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/config/AzureConfigurationParserUtilsTest.java similarity index 66% rename from oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/parser/AzureParserUtilsTest.java rename to oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/config/AzureConfigurationParserUtilsTest.java index 4f0fd433b8..84fac23d2b 100644 --- oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/parser/AzureParserUtilsTest.java +++ oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/config/AzureConfigurationParserUtilsTest.java @@ -14,25 +14,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.jackrabbit.oak.upgrade.cli.parser; +package org.apache.jackrabbit.oak.segment.azure.config; -import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.KEY_ACCOUNT_NAME; -import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.KEY_CONNECTION_STRING; -import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.KEY_CONTAINER_NAME; -import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.KEY_DIR; -import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.KEY_STORAGE_URI; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_ACCOUNT_NAME; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_CONNECTION_STRING; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_CONTAINER_NAME; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_DIR; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_STORAGE_URI; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.util.Map; +import org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils; import org.junit.Test; -public class AzureParserUtilsTest { +public class AzureConfigurationParserUtilsTest { @Test - public void testParseConnectionDetailsFromCustomConnection() throws CliArgumentException { + public void testParseConnectionDetailsFromCustomConnection() { StringBuilder conn = new StringBuilder(); StringBuilder connStr = new StringBuilder(); connStr.append("DefaultEndpointsProtocol=https;"); @@ -44,16 +45,16 @@ public class AzureParserUtilsTest { conn.append("ContainerName=oak-test;"); conn.append("Directory=repository"); - assertTrue(AzureParserUtils.isCustomAzureConnectionString(conn.toString())); + assertTrue(AzureConfigurationParserUtils.isCustomAzureConnectionString(conn.toString())); - Map config = AzureParserUtils.parseAzureConfigurationFromCustomConnection(conn.toString()); + Map config = AzureConfigurationParserUtils.parseAzureConfigurationFromCustomConnection(conn.toString()); assertEquals(connStr.toString(), config.get(KEY_CONNECTION_STRING)); assertEquals("oak-test", config.get(KEY_CONTAINER_NAME)); assertEquals("repository", config.get(KEY_DIR)); } @Test - public void testParseConnectionDetailsFromCustomConnectionShuffledKeys() throws CliArgumentException { + public void testParseConnectionDetailsFromCustomConnectionShuffledKeys() { StringBuilder conn = new StringBuilder(); conn.append("Directory=repository;"); conn.append("DefaultEndpointsProtocol=https;"); @@ -62,21 +63,21 @@ public class AzureParserUtilsTest { conn.append("BlobEndpoint=http://127.0.0.1:32806/myaccount;"); conn.append("AccountKey=mykey=="); - assertTrue(AzureParserUtils.isCustomAzureConnectionString(conn.toString())); + assertTrue(AzureConfigurationParserUtils.isCustomAzureConnectionString(conn.toString())); String azureConn = "DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey==;BlobEndpoint=http://127.0.0.1:32806/myaccount;"; - Map config = AzureParserUtils.parseAzureConfigurationFromCustomConnection(conn.toString()); + Map config = AzureConfigurationParserUtils.parseAzureConfigurationFromCustomConnection(conn.toString()); assertEquals(azureConn, config.get(KEY_CONNECTION_STRING)); assertEquals("oak-test", config.get(KEY_CONTAINER_NAME)); assertEquals("repository", config.get(KEY_DIR)); } @Test - public void testParseConnectionDetailsFromUri() throws CliArgumentException { + public void testParseConnectionDetailsFromUri() { String uri = "https://myaccount.blob.core.windows.net/oak-test/repository"; - assertFalse(AzureParserUtils.isCustomAzureConnectionString(uri)); + assertFalse(AzureConfigurationParserUtils.isCustomAzureConnectionString(uri)); - Map config = AzureParserUtils.parseAzureConfigurationFromUri(uri); + Map config = AzureConfigurationParserUtils.parseAzureConfigurationFromUri(uri); assertEquals("myaccount", config.get(KEY_ACCOUNT_NAME)); assertEquals("https://myaccount.blob.core.windows.net/oak-test", config.get(KEY_STORAGE_URI)); diff --git oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java index 99ac19ae26..a073f94e8f 100644 --- oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java +++ oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java @@ -19,12 +19,12 @@ package org.apache.jackrabbit.oak.upgrade.cli.node; import static org.apache.jackrabbit.oak.upgrade.cli.node.FileStoreUtils.asCloseable; import java.io.IOException; -import java.net.URI; import java.net.URISyntaxException; import java.security.InvalidKeyException; import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; +import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; import org.apache.jackrabbit.oak.segment.file.FileStore; import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; @@ -35,12 +35,10 @@ import org.apache.jackrabbit.oak.upgrade.cli.node.FileStoreUtils.NodeStoreWithFi import com.google.common.io.Closer; import com.google.common.io.Files; -import com.microsoft.azure.storage.CloudStorageAccount; import com.microsoft.azure.storage.StorageCredentials; import com.microsoft.azure.storage.StorageCredentialsAccountAndKey; import com.microsoft.azure.storage.StorageException; -import com.microsoft.azure.storage.StorageUri; -import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.CloudBlobDirectory; public class SegmentAzureFactory implements NodeStoreFactory { private final String accountName; @@ -132,28 +130,21 @@ public class SegmentAzureFactory implements NodeStoreFactory { } private AzurePersistence createAzurePersistence() throws StorageException, URISyntaxException, InvalidKeyException { - AzurePersistence azPersistence = null; + CloudBlobDirectory cloudBlobDirectory = null; if (accountName != null && uri != null) { String key = System.getenv("AZURE_SECRET_KEY"); StorageCredentials credentials = new StorageCredentialsAccountAndKey(accountName, key); - StorageUri storageUri = new StorageUri(new URI(uri)); - CloudBlobContainer cloudBlobContainer = new CloudBlobContainer(storageUri, credentials); - - azPersistence = new AzurePersistence(cloudBlobContainer.getDirectoryReference(dir)); + cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(credentials, uri, dir); } else if (connectionString != null && containerName != null) { - CloudStorageAccount cloud = CloudStorageAccount.parse(connectionString.toString()); - CloudBlobContainer container = cloud.createCloudBlobClient().getContainerReference(containerName); - container.createIfNotExists(); - - azPersistence = new AzurePersistence(container.getDirectoryReference(dir)); + cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(connectionString, containerName, dir); } - if (azPersistence == null) { + if (cloudBlobDirectory == null) { throw new IllegalArgumentException("Could not connect to Azure storage. Too few connection parameters specified!"); } - return azPersistence; + return new AzurePersistence(cloudBlobDirectory); } @Override diff --git oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreType.java oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreType.java index c150f0f4a7..ad05ad3178 100644 --- oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreType.java +++ oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreType.java @@ -17,19 +17,18 @@ package org.apache.jackrabbit.oak.upgrade.cli.parser; import static org.apache.commons.lang.StringUtils.removeStart; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_ACCOUNT_NAME; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_CONNECTION_STRING; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_CONTAINER_NAME; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_DIR; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_STORAGE_URI; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.isCustomAzureConnectionString; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.parseAzureConfigurationFromCustomConnection; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.parseAzureConfigurationFromUri; import static org.apache.jackrabbit.oak.upgrade.cli.node.Jackrabbit2Factory.isJcr2Repository; import static org.apache.jackrabbit.oak.upgrade.cli.node.Jackrabbit2Factory.isRepositoryXml; -import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.isCustomAzureConnectionString; -import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.parseAzureConfigurationFromCustomConnection; -import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.parseAzureConfigurationFromUri; import static org.apache.jackrabbit.oak.upgrade.cli.parser.StoreArguments.SEGMENT_AZURE_PREFIX; import static org.apache.jackrabbit.oak.upgrade.cli.parser.StoreArguments.SEGMENT_OLD_PREFIX; -import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.KEY_ACCOUNT_NAME; -import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.KEY_STORAGE_URI; -import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.KEY_CONNECTION_STRING; -import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.KEY_CONTAINER_NAME; -import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.KEY_DIR; - import java.util.Map;