commit 53c545225d1dd9b6fcbcfb99d264651ac36f1b04 Author: David Lavati Date: Wed Oct 9 13:09:33 2019 +0200 HIVE-22308 Add missing support of Azure Blobstore schemes Change-Id: I5e2fe35e5cb3118fe49c2ac3eb27bd185ea7de2a diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index afee315378..de9445e3fb 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -4813,7 +4813,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal /* BLOBSTORE section */ - HIVE_BLOBSTORE_SUPPORTED_SCHEMES("hive.blobstore.supported.schemes", "s3,s3a,s3n", + HIVE_BLOBSTORE_SUPPORTED_SCHEMES("hive.blobstore.supported.schemes", "abfs,abfss,http,https,s3,s3a,s3n,wasb,wasbs", "Comma-separated list of supported blobstore schemes."), HIVE_BLOBSTORE_USE_BLOBSTORE_AS_SCRATCHDIR("hive.blobstore.use.blobstore.as.scratchdir", false, diff --git a/common/src/test/org/apache/hadoop/hive/common/TestBlobStorageUtils.java b/common/src/test/org/apache/hadoop/hive/common/TestBlobStorageUtils.java index b135be82a2..1ac2857e22 100644 --- a/common/src/test/org/apache/hadoop/hive/common/TestBlobStorageUtils.java +++ b/common/src/test/org/apache/hadoop/hive/common/TestBlobStorageUtils.java @@ -40,7 +40,7 @@ @Before public void setUp() { - conf.set(HiveConf.ConfVars.HIVE_BLOBSTORE_SUPPORTED_SCHEMES.varname, "s3a,swift"); + conf.set(HiveConf.ConfVars.HIVE_BLOBSTORE_SUPPORTED_SCHEMES.varname, "s3a,swift,abfss,https,wasb"); conf.setBoolean(HiveConf.ConfVars.HIVE_BLOBSTORE_USE_BLOBSTORE_AS_SCRATCHDIR.varname, false); } @@ -49,6 +49,8 @@ public void testValidAndInvalidPaths() throws IOException { // Valid paths assertTrue(isBlobStoragePath(conf, new Path("s3a://bucket/path"))); assertTrue(isBlobStoragePath(conf, new Path("swift://bucket/path"))); + assertTrue(isBlobStoragePath(conf, new Path("abfss://bucket/path"))); + assertTrue(isBlobStoragePath(conf, new Path("https://bucket/path"))); // Invalid paths assertFalse(isBlobStoragePath(conf, new Path("/tmp/a-path"))); @@ -70,6 +72,9 @@ public void testValidAndInvalidFileSystems() { doReturn("swift").when(fs).getScheme(); assertTrue(isBlobStorageFileSystem(conf, fs)); + doReturn("wasb").when(fs).getScheme(); + assertTrue(isBlobStorageFileSystem(conf, fs)); + /* Invalid FileSystem schemes */ doReturn("hdfs").when(fs).getScheme(); @@ -86,6 +91,7 @@ public void testValidAndInvalidSchemes() { // Valid schemes assertTrue(isBlobStorageScheme(conf, "s3a")); assertTrue(isBlobStorageScheme(conf, "swift")); + assertTrue(isBlobStorageScheme(conf, "abfss")); // Invalid schemes assertFalse(isBlobStorageScheme(conf, "hdfs"));