commit fdd77fa942b7f540a4174282dd29a6dabb26e470 Author: David Lavati Date: Wed Oct 9 13:09:33 2019 +0200 HIVE-22308 Add missing support of Azure Blobstore schemes Change-Id: I5e2fe35e5cb3118fe49c2ac3eb27bd185ea7de2a diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index afee315378..7da934b717 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -4813,7 +4813,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal /* BLOBSTORE section */ - HIVE_BLOBSTORE_SUPPORTED_SCHEMES("hive.blobstore.supported.schemes", "s3,s3a,s3n", + HIVE_BLOBSTORE_SUPPORTED_SCHEMES("hive.blobstore.supported.schemes", "abfs,abfss,s3,s3a,s3n,wasb,wasbs", "Comma-separated list of supported blobstore schemes."), HIVE_BLOBSTORE_USE_BLOBSTORE_AS_SCRATCHDIR("hive.blobstore.use.blobstore.as.scratchdir", false, diff --git a/common/src/test/org/apache/hadoop/hive/common/TestBlobStorageUtils.java b/common/src/test/org/apache/hadoop/hive/common/TestBlobStorageUtils.java index b135be82a2..19871f93bc 100644 --- a/common/src/test/org/apache/hadoop/hive/common/TestBlobStorageUtils.java +++ b/common/src/test/org/apache/hadoop/hive/common/TestBlobStorageUtils.java @@ -40,7 +40,7 @@ @Before public void setUp() { - conf.set(HiveConf.ConfVars.HIVE_BLOBSTORE_SUPPORTED_SCHEMES.varname, "s3a,swift"); + conf.set(HiveConf.ConfVars.HIVE_BLOBSTORE_SUPPORTED_SCHEMES.varname, "s3a,swift,abfss,wasb"); conf.setBoolean(HiveConf.ConfVars.HIVE_BLOBSTORE_USE_BLOBSTORE_AS_SCRATCHDIR.varname, false); } @@ -49,6 +49,7 @@ public void testValidAndInvalidPaths() throws IOException { // Valid paths assertTrue(isBlobStoragePath(conf, new Path("s3a://bucket/path"))); assertTrue(isBlobStoragePath(conf, new Path("swift://bucket/path"))); + assertTrue(isBlobStoragePath(conf, new Path("abfss://bucket/path"))); // Invalid paths assertFalse(isBlobStoragePath(conf, new Path("/tmp/a-path"))); @@ -70,6 +71,9 @@ public void testValidAndInvalidFileSystems() { doReturn("swift").when(fs).getScheme(); assertTrue(isBlobStorageFileSystem(conf, fs)); + doReturn("wasb").when(fs).getScheme(); + assertTrue(isBlobStorageFileSystem(conf, fs)); + /* Invalid FileSystem schemes */ doReturn("hdfs").when(fs).getScheme(); @@ -86,6 +90,7 @@ public void testValidAndInvalidSchemes() { // Valid schemes assertTrue(isBlobStorageScheme(conf, "s3a")); assertTrue(isBlobStorageScheme(conf, "swift")); + assertTrue(isBlobStorageScheme(conf, "abfss")); // Invalid schemes assertFalse(isBlobStorageScheme(conf, "hdfs"));