diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java index 076f95ecbe..ff3837a0ad 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java @@ -161,7 +161,11 @@ public void authorize(Database db, Privilege[] readRequiredPriv, Privilege[] wri throw hiveException(ex); } - Path path = getDbLocation(db); + Path path; + path = wh.determineDatabaseExternalPath(db); + if (path == null) { + path = getDbLocation(db); + } // extract drop privileges DropPrivilegeExtractor privExtractor = new DropPrivilegeExtractor(readRequiredPriv, diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java index 5ec8ad8ab7..44022852a2 100755 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java @@ -205,6 +205,24 @@ public Path determineDatabasePath(Catalog cat, Database db) throws MetaException } } + /** + * Returns the external path if it exists in the expected location. Null otherwise + * @param db + * @return the path if it exists, null otherwise + * file system. + */ + public Path determineDatabaseExternalPath(Database db){ + if (hasExternalWarehouseRoot()) { + try { + return getDefaultExternalDatabasePath(db.getName()); + } catch (MetaException e) { + LOG.warn("Unable to determine external path for database " + db.getName()); + } + } + return null; + } + + private String dbDirFromDbName(Database db) throws MetaException { return db.getName().toLowerCase() + DATABASE_WAREHOUSE_SUFFIX; } @@ -296,7 +314,7 @@ public Path getDefaultDatabasePath(String dbName, boolean inExternalWH) throws M } } - private boolean hasExternalWarehouseRoot() { + public boolean hasExternalWarehouseRoot() { return !StringUtils.isBlank(whRootExternalString); } @@ -412,6 +430,16 @@ void addToChangeManagement(Path file) throws MetaException { } } + public boolean deleteDirIfEmpty(Path f) throws MetaException, IOException { + FileSystem fs = getFs(f); + if (FileUtils.isDirEmpty(fs, f)) { + return deleteDir(f, false, false, false); + } else { + LOG.info("Will not delete external directory " + f + " since it's not empty"); + } + return true; + } + public boolean deleteDir(Path f, boolean recursive, Database db) throws MetaException { return deleteDir(f, recursive, false, db); } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 22140a36f7..9438d3ae2f 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -34,6 +34,7 @@ import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Modifier; +import java.lang.reflect.UndeclaredThrowableException; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; @@ -1409,7 +1410,8 @@ private void create_database_core(RawStore ms, final Database db) LOG.error("No such catalog " + db.getCatalogName()); throw new InvalidObjectException("No such catalog " + db.getCatalogName()); } - Path dbPath = wh.determineDatabasePath(cat, db); + final Path dbPath = wh.determineDatabasePath(cat, db); + final Path dbExternalPath = wh.determineDatabaseExternalPath(db); db.setLocationUri(dbPath.toString()); if (db.getOwnerName() == null){ try { @@ -1421,18 +1423,72 @@ private void create_database_core(RawStore ms, final Database db) long time = System.currentTimeMillis()/1000; db.setCreateTime((int) time); boolean success = false; - boolean madeDir = false; + boolean madeManagedDir = false; + boolean madeExternalDir = false; boolean isReplicated = isDbReplicationTarget(db); Map transactionalListenersResponses = Collections.emptyMap(); try { firePreEvent(new PreCreateDatabaseEvent(db, this)); - if (!wh.isDir(dbPath)) { - LOG.debug("Creating database path " + dbPath); - if (!wh.mkdirs(dbPath)) { - throw new MetaException("Unable to create database path " + dbPath + - ", failed to create database " + db.getName()); + try { + // Since this may be done as random user (if doAs=true) end user may not have access + // to the default managed directory. We run this as an admin user + // if user explicitly specifies location within create statement, then we run this as end user. + UserGroupInformation ugi; + if(wh.getDefaultDatabasePath(db.getName()).toString().equals(dbPath)) { + ugi = UserGroupInformation.getLoginUser(); } - madeDir = true; + else { + ugi = UserGroupInformation.getCurrentUser(); + } + madeManagedDir = ugi.doAs(new PrivilegedExceptionAction() { + @Override + public Boolean run() throws MetaException { + if (!wh.isDir(dbPath)) { + LOG.info("Creating database path in managed directory " + dbPath); + if (!wh.mkdirs(dbPath)) { + throw new MetaException("Unable to create database managed path " + dbPath + + ", failed to create database " + db.getName()); + } + return true; + } + return false; + } + }); + if (madeManagedDir) { + LOG.info("Created database path in managed directory " + dbPath); + } + } catch (IOException | InterruptedException e) { + throw new MetaException("Unable to create database managed directory " + dbPath + + ", failed to create database " + db.getName()); + } + // We only create the external directory the external warehouse directory has been set + if (wh.hasExternalWarehouseRoot() && dbExternalPath != null) { + try { + madeExternalDir = UserGroupInformation.getCurrentUser().doAs( + new PrivilegedExceptionAction() { + @Override public Boolean run() throws MetaException { + if (!wh.isDir(dbExternalPath)) { + LOG.info("Creating database path in external directory " + dbExternalPath); + return wh.mkdirs(dbExternalPath); + } + return false; + } + }); + if (madeExternalDir) { + LOG.info("Created database path in external directory " + dbPath); + } else { + LOG.warn("Failed to create external path " + dbExternalPath + " for database " + db.getName() + + ". This may result in access not being allowed if the " + + "StorageBasedAuthorizationProvider is enabled "); + } + } catch (IOException | InterruptedException | UndeclaredThrowableException e) { + LOG.warn("Failed to create external path " + dbExternalPath + " for database " + db.getName() + + ". This may result in access not being allowed if the " + + "StorageBasedAuthorizationProvider is enabled: " + e.getMessage()); + } + } else { + LOG.info("Database external path won't be created since the external" + + "directory is not defined"); } ms.openTransaction(); @@ -1449,8 +1505,37 @@ private void create_database_core(RawStore ms, final Database db) } finally { if (!success) { ms.rollbackTransaction(); - if (madeDir) { - wh.deleteDir(dbPath, true, db); + + if (madeManagedDir) { + try { + UserGroupInformation.getLoginUser().doAs( + new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + wh.deleteDir(dbPath, true, db); + return null; + } + }); + } catch (IOException | InterruptedException e) { + LOG.error("Couldn't delete managed directory " + dbPath + " after " + + "it was created for database " + db.getName() + " " + e.getMessage()); + } + } + + if (madeExternalDir) { + try { + UserGroupInformation.getCurrentUser().doAs( + new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + wh.deleteDir(dbExternalPath, true, db); + return null; + } + }); + } catch (IOException | InterruptedException e) { + LOG.error("Couldn't delete external directory " + dbExternalPath + " after " + + "it was created for database " + db.getName() + " " + e.getMessage()); + } } } @@ -1782,12 +1867,45 @@ private void drop_database_core(RawStore ms, String catName, } // Delete the data in the database try { - wh.deleteDir(new Path(db.getLocationUri()), true, db); - } catch (Exception e) { - LOG.error("Failed to delete database directory: " + db.getLocationUri() + - " " + e.getMessage()); + final Database dbFinal = db; + UserGroupInformation ugi; + // We run this as end-user if db location is not in default metastore warehouse directory + if(wh.getDefaultDatabasePath(db.getName()).toString().equals(db.getLocationUri())) { + ugi = UserGroupInformation.getLoginUser(); + } + else { + ugi = UserGroupInformation.getCurrentUser(); + } + Boolean deleted = ugi.doAs(new PrivilegedExceptionAction() { + @Override + public Boolean run() throws MetaException { + return wh.deleteDir(new Path(dbFinal.getLocationUri()), true, dbFinal); + } + }); + if (!deleted) { + LOG.error("Failed to delete database folder " + db.getLocationUri()); + } + } catch (IOException | InterruptedException | UndeclaredThrowableException e) { + LOG.error("Might have failed to delete the database folder for database: " + db.getLocationUri() + + " " + e.getMessage()); } // it is not a terrible thing even if the data is not deleted + try { + final Path externalPath = wh.determineDatabaseExternalPath(db); + if (externalPath != null) { + Boolean deleted = UserGroupInformation.getCurrentUser().doAs(new PrivilegedExceptionAction() { + @Override public Boolean run() throws IOException, MetaException { + return wh.deleteDirIfEmpty(externalPath); + } + }); + if (!deleted) { + LOG.error("Failed to delete database external folder " + externalPath); + } + } + } catch (IOException | InterruptedException e) { + LOG.error("Might have failed to delete the database external folder for database: " + db.getName() + + " " + e.getMessage()); + } } if (!listeners.isEmpty()) { diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index dc91b71db9..3088961a3c 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.metastore; +import java.io.FileNotFoundException; import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.sql.Connection; @@ -44,6 +45,7 @@ import static org.mockito.Mockito.mock; import com.google.common.collect.Sets; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec; import org.apache.hadoop.hive.metastore.api.GetPartitionsProjectionSpec; import org.apache.hadoop.hive.metastore.api.GetPartitionsRequest; @@ -59,6 +61,7 @@ import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.MetastoreVersionInfo; import org.apache.hadoop.hive.metastore.utils.SecurityUtils; +import org.apache.hadoop.security.UserGroupInformation; import org.datanucleus.api.jdo.JDOPersistenceManager; import org.datanucleus.api.jdo.JDOPersistenceManagerFactory; import org.junit.Assert; @@ -1209,6 +1212,51 @@ public void testDatabaseLocationWithPermissionProblems() throws Exception { assertTrue("Database creation succeeded even with permission problem", createFailed); } + @Test + public void testExternalDirectory() throws Exception{ + String externalDirString = MetastoreConf.getVar(conf, ConfVars.WAREHOUSE_EXTERNAL); + Path externalDir = new Path(externalDirString); + silentDropDatabase(TEST_DB1_NAME); + + String dbLocation = + MetastoreConf.getVar(conf, ConfVars.WAREHOUSE) + "/test/_testDB_create_"; + + String dbExternalLocation = externalDirString + "/testdb1.db"; + + + FileSystem fs = FileSystem.get(new Path(dbLocation).toUri(), conf); + fs.mkdirs( + new Path(MetastoreConf.getVar(conf, ConfVars.WAREHOUSE)), + new FsPermission((short) 700)); + + fs.mkdirs(externalDir, new FsPermission((short) 0777)); + + Database db = new DatabaseBuilder() + .setName(TEST_DB1_NAME) + .setLocation(dbLocation) + .build(conf); + client.createDatabase(db); + FileStatus fileStatus = fs.getFileStatus(new Path(dbExternalLocation)); + + assertTrue("External folder should have been created", fileStatus.isDirectory()); + assertEquals("External folder should have the right permissions", new FsPermission((short) 0755), + fileStatus.getPermission()); + assertEquals("External folder should be owned by the right username", + UserGroupInformation.getCurrentUser().getShortUserName(), fileStatus.getOwner()); + client.dropDatabase(db.getName()); + + try { + fs.getFileStatus(new Path(dbExternalLocation)); + fail("External directory should have been deleted"); + } catch (FileNotFoundException e) { + } finally { + fs.delete(new Path(MetastoreConf.getVar(conf, ConfVars.WAREHOUSE) + "/test"), true); + fs.delete(new Path(MetastoreConf.getVar(conf, + ConfVars.WAREHOUSE_EXTERNAL) + "/test"), true); + } + } + + @Test public void testDatabaseLocation() throws Throwable { try { diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java index 437d7c352d..a6443eee9f 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java @@ -541,7 +541,7 @@ public void testAddPartitionForExternalTableNullLocation() throws Exception { client.getPartition(DB_NAME, tableName, Lists.newArrayList(DEFAULT_YEAR_VALUE)); Assert.assertNotNull(resultPart); Assert.assertNotNull(resultPart.getSd()); - String defaultTableLocation = table.getSd().getLocation(); + String defaultTableLocation = metaStore.getExternalWarehouseRoot() + "/" + DB_NAME + ".db/" + tableName; String defaulPartitionLocation = defaultTableLocation + "/year=2017"; Assert.assertEquals(defaulPartitionLocation, resultPart.getSd().getLocation()); } diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/AbstractMetaStoreService.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/AbstractMetaStoreService.java index dc6203f266..dbb2ff21d2 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/AbstractMetaStoreService.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/AbstractMetaStoreService.java @@ -112,7 +112,7 @@ public IMetaStoreClient getClient() throws MetaException { public Path getWarehouseRoot() throws MetaException { return warehouse.getWhRoot(); } - + /** * Returns the External MetaStore Warehouse root directory name. *