diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index a4693bd..bfb295a 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -1689,7 +1689,6 @@ private boolean drop_table_core(final RawStore ms, final String dbname, final St } } - checkTrashPurgeCombination(tblPath, dbname + "." + name, ifPurge, deleteData && !isExternal); // Drop the partitions and get a list of locations which need to be deleted partPaths = dropPartitionsAndGetLocations(ms, dbname, name, tblPath, tbl.getPartitionKeys(), deleteData && !isExternal); @@ -1731,46 +1730,6 @@ private boolean drop_table_core(final RawStore ms, final String dbname, final St } /** - * Will throw MetaException if combination of trash policy/purge can't be satisfied - * @param pathToData path to data which may potentially be moved to trash - * @param objectName db.table, or db.table.part - * @param ifPurge if PURGE options is specified - */ - private void checkTrashPurgeCombination(Path pathToData, String objectName, boolean ifPurge, - boolean deleteData) throws MetaException { - // There is no need to check TrashPurgeCombination in following cases since Purge/Trash - // is not applicable: - // a) deleteData is false -- drop an external table - // b) pathToData is null -- a view - // c) ifPurge is true -- force delete without Trash - if (!deleteData || pathToData == null || ifPurge) { - return; - } - - boolean trashEnabled = false; - try { - trashEnabled = 0 < hiveConf.getFloat("fs.trash.interval", -1); - } catch(NumberFormatException ex) { - // nothing to do - } - - if (trashEnabled) { - try { - HadoopShims.HdfsEncryptionShim shim = - ShimLoader.getHadoopShims().createHdfsEncryptionShim(FileSystem.get(hiveConf), hiveConf); - if (shim.isPathEncrypted(pathToData)) { - throw new MetaException("Unable to drop " + objectName + " because it is in an encryption zone" + - " and trash is enabled. Use PURGE option to skip trash."); - } - } catch (IOException ex) { - MetaException e = new MetaException(ex.getMessage()); - e.initCause(ex); - throw e; - } - } - } - - /** * Deletes the data in a table's location, if it fails logs an error * * @param tablePath @@ -3086,15 +3045,11 @@ private boolean drop_partition_common(RawStore ms, String db_name, String tbl_na if (isArchived) { archiveParentDir = MetaStoreUtils.getOriginalLocation(part); verifyIsWritablePath(archiveParentDir); - checkTrashPurgeCombination(archiveParentDir, db_name + "." + tbl_name + "." + part_vals, - mustPurge, deleteData && !isExternalTbl); } if ((part.getSd() != null) && (part.getSd().getLocation() != null)) { partPath = new Path(part.getSd().getLocation()); verifyIsWritablePath(partPath); - checkTrashPurgeCombination(partPath, db_name + "." + tbl_name + "." + part_vals, - mustPurge, deleteData && !isExternalTbl); } if (!ms.dropPartition(db_name, tbl_name, part_vals)) { @@ -3272,15 +3227,11 @@ public DropPartitionsResult drop_partitions_req( if (MetaStoreUtils.isArchived(part)) { Path archiveParentDir = MetaStoreUtils.getOriginalLocation(part); verifyIsWritablePath(archiveParentDir); - checkTrashPurgeCombination(archiveParentDir, dbName + "." + tblName + "." + - part.getValues(), mustPurge, deleteData && !isExternalTbl); archToDelete.add(archiveParentDir); } if ((part.getSd() != null) && (part.getSd().getLocation() != null)) { Path partPath = new Path(part.getSd().getLocation()); verifyIsWritablePath(partPath); - checkTrashPurgeCombination(partPath, dbName + "." + tblName + "." + part.getValues(), - mustPurge, deleteData && !isExternalTbl); dirsToDelete.add(new PathAndPartValSize(partPath, part.getValues().size())); } } diff --git a/ql/src/test/queries/clientpositive/encryption_drop_partition.q b/ql/src/test/queries/clientpositive/encryption_drop_partition.q index d968459..33e41fa 100644 --- a/ql/src/test/queries/clientpositive/encryption_drop_partition.q +++ b/ql/src/test/queries/clientpositive/encryption_drop_partition.q @@ -1,11 +1,11 @@ -- SORT_QUERY_RESULTS; --- we're setting this so that TestNegaiveCliDriver.vm doesn't stop processing after DROP TABLE fails; +-- we're setting this so that TestNegativeCliDriver.vm doesn't stop processing after DROP TABLE fails; set hive.cli.errors.ignore=true; set hive.exec.dynamic.partition.mode=nonstrict; set hive.mapred.mode=nonstrict; -DROP TABLE IF EXISTS encrypted_table_dp PURGE; +DROP TABLE IF EXISTS encrypted_table_dp; CREATE TABLE encrypted_table_dp (key INT, value STRING) partitioned by (p STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table_dp'; CRYPTO CREATE_KEY --keyName key_128 --bitLength 128; CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table_dp; @@ -23,8 +23,6 @@ DROP TABLE encrypted_ext_table_dp; SELECT * FROM encrypted_table_dp; ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23'); SELECT * FROM encrypted_table_dp; -ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23') PURGE; -SELECT * FROM encrypted_table_dp; TRUNCATE TABLE encrypted_table_dp PARTITION (p='2014-09-24'); SHOW PARTITIONS encrypted_table_dp; @@ -32,4 +30,3 @@ SELECT * FROM encrypted_table_dp; ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-24'); DROP TABLE encrypted_table_dp; -DROP TABLE encrypted_table_dp PURGE; diff --git a/ql/src/test/queries/clientpositive/encryption_drop_table.q b/ql/src/test/queries/clientpositive/encryption_drop_table.q index 76be118..6cffa2b 100644 --- a/ql/src/test/queries/clientpositive/encryption_drop_table.q +++ b/ql/src/test/queries/clientpositive/encryption_drop_table.q @@ -4,8 +4,8 @@ set hive.cli.errors.ignore=true; -DROP TABLE IF EXISTS encrypted_table PURGE; -DROP TABLE IF EXISTS encrypted_ext_table PURGE; +DROP TABLE IF EXISTS encrypted_table; +DROP TABLE IF EXISTS encrypted_ext_table; CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table'; CRYPTO CREATE_KEY --keyName key_128 --bitLength 128; @@ -22,10 +22,7 @@ SHOW TABLES; DROP TABLE default.encrypted_table; SHOW TABLES; -DROP TABLE default.encrypted_table PURGE; -SHOW TABLES; - -DROP TABLE IF EXISTS encrypted_table1 PURGE; +DROP TABLE IF EXISTS encrypted_table1; CREATE TABLE encrypted_table1 (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table1'; CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table1; INSERT OVERWRITE TABLE encrypted_table1 SELECT * FROM src; @@ -42,7 +39,4 @@ TRUNCATE TABLE encrypted_table1; DROP TABLE default.encrypted_table1; SHOW TABLES; -DROP TABLE default.encrypted_table1 PURGE; -SHOW TABLES; - CRYPTO DELETE_KEY --keyName key_128; diff --git a/ql/src/test/queries/clientpositive/encryption_drop_table_in_encrypted_db.q b/ql/src/test/queries/clientpositive/encryption_drop_table_in_encrypted_db.q new file mode 100644 index 0000000..f6c48e3 --- /dev/null +++ b/ql/src/test/queries/clientpositive/encryption_drop_table_in_encrypted_db.q @@ -0,0 +1,20 @@ +-- SORT_QUERY_RESULTS; + +set hive.cli.errors.ignore=true; + +DROP TABLE IF EXISTS encrypted_table; +DROP DATABASE IF EXISTS encrypted_db; + +-- create database encrypted_db in its default warehouse location {hiveconf:hive.metastore.warehouse.dir}/encrypted_db.db +CREATE DATABASE encrypted_db LOCATION '${hiveconf:hive.metastore.warehouse.dir}/encrypted_db.db'; +CRYPTO CREATE_KEY --keyName key_128 --bitLength 128; +CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/encrypted_db.db; + +CREATE TABLE encrypted_db.encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/encrypted_db.db/encrypted_table';; + +INSERT OVERWRITE TABLE encrypted_db.encrypted_table SELECT * FROM src; + +DROP TABLE encrypted_db.encrypted_table; + +DROP DATABASE encrypted_db; +CRYPTO DELETE_KEY --keyName key_128; diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out index e32feed..3c63c38 100644 --- a/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out +++ b/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out @@ -1,6 +1,6 @@ -PREHOOK: query: DROP TABLE IF EXISTS encrypted_table_dp PURGE +PREHOOK: query: DROP TABLE IF EXISTS encrypted_table_dp PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table_dp PURGE +POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table_dp POSTHOOK: type: DROPTABLE #### A masked pattern was here #### PREHOOK: type: CREATETABLE @@ -111,26 +111,7 @@ PREHOOK: query: ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23') PREHOOK: type: ALTERTABLE_DROPPARTS PREHOOK: Input: default@encrypted_table_dp PREHOOK: Output: default@encrypted_table_dp@p=2014-09-23 -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to drop default.encrypted_table_dp.[2014-09-23] because it is in an encryption zone and trash is enabled. Use PURGE option to skip trash. -PREHOOK: query: SELECT * FROM encrypted_table_dp -PREHOOK: type: QUERY -PREHOOK: Input: default@encrypted_table_dp -PREHOOK: Input: default@encrypted_table_dp@p=2014-09-23 -PREHOOK: Input: default@encrypted_table_dp@p=2014-09-24 -#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging -POSTHOOK: query: SELECT * FROM encrypted_table_dp -POSTHOOK: type: QUERY -POSTHOOK: Input: default@encrypted_table_dp -POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-23 -POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-24 -#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging -1 foo 2014-09-23 -2 bar 2014-09-24 -PREHOOK: query: ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23') PURGE -PREHOOK: type: ALTERTABLE_DROPPARTS -PREHOOK: Input: default@encrypted_table_dp -PREHOOK: Output: default@encrypted_table_dp@p=2014-09-23 -POSTHOOK: query: ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23') PURGE +POSTHOOK: query: ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23') POSTHOOK: type: ALTERTABLE_DROPPARTS POSTHOOK: Input: default@encrypted_table_dp POSTHOOK: Output: default@encrypted_table_dp@p=2014-09-23 @@ -172,17 +153,15 @@ PREHOOK: query: ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-24') PREHOOK: type: ALTERTABLE_DROPPARTS PREHOOK: Input: default@encrypted_table_dp PREHOOK: Output: default@encrypted_table_dp@p=2014-09-24 -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to drop default.encrypted_table_dp.[2014-09-24] because it is in an encryption zone and trash is enabled. Use PURGE option to skip trash. +POSTHOOK: query: ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-24') +POSTHOOK: type: ALTERTABLE_DROPPARTS +POSTHOOK: Input: default@encrypted_table_dp +POSTHOOK: Output: default@encrypted_table_dp@p=2014-09-24 PREHOOK: query: DROP TABLE encrypted_table_dp PREHOOK: type: DROPTABLE PREHOOK: Input: default@encrypted_table_dp PREHOOK: Output: default@encrypted_table_dp -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Unable to drop default.encrypted_table_dp because it is in an encryption zone and trash is enabled. Use PURGE option to skip trash.) -PREHOOK: query: DROP TABLE encrypted_table_dp PURGE -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@encrypted_table_dp -PREHOOK: Output: default@encrypted_table_dp -POSTHOOK: query: DROP TABLE encrypted_table_dp PURGE +POSTHOOK: query: DROP TABLE encrypted_table_dp POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@encrypted_table_dp POSTHOOK: Output: default@encrypted_table_dp diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out index a3f41d8..d3e0282 100644 --- a/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out +++ b/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out @@ -1,10 +1,10 @@ -PREHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE +PREHOOK: query: DROP TABLE IF EXISTS encrypted_table PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE +POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table POSTHOOK: type: DROPTABLE -PREHOOK: query: DROP TABLE IF EXISTS encrypted_ext_table PURGE +PREHOOK: query: DROP TABLE IF EXISTS encrypted_ext_table PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE IF EXISTS encrypted_ext_table PURGE +POSTHOOK: query: DROP TABLE IF EXISTS encrypted_ext_table POSTHOOK: type: DROPTABLE #### A masked pattern was here #### PREHOOK: type: CREATETABLE @@ -67,20 +67,7 @@ PREHOOK: query: DROP TABLE default.encrypted_table PREHOOK: type: DROPTABLE PREHOOK: Input: default@encrypted_table PREHOOK: Output: default@encrypted_table -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Unable to drop default.encrypted_table because it is in an encryption zone and trash is enabled. Use PURGE option to skip trash.) -PREHOOK: query: SHOW TABLES -PREHOOK: type: SHOWTABLES -PREHOOK: Input: database:default -POSTHOOK: query: SHOW TABLES -POSTHOOK: type: SHOWTABLES -POSTHOOK: Input: database:default -encrypted_table -src -PREHOOK: query: DROP TABLE default.encrypted_table PURGE -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@encrypted_table -PREHOOK: Output: default@encrypted_table -POSTHOOK: query: DROP TABLE default.encrypted_table PURGE +POSTHOOK: query: DROP TABLE default.encrypted_table POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@encrypted_table POSTHOOK: Output: default@encrypted_table @@ -91,9 +78,9 @@ POSTHOOK: query: SHOW TABLES POSTHOOK: type: SHOWTABLES POSTHOOK: Input: database:default src -PREHOOK: query: DROP TABLE IF EXISTS encrypted_table1 PURGE +PREHOOK: query: DROP TABLE IF EXISTS encrypted_table1 PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table1 PURGE +POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table1 POSTHOOK: type: DROPTABLE #### A masked pattern was here #### PREHOOK: type: CREATETABLE @@ -154,42 +141,22 @@ PREHOOK: query: DROP TABLE default.encrypted_table1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@encrypted_table1 PREHOOK: Output: default@encrypted_table1 -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Unable to drop default.encrypted_table1 because it is in an encryption zone and trash is enabled. Use PURGE option to skip trash.) -PREHOOK: query: SHOW TABLES -PREHOOK: type: SHOWTABLES -PREHOOK: Input: database:default -POSTHOOK: query: SHOW TABLES -POSTHOOK: type: SHOWTABLES -POSTHOOK: Input: database:default -encrypted_table1 -src -PREHOOK: query: TRUNCATE TABLE encrypted_table1 -PREHOOK: type: TRUNCATETABLE -PREHOOK: Output: default@encrypted_table1 -POSTHOOK: query: TRUNCATE TABLE encrypted_table1 -POSTHOOK: type: TRUNCATETABLE +POSTHOOK: query: DROP TABLE default.encrypted_table1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@encrypted_table1 POSTHOOK: Output: default@encrypted_table1 -PREHOOK: query: DROP TABLE default.encrypted_table1 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@encrypted_table1 -PREHOOK: Output: default@encrypted_table1 -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Unable to drop default.encrypted_table1 because it is in an encryption zone and trash is enabled. Use PURGE option to skip trash.) PREHOOK: query: SHOW TABLES PREHOOK: type: SHOWTABLES PREHOOK: Input: database:default POSTHOOK: query: SHOW TABLES POSTHOOK: type: SHOWTABLES POSTHOOK: Input: database:default -encrypted_table1 src -PREHOOK: query: DROP TABLE default.encrypted_table1 PURGE +FAILED: SemanticException [Error 10001]: Table not found encrypted_table1 +PREHOOK: query: DROP TABLE default.encrypted_table1 PREHOOK: type: DROPTABLE -PREHOOK: Input: default@encrypted_table1 -PREHOOK: Output: default@encrypted_table1 -POSTHOOK: query: DROP TABLE default.encrypted_table1 PURGE +POSTHOOK: query: DROP TABLE default.encrypted_table1 POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@encrypted_table1 -POSTHOOK: Output: default@encrypted_table1 PREHOOK: query: SHOW TABLES PREHOOK: type: SHOWTABLES PREHOOK: Input: database:default diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_drop_table_in_encrypted_db.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_drop_table_in_encrypted_db.q.out new file mode 100644 index 0000000..1287d01 --- /dev/null +++ b/ql/src/test/results/clientpositive/encrypted/encryption_drop_table_in_encrypted_db.q.out @@ -0,0 +1,53 @@ +PREHOOK: query: DROP TABLE IF EXISTS encrypted_table +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP DATABASE IF EXISTS encrypted_db +PREHOOK: type: DROPDATABASE +POSTHOOK: query: DROP DATABASE IF EXISTS encrypted_db +POSTHOOK: type: DROPDATABASE +#### A masked pattern was here #### +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:encrypted_db +#### A masked pattern was here #### +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:encrypted_db +#### A masked pattern was here #### +Encryption key created: 'key_128' +Encryption zone created: '/build/ql/test/data/warehouse/encrypted_db.db' using key: 'key_128' +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:encrypted_db +PREHOOK: Output: encrypted_db@encrypted_table +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:encrypted_db +POSTHOOK: Output: encrypted_db@encrypted_table +PREHOOK: query: INSERT OVERWRITE TABLE encrypted_db.encrypted_table SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: encrypted_db@encrypted_table +POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_db.encrypted_table SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: encrypted_db@encrypted_table +POSTHOOK: Lineage: encrypted_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: encrypted_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: DROP TABLE encrypted_db.encrypted_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: encrypted_db@encrypted_table +PREHOOK: Output: encrypted_db@encrypted_table +POSTHOOK: query: DROP TABLE encrypted_db.encrypted_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: encrypted_db@encrypted_table +POSTHOOK: Output: encrypted_db@encrypted_table +PREHOOK: query: DROP DATABASE encrypted_db +PREHOOK: type: DROPDATABASE +PREHOOK: Input: database:encrypted_db +PREHOOK: Output: database:encrypted_db +POSTHOOK: query: DROP DATABASE encrypted_db +POSTHOOK: type: DROPDATABASE +POSTHOOK: Input: database:encrypted_db +POSTHOOK: Output: database:encrypted_db