diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index aa3d72d..9ef9b46 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -710,7 +710,8 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\ encryption_drop_view.q \ encryption_drop_partition.q \ encryption_with_trash.q \ - encryption_ctas.q + encryption_ctas.q \ + encryption_trash.q beeline.positive.exclude=add_part_exist.q,\ alter1.q,\ diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index f7b2ed7..9639a42 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -1786,7 +1786,6 @@ private boolean drop_table_core(final RawStore ms, final String dbname, final St } } - checkTrashPurgeCombination(tblPath, dbname + "." + name, ifPurge, deleteData && !isExternal); // Drop the partitions and get a list of locations which need to be deleted partPaths = dropPartitionsAndGetLocations(ms, dbname, name, tblPath, tbl.getPartitionKeys(), deleteData && !isExternal); @@ -1825,46 +1824,6 @@ private boolean drop_table_core(final RawStore ms, final String dbname, final St } /** - * Will throw MetaException if combination of trash policy/purge can't be satisfied - * @param pathToData path to data which may potentially be moved to trash - * @param objectName db.table, or db.table.part - * @param ifPurge if PURGE options is specified - */ - private void checkTrashPurgeCombination(Path pathToData, String objectName, boolean ifPurge, - boolean deleteData) throws MetaException { - // There is no need to check TrashPurgeCombination in following cases since Purge/Trash - // is not applicable: - // a) deleteData is false -- drop an external table - // b) pathToData is null -- a view - // c) ifPurge is true -- force delete without Trash - if (!deleteData || pathToData == null || ifPurge) { - return; - } - - boolean trashEnabled = false; - try { - trashEnabled = 0 < hiveConf.getFloat("fs.trash.interval", -1); - } catch(NumberFormatException ex) { - // nothing to do - } - - if (trashEnabled) { - try { - HadoopShims.HdfsEncryptionShim shim = - ShimLoader.getHadoopShims().createHdfsEncryptionShim(FileSystem.get(hiveConf), hiveConf); - if (shim.isPathEncrypted(pathToData)) { - throw new MetaException("Unable to drop " + objectName + " because it is in an encryption zone" + - " and trash is enabled. Use PURGE option to skip trash."); - } - } catch (IOException ex) { - MetaException e = new MetaException(ex.getMessage()); - e.initCause(ex); - throw e; - } - } - } - - /** * Deletes the data in a table's location, if it fails logs an error * * @param tablePath @@ -3124,15 +3083,11 @@ private boolean drop_partition_common(RawStore ms, String db_name, String tbl_na if (isArchived) { archiveParentDir = MetaStoreUtils.getOriginalLocation(part); verifyIsWritablePath(archiveParentDir); - checkTrashPurgeCombination(archiveParentDir, db_name + "." + tbl_name + "." + part_vals, - mustPurge, deleteData && !isExternalTbl); } if ((part.getSd() != null) && (part.getSd().getLocation() != null)) { partPath = new Path(part.getSd().getLocation()); verifyIsWritablePath(partPath); - checkTrashPurgeCombination(partPath, db_name + "." + tbl_name + "." + part_vals, - mustPurge, deleteData && !isExternalTbl); } if (!ms.dropPartition(db_name, tbl_name, part_vals)) { @@ -3307,15 +3262,11 @@ public DropPartitionsResult drop_partitions_req( if (MetaStoreUtils.isArchived(part)) { Path archiveParentDir = MetaStoreUtils.getOriginalLocation(part); verifyIsWritablePath(archiveParentDir); - checkTrashPurgeCombination(archiveParentDir, dbName + "." + tblName + "." + - part.getValues(), mustPurge, deleteData && !isExternalTbl); archToDelete.add(archiveParentDir); } if ((part.getSd() != null) && (part.getSd().getLocation() != null)) { Path partPath = new Path(part.getSd().getLocation()); verifyIsWritablePath(partPath); - checkTrashPurgeCombination(partPath, dbName + "." + tblName + "." + part.getValues(), - mustPurge, deleteData && !isExternalTbl); dirsToDelete.add(new PathAndPartValSize(partPath, part.getValues().size())); } } diff --git ql/src/test/queries/clientpositive/encryption_trash.q ql/src/test/queries/clientpositive/encryption_trash.q new file mode 100644 index 0000000..80a5d4c --- /dev/null +++ ql/src/test/queries/clientpositive/encryption_trash.q @@ -0,0 +1,15 @@ +set fs.trash.interval=5; + +drop table IF EXISTS encrypted_table PURGE; + +CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table'; +CRYPTO CREATE_KEY --keyName key_128 --bitLength 128; +CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table; + +INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src; +select count(*) from encrypted_table; + +drop table encrypted_table; +drop table encrypted_table PURGE; +CRYPTO DELETE_KEY --keyName key_128; +set fs.trash.interval=0; diff --git ql/src/test/results/clientpositive/encrypted/encryption_trash.q.out ql/src/test/results/clientpositive/encrypted/encryption_trash.q.out new file mode 100644 index 0000000..bea61fa --- /dev/null +++ ql/src/test/results/clientpositive/encrypted/encryption_trash.q.out @@ -0,0 +1,48 @@ +PREHOOK: query: drop table IF EXISTS encrypted_table PURGE +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table IF EXISTS encrypted_table PURGE +POSTHOOK: type: DROPTABLE +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@encrypted_table +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@encrypted_table +Encryption key created: 'key_128' +Encryption zone created: '/build/ql/test/data/warehouse/default/encrypted_table' using key: 'key_128' +PREHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@encrypted_table +POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@encrypted_table +POSTHOOK: Lineage: encrypted_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: encrypted_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select count(*) from encrypted_table +PREHOOK: type: QUERY +PREHOOK: Input: default@encrypted_table +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging +POSTHOOK: query: select count(*) from encrypted_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@encrypted_table +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging +500 +PREHOOK: query: drop table encrypted_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@encrypted_table +PREHOOK: Output: default@encrypted_table +POSTHOOK: query: drop table encrypted_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@encrypted_table +POSTHOOK: Output: default@encrypted_table +PREHOOK: query: drop table encrypted_table PURGE +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table encrypted_table PURGE +POSTHOOK: type: DROPTABLE +Encryption key deleted: 'key_128'