diff --git data/scripts/q_test_cleanup_for_encryption.sql data/scripts/q_test_cleanup_for_encryption.sql index 08264ee..070cdbe 100644 --- data/scripts/q_test_cleanup_for_encryption.sql +++ data/scripts/q_test_cleanup_for_encryption.sql @@ -1 +1 @@ -DROP TABLE IF EXISTS src; +DROP TABLE IF EXISTS src PURGE; diff --git data/scripts/q_test_init_for_encryption.sql data/scripts/q_test_init_for_encryption.sql index 1822ebb..56b44e0 100644 --- data/scripts/q_test_init_for_encryption.sql +++ data/scripts/q_test_init_for_encryption.sql @@ -1,4 +1,4 @@ -DROP TABLE IF EXISTS src; +DROP TABLE IF EXISTS src PURGE; CREATE TABLE src(key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE; diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index a485408..b9d85f6 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -349,7 +349,8 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\ encryption_select_read_only_unencrypted_tbl.q,\ encryption_load_data_to_encrypted_tables.q, \ encryption_unencrypted_nonhdfs_external_tables.q \ - encryption_move_tbl.q + encryption_move_tbl.q \ + encryption_drop_table.q beeline.positive.exclude=add_part_exist.q,\ alter1.q,\ diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index f357c14..f60711e 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -372,6 +372,7 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, // Set the security key provider so that the MiniDFS cluster is initialized // with encryption conf.set(SECURITY_KEY_PROVIDER_URI_NAME, getKeyProviderURI()); + conf.setInt("fs.trash.interval", 50); dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null); fs = dfs.getFileSystem(); @@ -681,7 +682,7 @@ public void clearTablesCreatedDuringTests() throws Exception { if(tblObj.isIndexTable()) { continue; } - db.dropTable(dbName, tblName); + db.dropTable(dbName, tblName, true, true, clusterType == MiniClusterType.encrypted); } else { // this table is defined in srcTables, drop all indexes on it List indexes = db.getIndexes(dbName, tblName, (short)-1); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index e796048..5d8718c 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -55,6 +55,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.LogUtils; @@ -198,6 +199,7 @@ import org.apache.hadoop.hive.metastore.txn.TxnHandler; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge; @@ -1478,6 +1480,7 @@ private boolean drop_table_core(final RawStore ms, final String dbname, final St Path tblPath = null; List partPaths = null; Table tbl = null; + boolean ifPurge = false; try { ms.openTransaction(); // drop any partitions @@ -1489,6 +1492,17 @@ private boolean drop_table_core(final RawStore ms, final String dbname, final St throw new MetaException("Table metadata is corrupted"); } + /** + * Trash may be skipped iff: + * 1. deleteData == true, obviously. + * 2. tbl is external. + * 3. Either + * 3.1. User has specified PURGE from the commandline, and if not, + * 3.2. User has set the table to auto-purge. + */ + ifPurge = ((envContext != null) && Boolean.parseBoolean(envContext.getProperties().get("ifPurge"))) + || (tbl.isSetParameters() && "true".equalsIgnoreCase(tbl.getParameters().get("auto.purge"))); + firePreEvent(new PreDropTableEvent(tbl, deleteData, this)); boolean isIndexTable = isIndexTable(tbl); @@ -1521,6 +1535,19 @@ private boolean drop_table_core(final RawStore ms, final String dbname, final St } } + if(!ifPurge) { + String trashInterval = hiveConf.get("fs.trash.interval"); + boolean trashEnabled = trashInterval != null && trashInterval.length() > 0 + && Float.parseFloat(trashInterval) > 0; + if (trashEnabled) { + HadoopShims.HdfsEncryptionShim shim = + ShimLoader.getHadoopShims().createHdfsEncryptionShim(FileSystem.get(hiveConf), hiveConf); + if (shim.isPathEncrypted(tblPath)) { + throw new MetaException("Unable to drop table because it is in an encryption zone" + + " and trash is enabled. Use PURGE option to skip trash."); + } + } + } // Drop the partitions and get a list of locations which need to be deleted partPaths = dropPartitionsAndGetLocations(ms, dbname, name, tblPath, tbl.getPartitionKeys(), deleteData && !isExternal); @@ -1536,15 +1563,6 @@ private boolean drop_table_core(final RawStore ms, final String dbname, final St ms.rollbackTransaction(); } else if (deleteData && !isExternal) { // Data needs deletion. Check if trash may be skipped. - // Trash may be skipped iff: - // 1. deleteData == true, obviously. - // 2. tbl is external. - // 3. Either - // 3.1. User has specified PURGE from the commandline, and if not, - // 3.2. User has set the table to auto-purge. - boolean ifPurge = ((envContext != null) && Boolean.parseBoolean(envContext.getProperties().get("ifPurge"))) - || - (tbl.isSetParameters() && "true".equalsIgnoreCase(tbl.getParameters().get("auto.purge"))); // Delete the data in the partitions which have other locations deletePartitionData(partPaths, ifPurge); // Delete the data in the table diff --git ql/src/test/queries/clientpositive/encryption_drop_table.q ql/src/test/queries/clientpositive/encryption_drop_table.q new file mode 100644 index 0000000..193beea --- /dev/null +++ ql/src/test/queries/clientpositive/encryption_drop_table.q @@ -0,0 +1,18 @@ +-- SORT_QUERY_RESULTS; + +-- we're setting this so that TestNegaiveCliDriver.vm doesn't stop processing after DROP TABLE fails; + +set hive.cli.errors.ignore=true; + +DROP TABLE IF EXISTS encrypted_table PURGE; +CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table'; +CRYPTO CREATE_KEY --keyName key_128 --bitLength 128; +CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table; + +INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src; +SHOW TABLES; +DROP TABLE default.encrypted_table; +SHOW TABLES; +DROP TABLE default.encrypted_table PURGE; +SHOW TABLES; +CRYPTO DELETE_KEY --keyName key_128; \ No newline at end of file diff --git ql/src/test/queries/clientpositive/encryption_insert_partition_dynamic.q ql/src/test/queries/clientpositive/encryption_insert_partition_dynamic.q index 8674972..9556eed 100644 --- ql/src/test/queries/clientpositive/encryption_insert_partition_dynamic.q +++ ql/src/test/queries/clientpositive/encryption_insert_partition_dynamic.q @@ -6,8 +6,8 @@ set hive.enforce.bucketing=true; -- SORT_QUERY_RESULTS -- init -drop table IF EXISTS encryptedTable; -drop table IF EXISTS unencryptedTable; +drop table IF EXISTS encryptedTable PURGE; +drop table IF EXISTS unencryptedTable PURGE; create table encryptedTable(value string) partitioned by (key string) clustered by (value) into 2 buckets stored as orc @@ -52,6 +52,6 @@ insert into table unencryptedTable partition (key) select * from unencryptedTable order by key; -- clean up -drop table encryptedTable; +drop table encryptedTable PURGE; CRYPTO DELETE_KEY --keyName key_1; -drop table unencryptedTable; \ No newline at end of file +drop table unencryptedTable PURGE; \ No newline at end of file diff --git ql/src/test/queries/clientpositive/encryption_insert_partition_static.q ql/src/test/queries/clientpositive/encryption_insert_partition_static.q index 10fb965..9bff0bd 100644 --- ql/src/test/queries/clientpositive/encryption_insert_partition_static.q +++ ql/src/test/queries/clientpositive/encryption_insert_partition_static.q @@ -5,8 +5,8 @@ set hive.enforce.bucketing=true; -- SORT_QUERY_RESULTS -- init -drop table IF EXISTS encryptedTable; -drop table IF EXISTS unencryptedTable; +drop table IF EXISTS encryptedTable PURGE; +drop table IF EXISTS unencryptedTable PURGE; create table encryptedTable(key string, value string) partitioned by (ds string) clustered by (key) into 2 buckets stored as orc @@ -57,6 +57,6 @@ insert into table unencryptedTable partition select * from unencryptedTable order by key; -- clean up -drop table encryptedTable; +drop table encryptedTable PURGE; CRYPTO DELETE_KEY --keyName key_1; -drop table unencryptedTable; \ No newline at end of file +drop table unencryptedTable PURGE; \ No newline at end of file diff --git ql/src/test/queries/clientpositive/encryption_join_unencrypted_tbl.q ql/src/test/queries/clientpositive/encryption_join_unencrypted_tbl.q index adf82de..cf4d040 100644 --- ql/src/test/queries/clientpositive/encryption_join_unencrypted_tbl.q +++ ql/src/test/queries/clientpositive/encryption_join_unencrypted_tbl.q @@ -1,6 +1,6 @@ --SORT_QUERY_RESULTS -DROP TABLE IF EXISTS encrypted_table; +DROP TABLE IF EXISTS encrypted_table PURGE; CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table'; CRYPTO CREATE_KEY --keyName key_128 --bitLength 128; CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table; @@ -11,5 +11,5 @@ SELECT * FROM encrypted_table; EXPLAIN EXTENDED SELECT * FROM src t1 JOIN encrypted_table t2 WHERE t1.key = t2.key; -drop table encrypted_table; +drop table encrypted_table PURGE; CRYPTO DELETE_KEY --keyName key_128; \ No newline at end of file diff --git ql/src/test/queries/clientpositive/encryption_join_with_different_encryption_keys.q ql/src/test/queries/clientpositive/encryption_join_with_different_encryption_keys.q index c830247..dfd3406 100644 --- ql/src/test/queries/clientpositive/encryption_join_with_different_encryption_keys.q +++ ql/src/test/queries/clientpositive/encryption_join_with_different_encryption_keys.q @@ -1,12 +1,12 @@ --SORT_QUERY_RESULTS -- Java JCE must be installed in order to hava a key length of 256 bits -DROP TABLE IF EXISTS table_key_1; +DROP TABLE IF EXISTS table_key_1 PURGE; CREATE TABLE table_key_1 (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/table_key_1'; CRYPTO CREATE_KEY --keyName key_1 --bitLength 128; CRYPTO CREATE_ZONE --keyName key_1 --path ${hiveconf:hive.metastore.warehouse.dir}/table_key_1; -DROP TABLE IF EXISTS table_key_2; +DROP TABLE IF EXISTS table_key_2 PURGE; CREATE TABLE table_key_2 (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/table_key_2'; CRYPTO CREATE_KEY --keyName key_2 --bitLength 256; CRYPTO CREATE_ZONE --keyName key_2 --path ${hiveconf:hive.metastore.warehouse.dir}/table_key_2; @@ -17,8 +17,8 @@ INSERT OVERWRITE TABLE table_key_2 SELECT * FROM src; EXPLAIN EXTENDED SELECT * FROM table_key_1 t1 JOIN table_key_2 t2 WHERE (t1.key = t2.key); SELECT * FROM table_key_1 t1 JOIN table_key_2 t2 WHERE (t1.key = t2.key); -DROP TABLE table_key_1; -DROP TABLE table_key_2; +DROP TABLE table_key_1 PURGE; +DROP TABLE table_key_2 PURGE; CRYPTO DELETE_KEY --keyName key_1; CRYPTO DELETE_KEY --keyName key_2; \ No newline at end of file diff --git ql/src/test/queries/clientpositive/encryption_load_data_to_encrypted_tables.q ql/src/test/queries/clientpositive/encryption_load_data_to_encrypted_tables.q index e4d3a4c..a96807e 100644 --- ql/src/test/queries/clientpositive/encryption_load_data_to_encrypted_tables.q +++ ql/src/test/queries/clientpositive/encryption_load_data_to_encrypted_tables.q @@ -1,4 +1,4 @@ -DROP TABLE IF EXISTS encrypted_table; +DROP TABLE IF EXISTS encrypted_table PURGE; CREATE TABLE encrypted_table (key STRING, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/encrypted_table'; @@ -15,6 +15,6 @@ dfs -copyFromLocal ../../data/files/kv1.txt hdfs:///tmp/kv1.txt; LOAD DATA INPATH '/tmp/kv1.txt' OVERWRITE INTO TABLE encrypted_table; SELECT * FROM encrypted_table; -DROP TABLE encrypted_table; +DROP TABLE encrypted_table PURGE; crypto delete_key --keyName key1; \ No newline at end of file diff --git ql/src/test/queries/clientpositive/encryption_move_tbl.q ql/src/test/queries/clientpositive/encryption_move_tbl.q index 1168c6c..5a8c036 100644 --- ql/src/test/queries/clientpositive/encryption_move_tbl.q +++ ql/src/test/queries/clientpositive/encryption_move_tbl.q @@ -4,7 +4,7 @@ set hive.cli.errors.ignore=true; -DROP TABLE IF EXISTS encrypted_table; +DROP TABLE IF EXISTS encrypted_table PURGE; CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table'; CRYPTO CREATE_KEY --keyName key_128 --bitLength 128; CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table; @@ -14,5 +14,7 @@ SHOW TABLES; ALTER TABLE default.encrypted_table RENAME TO default.plain_table; SHOW TABLES; +DROP TABLE encrypted_table PURGE; + CRYPTO DELETE_KEY --keyName key_128; diff --git ql/src/test/queries/clientpositive/encryption_select_read_only_encrypted_tbl.q ql/src/test/queries/clientpositive/encryption_select_read_only_encrypted_tbl.q index c013425..c9ac035 100644 --- ql/src/test/queries/clientpositive/encryption_select_read_only_encrypted_tbl.q +++ ql/src/test/queries/clientpositive/encryption_select_read_only_encrypted_tbl.q @@ -1,6 +1,6 @@ -- SORT_QUERY_RESULTS -DROP TABLE IF EXISTS encrypted_table; +DROP TABLE IF EXISTS encrypted_table PURGE; CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table'; CRYPTO CREATE_KEY --keyName key_128 --bitLength 128; @@ -12,5 +12,5 @@ dfs -chmod -R 555 ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_tab SELECT count(*) FROM encrypted_table; -drop table encrypted_table; +drop table encrypted_table PURGE; CRYPTO DELETE_KEY --keyName key_128; diff --git ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out new file mode 100644 index 0000000..8f51fb9 --- /dev/null +++ ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out @@ -0,0 +1,56 @@ +PREHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE +POSTHOOK: type: DROPTABLE +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@encrypted_table +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@encrypted_table +Encryption key created: 'key_128' +Encryption zone created: '/build/ql/test/data/warehouse/default/encrypted_table' using key: 'key_128' +PREHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@encrypted_table +POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@encrypted_table +POSTHOOK: Lineage: encrypted_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: encrypted_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SHOW TABLES +PREHOOK: type: SHOWTABLES +POSTHOOK: query: SHOW TABLES +POSTHOOK: type: SHOWTABLES +encrypted_table +src +PREHOOK: query: DROP TABLE default.encrypted_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@encrypted_table +PREHOOK: Output: default@encrypted_table +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Unable to drop table because it is in an encryption zone and trash is enabled. Use PURGE option to skip trash.) +PREHOOK: query: SHOW TABLES +PREHOOK: type: SHOWTABLES +POSTHOOK: query: SHOW TABLES +POSTHOOK: type: SHOWTABLES +encrypted_table +src +PREHOOK: query: DROP TABLE default.encrypted_table PURGE +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@encrypted_table +PREHOOK: Output: default@encrypted_table +POSTHOOK: query: DROP TABLE default.encrypted_table PURGE +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@encrypted_table +POSTHOOK: Output: default@encrypted_table +PREHOOK: query: SHOW TABLES +PREHOOK: type: SHOWTABLES +POSTHOOK: query: SHOW TABLES +POSTHOOK: type: SHOWTABLES +src diff --git ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_dynamic.q.out ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_dynamic.q.out index 22cbce9..cb6dc5c 100644 --- ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_dynamic.q.out +++ ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_dynamic.q.out @@ -1,16 +1,16 @@ PREHOOK: query: -- SORT_QUERY_RESULTS -- init -drop table IF EXISTS encryptedTable +drop table IF EXISTS encryptedTable PURGE PREHOOK: type: DROPTABLE POSTHOOK: query: -- SORT_QUERY_RESULTS -- init -drop table IF EXISTS encryptedTable +drop table IF EXISTS encryptedTable PURGE POSTHOOK: type: DROPTABLE -PREHOOK: query: drop table IF EXISTS unencryptedTable +PREHOOK: query: drop table IF EXISTS unencryptedTable PURGE PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table IF EXISTS unencryptedTable +POSTHOOK: query: drop table IF EXISTS unencryptedTable PURGE POSTHOOK: type: DROPTABLE PREHOOK: query: create table encryptedTable(value string) partitioned by (key string) clustered by (value) into 2 buckets stored as orc @@ -882,21 +882,21 @@ val_501 501 val_502 502 val_86 86 PREHOOK: query: -- clean up -drop table encryptedTable +drop table encryptedTable PURGE PREHOOK: type: DROPTABLE PREHOOK: Input: default@encryptedtable PREHOOK: Output: default@encryptedtable POSTHOOK: query: -- clean up -drop table encryptedTable +drop table encryptedTable PURGE POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@encryptedtable POSTHOOK: Output: default@encryptedtable Encryption key deleted: 'key_1' -PREHOOK: query: drop table unencryptedTable +PREHOOK: query: drop table unencryptedTable PURGE PREHOOK: type: DROPTABLE PREHOOK: Input: default@unencryptedtable PREHOOK: Output: default@unencryptedtable -POSTHOOK: query: drop table unencryptedTable +POSTHOOK: query: drop table unencryptedTable PURGE POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@unencryptedtable POSTHOOK: Output: default@unencryptedtable diff --git ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out index 96f8b6a..8966608 100644 --- ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out +++ ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out @@ -1,16 +1,16 @@ PREHOOK: query: -- SORT_QUERY_RESULTS -- init -drop table IF EXISTS encryptedTable +drop table IF EXISTS encryptedTable PURGE PREHOOK: type: DROPTABLE POSTHOOK: query: -- SORT_QUERY_RESULTS -- init -drop table IF EXISTS encryptedTable +drop table IF EXISTS encryptedTable PURGE POSTHOOK: type: DROPTABLE -PREHOOK: query: drop table IF EXISTS unencryptedTable +PREHOOK: query: drop table IF EXISTS unencryptedTable PURGE PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table IF EXISTS unencryptedTable +POSTHOOK: query: drop table IF EXISTS unencryptedTable PURGE POSTHOOK: type: DROPTABLE PREHOOK: query: create table encryptedTable(key string, value string) partitioned by (ds string) clustered by (key) into 2 buckets stored as orc @@ -555,16 +555,16 @@ STAGE PLANS: Map Operator Tree: TableScan alias: encryptedtable - Statistics: Num rows: 12 Data size: 2767 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 2777 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 2767 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 2777 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 2767 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 2777 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col0 (type: string), _col1 (type: string) auto parallelism: false @@ -595,7 +595,7 @@ STAGE PLANS: serialization.ddl struct encryptedtable { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 1385 + totalSize 1392 #### A masked pattern was here #### serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde @@ -643,7 +643,7 @@ STAGE PLANS: serialization.ddl struct encryptedtable { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 1382 + totalSize 1385 #### A masked pattern was here #### serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde @@ -675,14 +675,14 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 2767 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 2777 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/ds=today/.hive-staging NumFilesPerFileSink: 1 Static Partition Specification: ds=today/ - Statistics: Num rows: 12 Data size: 2767 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 2777 Basic stats: COMPLETE Column stats: NONE #### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/ds=today/.hive-staging table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat @@ -775,21 +775,21 @@ POSTHOOK: Input: default@unencryptedtable@ds=today 502 val_502 today 86 val_86 today PREHOOK: query: -- clean up -drop table encryptedTable +drop table encryptedTable PURGE PREHOOK: type: DROPTABLE PREHOOK: Input: default@encryptedtable PREHOOK: Output: default@encryptedtable POSTHOOK: query: -- clean up -drop table encryptedTable +drop table encryptedTable PURGE POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@encryptedtable POSTHOOK: Output: default@encryptedtable Encryption key deleted: 'key_1' -PREHOOK: query: drop table unencryptedTable +PREHOOK: query: drop table unencryptedTable PURGE PREHOOK: type: DROPTABLE PREHOOK: Input: default@unencryptedtable PREHOOK: Output: default@unencryptedtable -POSTHOOK: query: drop table unencryptedTable +POSTHOOK: query: drop table unencryptedTable PURGE POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@unencryptedtable POSTHOOK: Output: default@unencryptedtable diff --git ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out index 0caf694..9b6f750 100644 --- ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out +++ ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out @@ -1,10 +1,10 @@ PREHOOK: query: --SORT_QUERY_RESULTS -DROP TABLE IF EXISTS encrypted_table +DROP TABLE IF EXISTS encrypted_table PURGE PREHOOK: type: DROPTABLE POSTHOOK: query: --SORT_QUERY_RESULTS -DROP TABLE IF EXISTS encrypted_table +DROP TABLE IF EXISTS encrypted_table PURGE POSTHOOK: type: DROPTABLE #### A masked pattern was here #### PREHOOK: type: CREATETABLE @@ -752,11 +752,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: drop table encrypted_table +PREHOOK: query: drop table encrypted_table PURGE PREHOOK: type: DROPTABLE PREHOOK: Input: default@encrypted_table PREHOOK: Output: default@encrypted_table -POSTHOOK: query: drop table encrypted_table +POSTHOOK: query: drop table encrypted_table PURGE POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@encrypted_table POSTHOOK: Output: default@encrypted_table diff --git ql/src/test/results/clientpositive/encrypted/encryption_join_with_different_encryption_keys.q.out ql/src/test/results/clientpositive/encrypted/encryption_join_with_different_encryption_keys.q.out index 7d8ffce..40f6898 100644 --- ql/src/test/results/clientpositive/encrypted/encryption_join_with_different_encryption_keys.q.out +++ ql/src/test/results/clientpositive/encrypted/encryption_join_with_different_encryption_keys.q.out @@ -1,12 +1,12 @@ PREHOOK: query: --SORT_QUERY_RESULTS -- Java JCE must be installed in order to hava a key length of 256 bits -DROP TABLE IF EXISTS table_key_1 +DROP TABLE IF EXISTS table_key_1 PURGE PREHOOK: type: DROPTABLE POSTHOOK: query: --SORT_QUERY_RESULTS -- Java JCE must be installed in order to hava a key length of 256 bits -DROP TABLE IF EXISTS table_key_1 +DROP TABLE IF EXISTS table_key_1 PURGE POSTHOOK: type: DROPTABLE #### A masked pattern was here #### PREHOOK: type: CREATETABLE @@ -20,9 +20,9 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@table_key_1 Encryption key created: 'key_1' Encryption zone created: '/build/ql/test/data/warehouse/table_key_1' using key: 'key_1' -PREHOOK: query: DROP TABLE IF EXISTS table_key_2 +PREHOOK: query: DROP TABLE IF EXISTS table_key_2 PURGE PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE IF EXISTS table_key_2 +POSTHOOK: query: DROP TABLE IF EXISTS table_key_2 PURGE POSTHOOK: type: DROPTABLE #### A masked pattern was here #### PREHOOK: type: CREATETABLE @@ -1310,19 +1310,19 @@ POSTHOOK: Input: default@table_key_2 498 val_498 498 val_498 498 val_498 498 val_498 498 val_498 498 val_498 -PREHOOK: query: DROP TABLE table_key_1 +PREHOOK: query: DROP TABLE table_key_1 PURGE PREHOOK: type: DROPTABLE PREHOOK: Input: default@table_key_1 PREHOOK: Output: default@table_key_1 -POSTHOOK: query: DROP TABLE table_key_1 +POSTHOOK: query: DROP TABLE table_key_1 PURGE POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@table_key_1 POSTHOOK: Output: default@table_key_1 -PREHOOK: query: DROP TABLE table_key_2 +PREHOOK: query: DROP TABLE table_key_2 PURGE PREHOOK: type: DROPTABLE PREHOOK: Input: default@table_key_2 PREHOOK: Output: default@table_key_2 -POSTHOOK: query: DROP TABLE table_key_2 +POSTHOOK: query: DROP TABLE table_key_2 PURGE POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@table_key_2 POSTHOOK: Output: default@table_key_2 diff --git ql/src/test/results/clientpositive/encrypted/encryption_load_data_to_encrypted_tables.q.out ql/src/test/results/clientpositive/encrypted/encryption_load_data_to_encrypted_tables.q.out index 46f36d8..35de14f 100644 --- ql/src/test/results/clientpositive/encrypted/encryption_load_data_to_encrypted_tables.q.out +++ ql/src/test/results/clientpositive/encrypted/encryption_load_data_to_encrypted_tables.q.out @@ -1,6 +1,6 @@ -PREHOOK: query: DROP TABLE IF EXISTS encrypted_table +PREHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table +POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE POSTHOOK: type: DROPTABLE #### A masked pattern was here #### PREHOOK: type: CREATETABLE @@ -1048,11 +1048,11 @@ POSTHOOK: Input: default@encrypted_table 400 val_400 200 val_200 97 val_97 -PREHOOK: query: DROP TABLE encrypted_table +PREHOOK: query: DROP TABLE encrypted_table PURGE PREHOOK: type: DROPTABLE PREHOOK: Input: default@encrypted_table PREHOOK: Output: default@encrypted_table -POSTHOOK: query: DROP TABLE encrypted_table +POSTHOOK: query: DROP TABLE encrypted_table PURGE POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@encrypted_table POSTHOOK: Output: default@encrypted_table diff --git ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out index d18827e..e0098f3 100644 --- ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out +++ ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out @@ -1,6 +1,6 @@ -PREHOOK: query: DROP TABLE IF EXISTS encrypted_table +PREHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table +POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE POSTHOOK: type: DROPTABLE #### A masked pattern was here #### PREHOOK: type: CREATETABLE @@ -41,3 +41,11 @@ POSTHOOK: query: SHOW TABLES POSTHOOK: type: SHOWTABLES encrypted_table src +PREHOOK: query: DROP TABLE encrypted_table PURGE +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@encrypted_table +PREHOOK: Output: default@encrypted_table +POSTHOOK: query: DROP TABLE encrypted_table PURGE +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@encrypted_table +POSTHOOK: Output: default@encrypted_table diff --git ql/src/test/results/clientpositive/encrypted/encryption_select_read_only_encrypted_tbl.q.out ql/src/test/results/clientpositive/encrypted/encryption_select_read_only_encrypted_tbl.q.out index c4c0dde..5d4d2af 100644 --- ql/src/test/results/clientpositive/encrypted/encryption_select_read_only_encrypted_tbl.q.out +++ ql/src/test/results/clientpositive/encrypted/encryption_select_read_only_encrypted_tbl.q.out @@ -1,10 +1,10 @@ PREHOOK: query: -- SORT_QUERY_RESULTS -DROP TABLE IF EXISTS encrypted_table +DROP TABLE IF EXISTS encrypted_table PURGE PREHOOK: type: DROPTABLE POSTHOOK: query: -- SORT_QUERY_RESULTS -DROP TABLE IF EXISTS encrypted_table +DROP TABLE IF EXISTS encrypted_table PURGE POSTHOOK: type: DROPTABLE #### A masked pattern was here #### PREHOOK: type: CREATETABLE @@ -35,11 +35,11 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@encrypted_table #### A masked pattern was here #### 500 -PREHOOK: query: drop table encrypted_table +PREHOOK: query: drop table encrypted_table PURGE PREHOOK: type: DROPTABLE PREHOOK: Input: default@encrypted_table PREHOOK: Output: default@encrypted_table -POSTHOOK: query: drop table encrypted_table +POSTHOOK: query: drop table encrypted_table PURGE POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@encrypted_table POSTHOOK: Output: default@encrypted_table diff --git shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index d349068..19324b8 100644 --- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -1215,6 +1215,9 @@ public boolean isPathEncrypted(Path path) throws IOException { } else { fullPath = path.getFileSystem(conf).makeQualified(path); } + if(!"hdfs".equalsIgnoreCase(path.toUri().getScheme())) { + return false; + } return (hdfsAdmin.getEncryptionZoneForPath(fullPath) != null); }