commit e5564dcd6b3b3815622e387c2d44edec67454bf3 Author: Vihang Karajgaonkar Date: Tue Feb 21 16:14:12 2017 -0800 HIVE-15880 : Allow insert overwrite query to use auto.purge table property diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java index 9a0521c27594663c0247bf72c5b80da75d0124f4..e586015e2cd55c194d7afe18eb1355b147815934 100644 --- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java @@ -616,15 +616,19 @@ static boolean copy(FileSystem srcFS, Path src, * @return true if move successful * @throws IOException */ - public static boolean moveToTrash(FileSystem fs, Path f, Configuration conf) + public static boolean moveToTrash(FileSystem fs, Path f, Configuration conf, boolean purge) throws IOException { LOG.debug("deleting " + f); boolean result = false; try { - result = Trash.moveToAppropriateTrash(fs, f, conf); - if (result) { - LOG.trace("Moved to trash: " + f); - return true; + if(purge) { + LOG.debug("purge is set to true. Not moving to Trash " + f); + } else { + result = Trash.moveToAppropriateTrash(fs, f, conf); + if (result) { + LOG.trace("Moved to trash: " + f); + return true; + } } } catch (IOException ioe) { // for whatever failure reason including that trash has lower encryption zone @@ -636,7 +640,6 @@ public static boolean moveToTrash(FileSystem fs, Path f, Configuration conf) if (!result) { LOG.error("Failed to delete " + f); } - return result; } diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java new file mode 100644 index 0000000000000000000000000000000000000000..abf97694e6ceb6aa2c5fbd292c29525860aa6ef7 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java @@ -0,0 +1,436 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestAutoPurgeTables { + private static final String driverName = "org.apache.hive.jdbc.HiveDriver"; + private static final String testDbName = "auto_purge_test_db"; + //private static final String testTableName = "auto_purge_test_table"; + private static final String INSERT_OVERWRITE_COMMAND_FORMAT = + "insert overwrite table " + testDbName + ".%s select 1, \"test\""; + private static final String TRUNCATE_TABLE_COMMAND_FORMAT = + "truncate table " + testDbName + ".%s"; + private static final String partitionedColumnName = "partCol"; + private static final String partitionedColumnValue1 = "20090619"; + private static final String INSERT_OVERWRITE_COMMAND_PARTITIONED_FORMAT = + "insert overwrite table " + testDbName + ".%s PARTITION (" + + partitionedColumnName + "=" + partitionedColumnValue1 + ")" + " select 1, \"test\""; + private static final String partitionedColumnValue2 = "20100720"; + private static HiveConf conf; + private static Connection con; + private static MiniHS2 miniHS2; + static final private Logger LOG = LoggerFactory.getLogger("TestAutoPurgeTables"); + + @Rule + public TestName name = new TestName(); + + private static Connection getConnection(String url) throws SQLException { + Connection con1; + con1 = DriverManager.getConnection(url, "", ""); + Assert.assertNotNull("Connection is null", con1); + Assert.assertFalse("Connection should not be closed", con1.isClosed()); + return con1; + } + + private static void createTestTable(Statement stmt, String isAutopurge, boolean isExternal, + boolean isPartitioned, String testTableName) throws SQLException { + String createTablePrefix; + if (isExternal) { + createTablePrefix = "create external table "; + } else { + createTablePrefix = "create table "; + } + if (isPartitioned) { + // create a partitioned table + stmt.execute(createTablePrefix + testDbName + "." + testTableName + " (id int, value string) " + + " partitioned by (" + partitionedColumnName + " STRING)"); + // load data + stmt.execute("insert into " + testDbName + "." + testTableName + " PARTITION (" + + partitionedColumnName + "=" + partitionedColumnValue1 + + ") values (1, \"dummy1\"), (2, \"dummy2\"), (3, \"dummy3\")"); + stmt.execute("insert into " + testDbName + "." + testTableName + " PARTITION (" + + partitionedColumnName + "=" + partitionedColumnValue2 + + ") values (4, \"dummy4\"), (5, \"dummy5\"), (6, \"dummy6\")"); + } else { + // create a table + stmt.execute(createTablePrefix + testDbName + "." + testTableName + " (id int, value string)"); + // load data + stmt.execute("insert into " + testDbName + "." + testTableName + + " values (1, \"dummy1\"), (2, \"dummy2\"), (3, \"dummy3\")"); + } + if (isAutopurge != null) { + stmt.execute("alter table " + testDbName + "." + testTableName + + " set tblproperties (\"auto.purge\"=\"" + isAutopurge + "\")"); + } + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + conf = new HiveConf(TestAutoPurgeTables.class); + // enable trash so it can be tested + conf.setFloat("fs.trash.checkpoint.interval", 30); + conf.setFloat("fs.trash.interval", 30); + // Create test database and base tables once for all the test + miniHS2 = new MiniHS2.Builder().withConf(conf).build(); + miniHS2.start(new HashMap()); + Class.forName(driverName); + con = getConnection(miniHS2.getBaseJdbcURL() + ";create=true"); + try (Statement stmt = con.createStatement()) { + Assert.assertNotNull("Statement is null", stmt); + stmt.execute("set hive.support.concurrency = false"); + stmt.execute("drop database if exists " + testDbName + " cascade"); + stmt.execute("create database " + testDbName); + } + } + + @AfterClass + public static void tearDownAfterClass() { + Statement stmt = null; + try { + stmt = con.createStatement(); + // drop test db and its tables and views + stmt.execute("set hive.support.concurrency = false"); + stmt.execute("drop database if exists " + testDbName + " cascade"); + FileSystem fs = FileSystem.get(conf); + fs.deleteOnExit(ShimLoader.getHadoopShims().getCurrentTrashPath(conf, fs)); + } catch (SQLException | IOException e) { + e.printStackTrace(); + } finally { + if (stmt != null) { + try { + stmt.close(); + } catch (SQLException e) { + // + } + } + if (con != null) { + try { + con.close(); + } catch (SQLException e) { + // + } + } + if (miniHS2 != null) { + miniHS2.cleanup(); + miniHS2.stop(); + miniHS2 = null; + } + } + } + + @Before + public void afterTest() throws Exception { + FileSystem fs = FileSystem.get(conf); + Path trashDir = ShimLoader.getHadoopShims().getCurrentTrashPath(conf, fs); + fs.delete(trashDir, true); + } + + /** + * Tests if previous table data skips trash when insert overwrite table .. is run against a table + * which has auto.purge property set + * + * @throws Exception + */ + @Test + public void testAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil("true", false, false, false, name.getMethodName()); + } + + /** + * Tests when auto.purge is set to a invalid string, trash should be used for insert overwrite + * queries + * + * @throws Exception + */ + @Test + public void testAutoPurgeInvalid() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil("invalid", false, false, false, name.getMethodName()); + } + + /** + * Test when auto.purge property is not set. Data should be moved to trash for insert overwrite + * queries + * + * @throws Exception + */ + @Test + public void testAutoPurgeUnset() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil(null, false, false, false, name.getMethodName()); + } + + /** + * Tests if the auto.purge property works correctly for external tables. Old data should skip + * trash when insert overwrite table .. is run when auto.purge is set to true + * + * @throws Exception + */ + @Test + public void testExternalTable() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil("true", true, false, false, name.getMethodName()); + } + + /** + * Tests auto.purge when managed table is partitioned. Old data should skip trash when insert + * overwrite table .. is run and auto.purge property is set to true + * + * @throws Exception + */ + @Test + public void testPartitionedTable() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil("true", false, true, false, name.getMethodName()); + } + + /** + * Tests auto.purge for an external, partitioned table. Old partition data should skip trash when + * auto.purge is set to true + * + * @throws Exception + */ + @Test + public void testExternalPartitionedTable() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil("true", true, true, false, name.getMethodName()); + } + + /** + * Tests when auto.purge is set to false, older data is moved to Trash when insert overwrite table + * .. is run + * + * @throws Exception + */ + @Test + public void testNoAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil("false", false, false, false, name.getMethodName()); + } + + /** + * Tests when auto.purge is set to false on a external table, older data is moved to Trash when + * insert overwrite table .. is run + * + * @throws Exception + */ + @Test + public void testExternalNoAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil("false", true, false, false, name.getMethodName()); + } + + /** + * Tests when auto.purge is set to false on a partitioned table, older data is moved to Trash when + * insert overwrite table .. is run + * + * @throws Exception + */ + @Test + public void testPartitionedNoAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil("false", false, true, false, name.getMethodName()); + } + + /** + * Tests when auto.purge is set to false on a partitioned external table, older data is moved to + * Trash when insert overwrite table .. is run + * + * @throws Exception + */ + @Test + public void testPartitionedExternalNoAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil("false", true, true, false, name.getMethodName()); + } + + //truncate on external table is not allowed + @Test(expected = SQLException.class) + public void testTruncatePartitionedExternalNoAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil(String.valueOf(false), true, true, true, name.getMethodName()); + } + + //truncate on external table is not allowed + @Test(expected = SQLException.class) + public void testTruncateExternalNoAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil(String.valueOf(false), true, false, true, name.getMethodName()); + } + + @Test + public void testTruncatePartitionedNoAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil(String.valueOf(false), false, true, true, name.getMethodName()); + } + + @Test + public void testTruncateNoAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil(String.valueOf(false), false, false, true, name.getMethodName()); + } + + @Test + public void testTruncateInvalidAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil("invalid", false, false, true, name.getMethodName()); + } + + @Test + public void testTruncateUnsetAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil(null, false, false, true, name.getMethodName()); + } + + //truncate on external table is not allowed + @Test(expected = SQLException.class) + public void testTruncatePartitionedExternalAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil(String.valueOf(true), true, true, true, name.getMethodName()); + } + + //truncate on external table is not allowed + @Test(expected = SQLException.class) + public void testTruncateExternalAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil(String.valueOf(true), true, false, true, name.getMethodName()); + } + + @Test + public void testTruncatePartitionedAutoPurge() throws Exception { + LOG.info("Running " + name.getMethodName()); + testUtil(String.valueOf(true), false, true, true, name.getMethodName()); + } + + /** + * Test util method to run the insert overwrite table or truncate table test on a table + * + * @param autoPurgePropValue - string value of the auto.purge property for the test table. Ignored + * if null + * @param isExternal - if set creates a external table for the test + * @param isPartitioned - if set creates a partitioned table for the test + * @param isTruncateTest - if set uses truncate table command for the test. Otherwise uses Insert + * overwrite table command for the test + * @param testTableName - test table name + * @throws Exception + */ + private void testUtil(String autoPurgePropValue, boolean isExternal, boolean isPartitioned, + boolean isTruncateTest, String testTableName) throws Exception { + testUtil(autoPurgePropValue, isExternal, isPartitioned, + !"true".equalsIgnoreCase(autoPurgePropValue), isTruncateTest, testTableName); + } + /** + * Test util method to run the insert overwrite table or truncate table test on a table + * + * @param isAutoPurge - If set, creates a table with auto.purge with the given value + * @param isExternal - if set creates a external table for the test + * @param isPartitioned - if set creates a partitioned table for the test + * @param purgeExpected - if set the assert condition for the test is such that it expectes old + * table data to be moved to trash. If not creates a assert condition to make sure that + * data is not moved to trash + * @param isTruncateTest - if set uses truncate table command for the test. Otherwise uses Insert + * overwrite table command for the test + * @param testTableName - table name for the test table + * @throws Exception + */ + private void testUtil(String isAutoPurge, boolean isExternal, boolean isPartitioned, + boolean purgeExpected, boolean isTruncateTest, String testTableName) throws Exception { + try (Statement stmt = con.createStatement()) { + // create a test table with auto.purge = true + createTestTable(stmt, isAutoPurge, isExternal, isPartitioned, testTableName); + int numFilesInTrashBefore = getTrashFileCount(); + String command = getCommand(isTruncateTest, isPartitioned, testTableName); + stmt.execute(command); + int numFilesInTrashAfter = getTrashFileCount(); + if (purgeExpected) { + Assert.assertTrue( + String.format( + "Data should have been moved to trash. Number of files in trash: before : %d after %d", + numFilesInTrashBefore, numFilesInTrashAfter), + numFilesInTrashBefore < numFilesInTrashAfter); + } else { + Assert.assertEquals( + String.format( + "Data should not have been moved to trash. Number of files in trash: before : %d after %d", + numFilesInTrashBefore, numFilesInTrashAfter), + numFilesInTrashBefore, numFilesInTrashAfter); + } + } + } + + private static String getCommand(boolean isTruncateTest, boolean isPartitioned, String testTableName) { + if (isTruncateTest) { + return String.format(TRUNCATE_TABLE_COMMAND_FORMAT, testTableName); + } else if (isPartitioned) { + return String.format(INSERT_OVERWRITE_COMMAND_PARTITIONED_FORMAT, testTableName); + } else { + return String.format(INSERT_OVERWRITE_COMMAND_FORMAT, testTableName); + } + } + + private int getTrashFileCount() throws Exception { + FileSystem fs = FileSystem.get(conf); + Path trashDir = ShimLoader.getHadoopShims().getCurrentTrashPath(conf, fs); + return getFileCount(fs, trashDir); + } + + private int getFileCount(FileSystem fs, Path path) throws Exception { + try { + int count = 0; + if (!fs.exists(path)) { + return count; + } + RemoteIterator lfs = fs.listFiles(path, true); + while (lfs.hasNext()) { + LocatedFileStatus lf = lfs.next(); + LOG.info(lf.getPath().toString()); + if (lf.isFile()) { + count++; + } + } + return count; + } catch (IOException e) { + throw new Exception("Exception while list files on " + path, e); + } + } +} diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 3c98a57ee378dafe2ec03cf86a1c890bc61ac748..7a70c9c829b61da104bb8caa755bb1e7da471f79 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -741,7 +741,8 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\ encryption_drop_view.q \ encryption_drop_partition.q \ encryption_with_trash.q \ - encryption_ctas.q + encryption_ctas.q \ + encryption_auto_purge_tables.q beeline.positive.include=drop_with_concurrency.q,\ escape_comments.q diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java index df698c878d16f62a5cfdab62400c93fa286cf63d..b7d7b50bb9b99ee941e4211c174cc50a037ad327 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java @@ -25,35 +25,23 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.Trash; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.metastore.api.MetaException; public class HiveMetaStoreFsImpl implements MetaStoreFS { public static final Logger LOG = LoggerFactory - .getLogger("hive.metastore.hivemetastoressimpl"); + .getLogger("hive.metastore.hivemetastoreFsimpl"); @Override public boolean deleteDir(FileSystem fs, Path f, boolean recursive, boolean ifPurge, Configuration conf) throws MetaException { - LOG.debug("deleting " + f); - try { - if (ifPurge) { - LOG.info("Not moving "+ f +" to trash"); - } else if (Trash.moveToAppropriateTrash(fs, f, conf)) { - LOG.info("Moved to trash: " + f); - return true; - } - - if (fs.delete(f, true)) { - LOG.debug("Deleted the diretory " + f); - return true; - } - + FileUtils.moveToTrash(fs, f, conf, ifPurge); if (fs.exists(f)) { throw new MetaException("Unable to delete directory: " + f); } + return true; } catch (FileNotFoundException e) { return true; // ok even if there is not data } catch (Exception e) { @@ -61,5 +49,4 @@ public boolean deleteDir(FileSystem fs, Path f, boolean recursive, } return false; } - } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index f13781943da240bcc81873daf4682ffee7415a5a..bb85896d5dd341c36a7594233b99f5d66069ec67 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -4652,7 +4652,7 @@ private int truncateTable(Hive db, TruncateTableDesc truncateTableDesc) throws H Map partSpec = truncateTableDesc.getPartSpec(); Table table = db.getTable(tableName, true); - + boolean isAutopurge = "true".equalsIgnoreCase(table.getProperty("auto.purge")); try { // this is not transactional for (Path location : getLocations(db, table, partSpec)) { @@ -4663,7 +4663,7 @@ private int truncateTable(Hive db, TruncateTableDesc truncateTableDesc) throws H HdfsUtils.HadoopFileStatus status = new HdfsUtils.HadoopFileStatus(conf, fs, location); FileStatus targetStatus = fs.getFileStatus(location); String targetGroup = targetStatus == null ? null : targetStatus.getGroup(); - FileUtils.moveToTrash(fs, location, conf); + FileUtils.moveToTrash(fs, location, conf, isAutopurge); fs.mkdirs(location); HdfsUtils.setFullFileStatus(conf, status, targetGroup, fs, location, false); } else { @@ -4671,7 +4671,7 @@ private int truncateTable(Hive db, TruncateTableDesc truncateTableDesc) throws H if (statuses == null || statuses.length == 0) { continue; } - boolean success = Hive.trashFiles(fs, statuses, conf); + boolean success = Hive.trashFiles(fs, statuses, conf, isAutopurge); if (!success) { throw new HiveException("Error in deleting the contents of " + location.toString()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index ed854bf48b7507857e28c155a90963204a05c913..f64cfda6da2b82ec277889378ea111e17f044ed3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1646,8 +1646,9 @@ public Partition loadPartition(Path loadPath, Table tbl, PerfLogger perfLogger = SessionState.getPerfLogger(); perfLogger.PerfLogBegin("MoveTask", "FileMoves"); if (replace || (oldPart == null && !isAcid)) { + boolean isAutoPurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge")); replaceFiles(tbl.getPath(), loadPath, newPartPath, oldPartPath, getConf(), - isSrcLocal); + isSrcLocal, isAutoPurge); } else { if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary() && oldPart != null) { newFiles = Collections.synchronizedList(new ArrayList()); @@ -2012,7 +2013,8 @@ public void loadTable(Path loadPath, String tableName, boolean replace, boolean } if (replace) { Path tableDest = tbl.getPath(); - replaceFiles(tableDest, loadPath, tableDest, tableDest, sessionConf, isSrcLocal); + boolean isAutopurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge")); + replaceFiles(tableDest, loadPath, tableDest, tableDest, sessionConf, isSrcLocal, isAutopurge); } else { FileSystem fs; try { @@ -3398,11 +3400,13 @@ private static void moveAcidDeltaFiles(String deltaFileType, PathFilter pathFilt * @param oldPath * The directory where the old data location, need to be cleaned up. Most of time, will be the same * as destf, unless its across FileSystem boundaries. + * @param purge + * When set to true files which needs to be deleted are not moved to Trash * @param isSrcLocal * If the source directory is LOCAL */ protected void replaceFiles(Path tablePath, Path srcf, Path destf, Path oldPath, HiveConf conf, - boolean isSrcLocal) throws HiveException { + boolean isSrcLocal, boolean purge) throws HiveException { try { FileSystem destFs = destf.getFileSystem(conf); @@ -3435,7 +3439,7 @@ protected void replaceFiles(Path tablePath, Path srcf, Path destf, Path oldPath, // existing content might result in incorrect (extra) data. // But not sure why we changed not to delete the oldPath in HIVE-8750 if it is // not the destf or its subdir? - oldPathDeleted = trashFiles(oldFs, statuses, conf); + oldPathDeleted = trashFiles(oldFs, statuses, conf, purge); } } catch (IOException e) { if (isOldPathUnderDestf) { @@ -3495,7 +3499,8 @@ protected void replaceFiles(Path tablePath, Path srcf, Path destf, Path oldPath, * @return true if deletion successful * @throws IOException */ - public static boolean trashFiles(final FileSystem fs, final FileStatus[] statuses, final Configuration conf) + public static boolean trashFiles(final FileSystem fs, final FileStatus[] statuses, + final Configuration conf, final boolean purge) throws IOException { boolean result = true; @@ -3509,13 +3514,13 @@ public static boolean trashFiles(final FileSystem fs, final FileStatus[] statuse final SessionState parentSession = SessionState.get(); for (final FileStatus status : statuses) { if (null == pool) { - result &= FileUtils.moveToTrash(fs, status.getPath(), conf); + result &= FileUtils.moveToTrash(fs, status.getPath(), conf, purge); } else { futures.add(pool.submit(new Callable() { @Override public Boolean call() throws Exception { SessionState.setCurrentSessionState(parentSession); - return FileUtils.moveToTrash(fs, status.getPath(), conf); + return FileUtils.moveToTrash(fs, status.getPath(), conf, purge); } })); } diff --git a/ql/src/test/queries/clientpositive/encryption_auto_purge_tables.q b/ql/src/test/queries/clientpositive/encryption_auto_purge_tables.q new file mode 100644 index 0000000000000000000000000000000000000000..b96a0a037683efc5fc52dec27ae7d36a4d512245 --- /dev/null +++ b/ql/src/test/queries/clientpositive/encryption_auto_purge_tables.q @@ -0,0 +1,38 @@ +-- SORT_QUERY_RESULTS; + +-- we're setting this so that TestNegaiveCliDriver.vm doesn't stop processing after DROP TABLE fails; + +DROP TABLE IF EXISTS encrypted_table PURGE; +DROP TABLE IF EXISTS encrypted_ext_table PURGE; + +CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table'; +CRYPTO CREATE_KEY --keyName key_128 --bitLength 128; +CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table; + +SHOW TABLES; + +ALTER TABLE encrypted_table SET TBLPROPERTIES("auto.purge"="true"); + +INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src; +SELECT COUNT(*) from encrypted_table; + +TRUNCATE TABLE encrypted_table; +SELECT COUNT(*) FROM encrypted_table; + +INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src; +SELECT COUNT(*) FROM encrypted_table; + +CREATE EXTERNAL TABLE encrypted_ext_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table'; +ALTER TABLE encrypted_ext_table SET TBLPROPERTIES("auto.purge"="true"); + +INSERT OVERWRITE TABLE encrypted_ext_table SELECT * FROM src; +SELECT COUNT(*) from encrypted_ext_table; + +DROP TABLE encrypted_table; +DROP TABLE encrypted_ext_table; +SHOW TABLES; + +-- cleanup +DROP TABLE IF EXISTS encrypted_table PURGE; +DROP TABLE IF EXISTS encrypted_ext_table PURGE; +CRYPTO DELETE_KEY --keyName key_128; diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_auto_purge_tables.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_auto_purge_tables.q.out new file mode 100644 index 0000000000000000000000000000000000000000..a2d7dd2cc9788b233e5abdbc99a9109038fa3ada --- /dev/null +++ b/ql/src/test/results/clientpositive/encrypted/encryption_auto_purge_tables.q.out @@ -0,0 +1,157 @@ +PREHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE IF EXISTS encrypted_ext_table PURGE +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS encrypted_ext_table PURGE +POSTHOOK: type: DROPTABLE +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@encrypted_table +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@encrypted_table +Encryption key created: 'key_128' +Encryption zone created: '/build/ql/test/data/warehouse/default/encrypted_table' using key: 'key_128' +PREHOOK: query: SHOW TABLES +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: SHOW TABLES +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +encrypted_table +src +PREHOOK: query: ALTER TABLE encrypted_table SET TBLPROPERTIES("auto.purge"="true") +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@encrypted_table +PREHOOK: Output: default@encrypted_table +POSTHOOK: query: ALTER TABLE encrypted_table SET TBLPROPERTIES("auto.purge"="true") +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@encrypted_table +POSTHOOK: Output: default@encrypted_table +PREHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@encrypted_table +POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@encrypted_table +POSTHOOK: Lineage: encrypted_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: encrypted_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT COUNT(*) from encrypted_table +PREHOOK: type: QUERY +PREHOOK: Input: default@encrypted_table +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging +POSTHOOK: query: SELECT COUNT(*) from encrypted_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@encrypted_table +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging +500 +PREHOOK: query: TRUNCATE TABLE encrypted_table +PREHOOK: type: TRUNCATETABLE +PREHOOK: Output: default@encrypted_table +POSTHOOK: query: TRUNCATE TABLE encrypted_table +POSTHOOK: type: TRUNCATETABLE +POSTHOOK: Output: default@encrypted_table +PREHOOK: query: SELECT COUNT(*) FROM encrypted_table +PREHOOK: type: QUERY +PREHOOK: Input: default@encrypted_table +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging +POSTHOOK: query: SELECT COUNT(*) FROM encrypted_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@encrypted_table +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging +0 +PREHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@encrypted_table +POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@encrypted_table +POSTHOOK: Lineage: encrypted_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: encrypted_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT COUNT(*) FROM encrypted_table +PREHOOK: type: QUERY +PREHOOK: Input: default@encrypted_table +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging +POSTHOOK: query: SELECT COUNT(*) FROM encrypted_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@encrypted_table +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging +500 +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@encrypted_ext_table +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@encrypted_ext_table +PREHOOK: query: ALTER TABLE encrypted_ext_table SET TBLPROPERTIES("auto.purge"="true") +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@encrypted_ext_table +PREHOOK: Output: default@encrypted_ext_table +POSTHOOK: query: ALTER TABLE encrypted_ext_table SET TBLPROPERTIES("auto.purge"="true") +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@encrypted_ext_table +POSTHOOK: Output: default@encrypted_ext_table +PREHOOK: query: INSERT OVERWRITE TABLE encrypted_ext_table SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@encrypted_ext_table +POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_ext_table SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@encrypted_ext_table +POSTHOOK: Lineage: encrypted_ext_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: encrypted_ext_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT COUNT(*) from encrypted_ext_table +PREHOOK: type: QUERY +PREHOOK: Input: default@encrypted_ext_table +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging +POSTHOOK: query: SELECT COUNT(*) from encrypted_ext_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@encrypted_ext_table +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging +500 +PREHOOK: query: DROP TABLE encrypted_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@encrypted_table +PREHOOK: Output: default@encrypted_table +POSTHOOK: query: DROP TABLE encrypted_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@encrypted_table +POSTHOOK: Output: default@encrypted_table +PREHOOK: query: DROP TABLE encrypted_ext_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@encrypted_ext_table +PREHOOK: Output: default@encrypted_ext_table +POSTHOOK: query: DROP TABLE encrypted_ext_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@encrypted_ext_table +POSTHOOK: Output: default@encrypted_ext_table +PREHOOK: query: SHOW TABLES +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: SHOW TABLES +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +src +PREHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE IF EXISTS encrypted_ext_table PURGE +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS encrypted_ext_table PURGE +POSTHOOK: type: DROPTABLE