commit 4fbc887d1c39ca5d56d6dc43d9bc6710dbc2e82c Author: Vihang Karajgaonkar Date: Tue Feb 21 16:14:12 2017 -0800 HIVE-15880 : Allow insert overwrite query to use auto.purge table property diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java index 9e07c08374b3bd89547abaf50a30350b6eb24176..c906ee782f864c572288ed13ccf8ae299aca909c 100644 --- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java @@ -596,15 +596,19 @@ public static boolean copy(FileSystem srcFS, Path src, * @return true if move successful * @throws IOException */ - public static boolean moveToTrash(FileSystem fs, Path f, Configuration conf) + public static boolean moveToTrash(FileSystem fs, Path f, Configuration conf, boolean isAutopurge) throws IOException { LOG.debug("deleting " + f); boolean result = false; try { - result = Trash.moveToAppropriateTrash(fs, f, conf); - if (result) { - LOG.trace("Moved to trash: " + f); - return true; + if(isAutopurge) { + LOG.debug("auto.purge is true not moving to Trash " + f); + } else { + result = Trash.moveToAppropriateTrash(fs, f, conf); + if (result) { + LOG.trace("Moved to trash: " + f); + return true; + } } } catch (IOException ioe) { // for whatever failure reason including that trash has lower encryption zone diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index adabe70fa8f0fe1b990c6ac578a14ff5af06fc93..b07d672001d267e1882d2ab1452c9a53127065c5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -4526,7 +4526,7 @@ private int truncateTable(Hive db, TruncateTableDesc truncateTableDesc) throws H Map partSpec = truncateTableDesc.getPartSpec(); Table table = db.getTable(tableName, true); - + boolean isAutopurge = "true".equalsIgnoreCase(table.getProperty("auto.purge")); try { // this is not transactional for (Path location : getLocations(db, table, partSpec)) { @@ -4537,7 +4537,7 @@ private int truncateTable(Hive db, TruncateTableDesc truncateTableDesc) throws H HdfsUtils.HadoopFileStatus status = new HdfsUtils.HadoopFileStatus(conf, fs, location); FileStatus targetStatus = fs.getFileStatus(location); String targetGroup = targetStatus == null ? null : targetStatus.getGroup(); - FileUtils.moveToTrash(fs, location, conf); + FileUtils.moveToTrash(fs, location, conf, isAutopurge); fs.mkdirs(location); HdfsUtils.setFullFileStatus(conf, status, targetGroup, fs, location, false); } else { @@ -4545,7 +4545,7 @@ private int truncateTable(Hive db, TruncateTableDesc truncateTableDesc) throws H if (statuses == null || statuses.length == 0) { continue; } - boolean success = Hive.trashFiles(fs, statuses, conf); + boolean success = Hive.trashFiles(fs, statuses, conf, isAutopurge); if (!success) { throw new HiveException("Error in deleting the contents of " + location.toString()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index ed854bf48b7507857e28c155a90963204a05c913..7e7cc4eeec0cf7520f18a338c157620c1018b912 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1646,8 +1646,9 @@ public Partition loadPartition(Path loadPath, Table tbl, PerfLogger perfLogger = SessionState.getPerfLogger(); perfLogger.PerfLogBegin("MoveTask", "FileMoves"); if (replace || (oldPart == null && !isAcid)) { + boolean isAutoPurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge")); replaceFiles(tbl.getPath(), loadPath, newPartPath, oldPartPath, getConf(), - isSrcLocal); + isSrcLocal, isAutoPurge); } else { if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary() && oldPart != null) { newFiles = Collections.synchronizedList(new ArrayList()); @@ -2012,7 +2013,8 @@ public void loadTable(Path loadPath, String tableName, boolean replace, boolean } if (replace) { Path tableDest = tbl.getPath(); - replaceFiles(tableDest, loadPath, tableDest, tableDest, sessionConf, isSrcLocal); + boolean isAutopurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge")); + replaceFiles(tableDest, loadPath, tableDest, tableDest, sessionConf, isSrcLocal, isAutopurge); } else { FileSystem fs; try { @@ -3402,7 +3404,7 @@ private static void moveAcidDeltaFiles(String deltaFileType, PathFilter pathFilt * If the source directory is LOCAL */ protected void replaceFiles(Path tablePath, Path srcf, Path destf, Path oldPath, HiveConf conf, - boolean isSrcLocal) throws HiveException { + boolean isSrcLocal, boolean isAutopurge) throws HiveException { try { FileSystem destFs = destf.getFileSystem(conf); @@ -3435,7 +3437,7 @@ protected void replaceFiles(Path tablePath, Path srcf, Path destf, Path oldPath, // existing content might result in incorrect (extra) data. // But not sure why we changed not to delete the oldPath in HIVE-8750 if it is // not the destf or its subdir? - oldPathDeleted = trashFiles(oldFs, statuses, conf); + oldPathDeleted = trashFiles(oldFs, statuses, conf, isAutopurge); } } catch (IOException e) { if (isOldPathUnderDestf) { @@ -3495,7 +3497,7 @@ protected void replaceFiles(Path tablePath, Path srcf, Path destf, Path oldPath, * @return true if deletion successful * @throws IOException */ - public static boolean trashFiles(final FileSystem fs, final FileStatus[] statuses, final Configuration conf) + public static boolean trashFiles(final FileSystem fs, final FileStatus[] statuses, final Configuration conf, final boolean isAutopurge) throws IOException { boolean result = true; @@ -3509,13 +3511,13 @@ public static boolean trashFiles(final FileSystem fs, final FileStatus[] statuse final SessionState parentSession = SessionState.get(); for (final FileStatus status : statuses) { if (null == pool) { - result &= FileUtils.moveToTrash(fs, status.getPath(), conf); + result &= FileUtils.moveToTrash(fs, status.getPath(), conf, isAutopurge); } else { futures.add(pool.submit(new Callable() { @Override public Boolean call() throws Exception { SessionState.setCurrentSessionState(parentSession); - return FileUtils.moveToTrash(fs, status.getPath(), conf); + return FileUtils.moveToTrash(fs, status.getPath(), conf, isAutopurge); } })); }