diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index ace24be3c5e..4fb1c864fbd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -324,11 +324,15 @@ public static void clearWork(Configuration conf) { try { if (!HiveConf.getBoolVar(conf, ConfVars.HIVE_RPC_QUERY_PLAN)) { FileSystem fs = mapPath.getFileSystem(conf); - if (fs.exists(mapPath)) { + try { fs.delete(mapPath, true); + } catch (FileNotFoundException e) { + // delete if exists, don't panic if it doesn't } - if (fs.exists(reducePath)) { + try { fs.delete(reducePath, true); + } catch (FileNotFoundException e) { + // delete if exists, don't panic if it doesn't } } } catch (Exception e) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java index a7fd0ef2fae..516b6f41269 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java @@ -81,6 +81,7 @@ import javax.security.auth.login.LoginException; import java.io.BufferedReader; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; import java.io.Serializable; @@ -525,8 +526,10 @@ private Long incrementalDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot, Hive Path bootstrapRoot = new Path(dumpRoot, ReplUtils.INC_BOOTSTRAP_ROOT_DIR_NAME); Path metadataPath = new Path(bootstrapRoot, EximUtil.METADATA_PATH_NAME); FileSystem fs = FileSystem.get(metadataPath.toUri(), conf); - if (fs.exists(metadataPath)) { + try { fs.delete(metadataPath, true); + } catch (FileNotFoundException e) { + // no worries } Path dbRootMetadata = new Path(metadataPath, dbName); Path dbRootData = new Path(bootstrapRoot, EximUtil.DATA_PATH_NAME + File.separator + dbName); @@ -588,8 +591,10 @@ private void cleanFailedEventDirIfExists(Path dumpDir, long resumeFrom) throws I @Override public Void execute() throws IOException { FileSystem fs = FileSystem.get(nextEventRoot.toUri(), conf); - if (fs.exists(nextEventRoot)) { + try { fs.delete(nextEventRoot, true); + } catch (FileNotFoundException e) { + // no worries } return null; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java index 270c5909fce..1896c8bc519 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java @@ -2481,9 +2481,6 @@ static boolean isCompacted(Path baseOrDeltaDir, FileSystem fs, HdfsDirSnapshot d if (dirSnapshot != null && !dirSnapshot.contains(formatFile)) { return false; } - if(dirSnapshot == null && !fs.exists(formatFile)) { - return false; - } try (FSDataInputStream strm = fs.open(formatFile)) { Map metaData = new ObjectMapper().readValue(strm, Map.class); if(!CURRENT_VERSION.equalsIgnoreCase(metaData.get(Field.VERSION))) { @@ -2498,8 +2495,9 @@ static boolean isCompacted(Path baseOrDeltaDir, FileSystem fs, HdfsDirSnapshot d throw new IllegalArgumentException("Unexpected value for " + Field.DATA_FORMAT + ": " + dataFormat); } - } - catch(IOException e) { + } catch (FileNotFoundException e) { + return false; + } catch(IOException e) { String msg = "Failed to read " + baseOrDeltaDir + "/" + METADATA_FILE + ": " + e.getMessage(); LOG.error(msg, e); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FixAcidKeyIndex.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FixAcidKeyIndex.java index 8a7437e8934..5f6e832d528 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FixAcidKeyIndex.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FixAcidKeyIndex.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.ql.io.orc; +import java.io.FileNotFoundException; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; @@ -186,8 +187,10 @@ static void recoverFile(Configuration conf, Path inputPath, String backup) throw Path recoveredPath = getRecoveryFile(inputPath); // make sure that file does not exist - if (fs.exists(recoveredPath)) { + try { fs.delete(recoveredPath, false); + } catch (FileNotFoundException e) { + // no problem, we're just making sure the file doesn't exist } // Writer should match the orc configuration from the original file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java index 1059cb227f2..92d2fb07952 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java @@ -1354,7 +1354,7 @@ private AcidDirInfo callInternal() throws IOException { } } // Fall back to regular API and create statuses without ID. - List children = HdfsUtils.listLocatedStatus(fs.get(), + List children = HdfsUtils.listLocatedStatus(fs.get(), //frogmethod THIS parsedDelta.getPath(), bucketFilter); for (FileStatus child : children) { HdfsFileStatusWithId fileId = AcidUtils.createOriginalObj(null, child); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 8497ec5d52c..bf25bcfbd18 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -4363,10 +4363,12 @@ public void recycleDirToCmPath(Path dataPath, boolean isPurge) throws HiveExcept private static void deleteAndRename(FileSystem destFs, Path destFile, FileStatus srcStatus, Path destPath) throws IOException { - if (destFs.exists(destFile)) { + try { // rename cannot overwrite non empty destination directory, so deleting the destination before renaming. destFs.delete(destFile); - LOG.info("Deleting destination file" + destFile.toUri()); + LOG.info("Deleted destination file" + destFile.toUri()); + } catch (FileNotFoundException e) { + // no worries } if(!destFs.rename(srcStatus.getPath(), destFile)) { throw new IOException("rename for src path: " + srcStatus.getPath() + " to dest:" diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java index 437072cfa47..3fb271dab13 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java @@ -254,10 +254,13 @@ public void renameFileCopiedFromCmPath(Path toPath, FileSystem dstFs, List