diff --git ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java index da00bb3363..cff357d09e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java @@ -122,6 +122,14 @@ public boolean accept(Path p){ } }; + public static final String TMP_PREFIX = "_tmp"; + public static final PathFilter tmpFileFilter = new PathFilter(){ + @Override + public boolean accept(Path p){ + return p.getName().startsWith(TMP_PREFIX); + } + }; + private static final HadoopShims SHIMS = ShimLoader.getHadoopShims(); /** @@ -796,14 +804,21 @@ public static Directory getAcidState(Path directory, } TxnBase bestBase = new TxnBase(); final List original = new ArrayList<>(); + List tmpDirectories = new ArrayList(); if (childrenWithId != null) { + for (HdfsFileStatusWithId child : SHIMS.listLocatedHdfsStatus(fs, directory, tmpFileFilter)) { + tmpDirectories.add(child.getFileStatus()); + } for (HdfsFileStatusWithId child : childrenWithId) { + if (isUnderCompaction(fs, child.getFileStatus(), tmpDirectories)) continue; getChildState(child.getFileStatus(), child, txnList, working, originalDirectories, original, obsolete, bestBase, ignoreEmptyFiles); } } else { + tmpDirectories = HdfsUtils.listLocatedStatus(fs, directory, tmpFileFilter); List children = HdfsUtils.listLocatedStatus(fs, directory, hiddenFileFilter); for (FileStatus child : children) { + if (isUnderCompaction(fs, child, tmpDirectories)) continue; getChildState( child, null, txnList, working, originalDirectories, original, obsolete, bestBase, ignoreEmptyFiles); } @@ -917,6 +932,21 @@ public Path getBaseDirectory() { } }; } + private static boolean isUnderCompaction(FileSystem fs, FileStatus child, + List tmpDirectories) throws IOException { + if (tmpDirectories.size() == 0 || !child.isDir()) { + return false; + } + for (FileStatus tmpDir : tmpDirectories) { + List childTmpDirectories = HdfsUtils.listLocatedStatus(fs, tmpDir.getPath(), hiddenFileFilter); + for (FileStatus childTmpDir : childTmpDirectories) { + if (childTmpDir.getPath().getName().equals( child.getPath().getName() ) && childTmpDir.isDir()) { + return true; + } + } + } + return false; + } /** * We can only use a 'base' if it doesn't have an open txn (from specific reader's point of view) * A 'base' with open txn in its range doesn't have 'enough history' info to produce a correct