diff --git ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java index eb75308e83..7d40d579f9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java @@ -1068,13 +1068,18 @@ public boolean isBaseInRawFormat() { * Note that such base is NOT obsolete. Obsolete files are those that are "covered" by other * files within the snapshot. */ - private static boolean isValidBase(long baseTxnId, ValidTxnList txnList) { + private static boolean isValidBase(long baseTxnId, ValidTxnList txnList, Path baseDir, + FileSystem fs) throws IOException { if(baseTxnId == Long.MIN_VALUE) { //such base is created by 1st compaction in case of non-acid to acid table conversion //By definition there are no open txns with id < 1. return true; } - return txnList.isValidBase(baseTxnId); + if(MetaDataFile.isCompacted(baseDir, fs)) { + return txnList.isValidBase(baseTxnId); + } + //this is the IOW case + return txnList.isTxnValid(baseTxnId); } private static void getChildState(FileStatus child, HdfsFileStatusWithId childWithId, ValidTxnList txnList, List working, List originalDirectories, @@ -1091,12 +1096,12 @@ private static void getChildState(FileStatus child, HdfsFileStatusWithId childWi bestBase.oldestBaseTxnId = txn; } if (bestBase.status == null) { - if(isValidBase(txn, txnList)) { + if(isValidBase(txn, txnList, p, fs)) { bestBase.status = child; bestBase.txn = txn; } } else if (bestBase.txn < txn) { - if(isValidBase(txn, txnList)) { + if(isValidBase(txn, txnList, p, fs)) { obsolete.add(bestBase.status); bestBase.status = child; bestBase.txn = txn; @@ -1499,17 +1504,14 @@ public static boolean isRemovedInsertOnlyTable(Set removedSet) { String DATA_FORMAT = "dataFormat"; } private interface Value { - //plain ORC file - String RAW = "raw"; - //result of acid write, i.e. decorated with ROW__ID info - String NATIVE = "native"; + //written by Major compaction + String COMPACTED = "compacted"; } /** * @param baseOrDeltaDir detla or base dir, must exist */ - public static void createMetaFile(Path baseOrDeltaDir, FileSystem fs, boolean isRawFormat) - throws IOException { + public static void createCompactorMarker(Path baseOrDeltaDir, FileSystem fs) throws IOException { /** * create _meta_data json file in baseOrDeltaDir * write thisFileVersion, dataFormat @@ -1519,7 +1521,7 @@ public static void createMetaFile(Path baseOrDeltaDir, FileSystem fs, boolean is Path formatFile = new Path(baseOrDeltaDir, METADATA_FILE); Map metaData = new HashMap<>(); metaData.put(Field.VERSION, CURRENT_VERSION); - metaData.put(Field.DATA_FORMAT, isRawFormat ? Value.RAW : Value.NATIVE); + metaData.put(Field.DATA_FORMAT, Value.COMPACTED); try (FSDataOutputStream strm = fs.create(formatFile, false)) { new ObjectMapper().writeValue(strm, metaData); } catch (IOException ioe) { @@ -1529,8 +1531,7 @@ public static void createMetaFile(Path baseOrDeltaDir, FileSystem fs, boolean is throw ioe; } } - //should be useful for import/export - public static boolean isImport(Path baseOrDeltaDir, FileSystem fs) throws IOException { + static boolean isCompacted(Path baseOrDeltaDir, FileSystem fs) throws IOException { Path formatFile = new Path(baseOrDeltaDir, METADATA_FILE); if(!fs.exists(formatFile)) { return false; @@ -1543,9 +1544,7 @@ public static boolean isImport(Path baseOrDeltaDir, FileSystem fs) throws IOExce } String dataFormat = metaData.getOrDefault(Field.DATA_FORMAT, "null"); switch (dataFormat) { - case Value.NATIVE: - return false; - case Value.RAW: + case Value.COMPACTED: return true; default: throw new IllegalArgumentException("Unexpected value for " + Field.DATA_FORMAT diff --git ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java index 9152b4e08a..8beda1f9f9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java +++ ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java @@ -943,6 +943,9 @@ public void commitJob(JobContext context) throws IOException { fs.rename(fileStatus.getPath(), newPath); } fs.delete(tmpLocation, true); + if(conf.getBoolean(IS_MAJOR, false)) { + AcidUtils.MetaDataFile.createCompactorMarker(finalLocation, fs); + } } @Override