diff --git ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java index e5f30473ee..1a8132788c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java +++ ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java @@ -331,12 +331,39 @@ void run(HiveConf conf, String jobName, Table t, Partition p, StorageDescriptor return; } + StringBuilder msg = new StringBuilder(); + msg.append("Skipping minor compaction as"); + if ((ci.type == CompactionType.MINOR) && !shouldMinorCompact(parsedDeltas, msg)) { + LOG.error(msg.toString()); + return; + } + launchCompactionJob(job, baseDir, ci.type, dirsToSearch, dir.getCurrentDirectories(), dir.getCurrentDirectories().size(), dir.getObsolete().size(), conf, msc, ci.id, jobName); su.gatherStats(); } + private boolean shouldMinorCompact(List parsedDeltas, StringBuilder msg) { + switch (parsedDeltas.size()) { + case (0): + msg.append(" found no delta directories"); + return false; + case (1): + msg.append(" found a single delta directory"); + return false; + case (2): + if ((parsedDeltas.get(0).isDeleteDelta() && parsedDeltas.get(1).isDeleteDelta()) + || (!parsedDeltas.get(0).isDeleteDelta() && !parsedDeltas.get(1).isDeleteDelta())) { + return true; + } + msg.append(" found a single delta directory and a single delete delta directory"); + return false; + default: + return true; + } + } + /** * @param sd (this is the resolved StorageDescriptor, i.e. resolved to table or partition) * @param writeIds (valid write ids used to filter rows while they're being read for compaction) diff --git ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java index d9e4468c34..76e34113ad 100644 --- ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java +++ ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java @@ -351,25 +351,17 @@ public void minorWithOpenInMiddle() throws Exception { Assert.assertEquals(1, compacts.size()); Assert.assertEquals("ready for cleaning", compacts.get(0).getState()); - // There should still now be 5 directories in the location + // There should still now be 4 directories in the location FileSystem fs = FileSystem.get(conf); FileStatus[] stat = fs.listStatus(new Path(t.getSd().getLocation())); - Assert.assertEquals(toString(stat),6 , stat.length); + Assert.assertEquals(toString(stat), 4, stat.length); // Find the new delta file and make sure it has the right contents Arrays.sort(stat); Assert.assertEquals("base_20", stat[0].getPath().getName()); - /** - * this may look a bit odd. Compactor is capped at min open write id which is 23 in this case - * so the minor compaction above only 1 dir as input, delta_21_22 and outputs - * delta_21_22_v28 (and matching delete_delta) (HIVE-9995/HIVE-20901) - */ - Assert.assertEquals(makeDeleteDeltaDirNameCompacted(21, 22) + "_v0000028", - stat[1].getPath().getName()); - Assert.assertEquals(makeDeltaDirNameCompacted(21, 22), stat[2].getPath().getName()); - Assert.assertEquals(makeDeltaDirNameCompacted(21, 22) + "_v0000028", stat[3].getPath().getName()); - Assert.assertEquals(makeDeltaDirName(23, 25), stat[4].getPath().getName()); - Assert.assertEquals(makeDeltaDirName(26, 27), stat[5].getPath().getName()); + Assert.assertEquals(makeDeltaDirName(21, 22), stat[1].getPath().getName()); + Assert.assertEquals(makeDeltaDirName(23, 25), stat[2].getPath().getName()); + Assert.assertEquals(makeDeltaDirName(26, 27), stat[3].getPath().getName()); } @Test