diff --git ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java index 8daa5c05f6..b4beec8136 100644 --- ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java +++ ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java @@ -272,7 +272,7 @@ private CompactionType determineCompactionType(CompactionInfo ci, ValidTxnList t } baseSize = sumDirSize(fs, base); } - + //we should either have base/ or originals but not both List originals = dir.getOriginalFiles(); for (HdfsFileStatusWithId origStat : originals) { baseSize += origStat.getFileStatus().getLen(); @@ -281,6 +281,7 @@ private CompactionType determineCompactionType(CompactionInfo ci, ValidTxnList t long deltaSize = 0; List deltas = dir.getCurrentDirectories(); for (AcidUtils.ParsedDelta delta : deltas) { + //this should only look at inserts not delete_delta stat = fs.getFileStatus(delta.getPath()); if (!stat.isDir()) { LOG.error("Was assuming delta " + delta.getPath().toString() + " is a directory, " + diff --git ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java index c827dc4a0e..d30ec11fdf 100644 --- ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java +++ ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java @@ -522,5 +522,51 @@ public void testCtasBucketed() throws Exception { // Assert.assertEquals("Wrong msg", ErrorMsg.CTAS_PARCOL_COEXISTENCE.getErrorCode(), cpr.getErrorCode()); Assert.assertTrue(cpr.getErrorMessage().contains("CREATE-TABLE-AS-SELECT does not support")); } + + /** + * HIVE-16669 + */ + @Test + public void testCompaction() throws Exception { + runStatementOnDriver("drop table if exists T"); + runStatementOnDriver("create table T(a int, b int) stored as ORC TBLPROPERTIES ('transactional'='true')"); + int[][] values = {{1,2},{3,4},{5,6},{7,8},{9,10}}; + + runStatementOnDriver("insert into T " + makeValuesClause(values)); + runStatementOnDriver("alter table T compact 'major'"); + TestTxnCommands2.runWorker(hiveConf); +/* +{"transactionid":15,"bucketid":536870912,"rowid":0} 1 2 warehouse/t/base_0000015/bucket_00000 +{"transactionid":15,"bucketid":536870912,"rowid":1} 3 4 warehouse/t/base_0000015/bucket_00000 +{"transactionid":15,"bucketid":536870912,"rowid":2} 5 6 warehouse/t/base_0000015/bucket_00000 +{"transactionid":15,"bucketid":536870912,"rowid":3} 7 8 warehouse/t/base_0000015/bucket_00000 +{"transactionid":15,"bucketid":536870912,"rowid":4} 9 10 warehouse/t/base_0000015/bucket_00000 +*/ + String[][] expected = { + {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "warehouse/t/base_0000015/bucket_00000"}, + {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "warehouse/t/base_0000015/bucket_00000"}, + {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":2}\t5\t6", "warehouse/t/base_0000015/bucket_00000"}, + {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":3}\t7\t8", "warehouse/t/base_0000015/bucket_00000"}, + {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":4}\t9\t10", "warehouse/t/base_0000015/bucket_00000"}, + + }; + List rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from T order by ROW__ID"); + checkExpected(rs, expected, "after major compact"); + + runStatementOnDriver("insert into T values(11,12)"); + runStatementOnDriver("delete from T where a <= 3"); + + runStatementOnDriver("alter table T compact 'minor'"); + TestTxnCommands2.runWorker(hiveConf); + + String[][] expected2 = {//this is result for MAJOR + {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":2}\t5\t6", "warehouse/t/base_0000019/bucket_00000"}, + {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":3}\t7\t8", "warehouse/t/base_0000019/bucket_00000"}, + {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":4}\t9\t10", "warehouse/t/base_0000019/bucket_00000"}, + {"{\"transactionid\":18,\"bucketid\":536870912,\"rowid\":0}\t11\t12", "warehouse/t/base_0000019/bucket_00000"} + }; + rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from T order by ROW__ID"); + checkExpected(rs, expected2, "after major compact2"); + } }