commit 4d406bbe1a6511103c609a3951ffd198bddfe8e9 Author: Owen O'Malley Date: Thu Apr 3 12:01:35 2014 +0200 HIVE-6830. Remove restriction that base must be completely covered in ACID. diff --git ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java index f4f0631..c7c7c0f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java @@ -306,7 +306,6 @@ public static Directory getAcidState(Path directory, List working = new ArrayList(); final List original = new ArrayList(); final List obsolete = new ArrayList(); - Path ignoredBase = null; List children = SHIMS.listLocatedStatus(fs, directory, hiddenFileFilter); for(FileStatus child: children) { @@ -314,20 +313,15 @@ public static Directory getAcidState(Path directory, String fn = p.getName(); if (fn.startsWith(BASE_PREFIX) && child.isDir()) { long txn = parseBase(p); - if (txnList.isTxnRangeCommitted(0, txn) != - ValidTxnList.RangeResponse.ALL) { - ignoredBase = p; + if (bestBase == null) { + bestBase = child; + bestBaseTxn = txn; + } else if (bestBaseTxn < txn) { + obsolete.add(bestBase); + bestBase = child; + bestBaseTxn = txn; } else { - if (bestBase == null) { - bestBase = child; - bestBaseTxn = txn; - } else if (bestBaseTxn < txn) { - obsolete.add(bestBase); - bestBase = child; - bestBaseTxn = txn; - } else { - obsolete.add(child); - } + obsolete.add(child); } } else if (fn.startsWith(DELTA_PREFIX) && child.isDir()) { ParsedDelta delta = parseDelta(child); @@ -341,13 +335,6 @@ public static Directory getAcidState(Path directory, } } - // Complain if all of the bases were too recent for the minimum excluded - // transaction. - if (bestBase == null && ignoredBase != null) { - throw new IllegalArgumentException("All base directories were ignored," + - " such as " + ignoredBase + " by " + txnList); - } - // if we have a base, the original files are obsolete. if (bestBase != null) { obsolete.addAll(original); diff --git ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java index 8a2e59a..b174496 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java @@ -192,31 +192,18 @@ public void testBestBase() throws Exception { Path part = new MockPath(fs, "/tbl/part1"); AcidUtils.Directory dir = AcidUtils.getAcidState(part, conf, new ValidTxnListImpl("150:")); - assertEquals("mock:/tbl/part1/base_100", dir.getBaseDirectory().toString()); + assertEquals("mock:/tbl/part1/base_200", dir.getBaseDirectory().toString()); List obsoletes = dir.getObsolete(); - assertEquals(3, obsoletes.size()); + assertEquals(4, obsoletes.size()); assertEquals("mock:/tbl/part1/base_10", obsoletes.get(0).getPath().toString()); - assertEquals("mock:/tbl/part1/base_25", obsoletes.get(1).getPath().toString()); - assertEquals("mock:/tbl/part1/base_5", obsoletes.get(2).getPath().toString()); + assertEquals("mock:/tbl/part1/base_100", obsoletes.get(1).getPath().toString()); + assertEquals("mock:/tbl/part1/base_25", obsoletes.get(2).getPath().toString()); + assertEquals("mock:/tbl/part1/base_5", obsoletes.get(3).getPath().toString()); assertEquals(0, dir.getOriginalFiles().size()); assertEquals(0, dir.getCurrentDirectories().size()); - dir = AcidUtils.getAcidState(part, conf, new ValidTxnListImpl("150:99")); - assertEquals("mock:/tbl/part1/base_25", dir.getBaseDirectory().toString()); - obsoletes = dir.getObsolete(); - assertEquals(2, obsoletes.size()); - assertEquals("mock:/tbl/part1/base_10", obsoletes.get(0).getPath().toString()); - assertEquals("mock:/tbl/part1/base_5", obsoletes.get(1).getPath().toString()); - dir = AcidUtils.getAcidState(part, conf, new ValidTxnListImpl("150:25")); - assertEquals("mock:/tbl/part1/base_10", dir.getBaseDirectory().toString()); - obsoletes = dir.getObsolete(); - assertEquals(1, obsoletes.size()); - assertEquals("mock:/tbl/part1/base_5", obsoletes.get(0).getPath().toString()); - try { - AcidUtils.getAcidState(part, conf, new ValidTxnListImpl("150:2")); - fail("should not reach here."); - } catch (IllegalArgumentException iae) { - //expected - } + // we should always get the latest base + dir = AcidUtils.getAcidState(part, conf, new ValidTxnListImpl("10:")); + assertEquals("mock:/tbl/part1/base_200", dir.getBaseDirectory().toString()); } @Test