diff --git data/conf/hive-site.xml data/conf/hive-site.xml index 7a69711..62364fe 100644 --- data/conf/hive-site.xml +++ data/conf/hive-site.xml @@ -70,7 +70,7 @@ javax.jdo.option.ConnectionURL - jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true + jdbc:derby:memory:${test.tmp.dir}/junit_metastore_db;create=true diff --git metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index c0518ad..cdcd790 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -305,6 +305,7 @@ public GetOpenTxnsInfoResponse getOpenTxnsInfo() throws MetaException { "initialized, no record found in next_txn_id"); } long hwm = rs.getLong(1); + LOG.info("zw hwm: " + hwm); if (rs.wasNull()) { throw new MetaException("Transaction tables not properly " + "initialized, null record found in next_txn_id"); diff --git pom.xml pom.xml index 3c46ad1..3ddec7a 100644 --- pom.xml +++ pom.xml @@ -99,7 +99,7 @@ 2.4 2.4 2.4.3 - 2.19.1 + 2.18.1 2.4 2.8 2.9 diff --git ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java index da00bb3..f23992a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java @@ -814,9 +814,11 @@ public static Directory getAcidState(Path directory, // Add original files to obsolete list if any for (HdfsFileStatusWithId fswid : original) { obsolete.add(fswid.getFileStatus()); + LOG.info("zw getChildState1 adds obsolete: " + fswid.getFileStatus().getPath().getName()); } // Add original direcotries to obsolete list if any obsolete.addAll(originalDirectories); + LOG.info("zw getChildState2 adds obsolete dirs: " + originalDirectories); // remove the entries so we don't get confused later and think we should // use them. original.clear(); @@ -873,6 +875,7 @@ else if (prev != null && next.maxTransaction == prev.maxTransaction } else { obsolete.add(next.path); + LOG.info("zw getChildState3 adds obsolete: " + next.getPath().getName()); } } @@ -953,11 +956,13 @@ private static void getChildState(FileStatus child, HdfsFileStatusWithId childWi } else if (bestBase.txn < txn) { if(isValidBase(txn, txnList)) { obsolete.add(bestBase.status); + LOG.info("zw getChildState1 adds obsolete: " + bestBase.status.getPath().getName()); bestBase.status = child; bestBase.txn = txn; } } else { obsolete.add(child); + LOG.info("zw getChildState2 adds obsolete: " + bestBase.status.getPath().getName()); } } else if ((fn.startsWith(DELTA_PREFIX) || fn.startsWith(DELETE_DELTA_PREFIX)) && child.isDir()) { diff --git ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java index 1aef7ac..8536fce 100644 --- ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java +++ ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java @@ -305,6 +305,7 @@ private void launchCompactionJob(JobConf job, Path baseDir, CompactionType compa mrJob = job; } + LOG.info("zw VALID_TXNS_KEY: " + job.get(ValidTxnList.VALID_TXNS_KEY)); LOG.info("Submitting " + compactionType + " compaction job '" + job.getJobName() + "' to " + job.getQueueName() + " queue. " + "(current delta dirs count=" + curDirNumber + diff --git ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java index e85f49c..50c5f2c 100644 --- ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java +++ ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java @@ -418,6 +418,10 @@ public void minorPartitionWithBase() throws Exception { // There should still be four directories in the location. FileSystem fs = FileSystem.get(conf); FileStatus[] stat = fs.listStatus(new Path(p.getSd().getLocation())); + LOG.info("zw minorPartitionWithBase length: " + stat.length); + for (FileStatus aStat : stat) { + LOG.info("zw minorPartitionWithBase file: " + aStat.getPath().getName()); + } Assert.assertEquals(4, stat.length); // Find the new delta file and make sure it has the right contents @@ -469,6 +473,10 @@ public void minorTableNoBase() throws Exception { if (stat[i].getPath().getName().equals(makeDeltaDirNameCompacted(1, 4))) { sawNewDelta = true; FileStatus[] buckets = fs.listStatus(stat[i].getPath()); + LOG.info("zw minorTableNoBase length: " + stat.length); + for (FileStatus aStat : stat) { + LOG.info("zw minorTableNoBase file: " + aStat.getPath().getName()); + } Assert.assertEquals(2, buckets.length); Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]")); Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]")); @@ -505,6 +513,10 @@ public void majorTableWithBase() throws Exception { // There should still now be 5 directories in the location FileSystem fs = FileSystem.get(conf); FileStatus[] stat = fs.listStatus(new Path(t.getSd().getLocation())); + LOG.info("zw majorTableWithBase length: " + stat.length); + for (FileStatus aStat : stat) { + LOG.info("zw majorTableWithBase file: " + aStat.getPath().getName()); + } Assert.assertEquals(4, stat.length); // Find the new delta file and make sure it has the right contents @@ -653,6 +665,10 @@ public void majorPartitionWithBase() throws Exception { // There should still be four directories in the location. FileSystem fs = FileSystem.get(conf); FileStatus[] stat = fs.listStatus(new Path(p.getSd().getLocation())); + LOG.info("zw majorPartitionWithBase length: " + stat.length); + for (FileStatus aStat : stat) { + LOG.info("zw majorPartitionWithBase file: " + aStat.getPath().getName()); + } Assert.assertEquals(4, stat.length); // Find the new delta file and make sure it has the right contents @@ -696,6 +712,10 @@ public void majorTableNoBase() throws Exception { // There should now be 3 directories in the location FileSystem fs = FileSystem.get(conf); FileStatus[] stat = fs.listStatus(new Path(t.getSd().getLocation())); + LOG.info("zw majorTableNoBase length: " + stat.length); + for (FileStatus aStat : stat) { + LOG.info("zw majorTableNoBase file: " + aStat.getPath().getName()); + } Assert.assertEquals(3, stat.length); // Find the new delta file and make sure it has the right contents @@ -791,6 +811,10 @@ public void minorTableLegacy() throws Exception { if (stat[i].getPath().getName().equals(makeDeltaDirNameCompacted(21, 24))) { sawNewDelta = true; FileStatus[] buckets = fs.listStatus(stat[i].getPath()); + LOG.info("zw minorTableLegacy length: " + stat.length); + for (FileStatus aStat : stat) { + LOG.info("zw minorTableLegacy file: " + aStat.getPath().getName()); + } Assert.assertEquals(2, buckets.length); Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]")); Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]")); @@ -828,6 +852,10 @@ public void majorPartitionWithBaseMissingBuckets() throws Exception { // There should still be four directories in the location. FileSystem fs = FileSystem.get(conf); FileStatus[] stat = fs.listStatus(new Path(p.getSd().getLocation())); + LOG.info("zw majorPartitionWithBaseMissingBuckets length: " + stat.length); + for (FileStatus aStat : stat) { + LOG.info("zw majorPartitionWithBaseMissingBuckets file: " + aStat.getPath().getName()); + } Assert.assertEquals(4, stat.length); // Find the new delta file and make sure it has the right contents