diff --git data/conf/hive-site.xml data/conf/hive-site.xml
index 7a69711..62364fe 100644
--- data/conf/hive-site.xml
+++ data/conf/hive-site.xml
@@ -70,7 +70,7 @@
javax.jdo.option.ConnectionURL
- jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true
+ jdbc:derby:memory:${test.tmp.dir}/junit_metastore_db;create=true
diff --git metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index c0518ad..cdcd790 100644
--- metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++ metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -305,6 +305,7 @@ public GetOpenTxnsInfoResponse getOpenTxnsInfo() throws MetaException {
"initialized, no record found in next_txn_id");
}
long hwm = rs.getLong(1);
+ LOG.info("zw hwm: " + hwm);
if (rs.wasNull()) {
throw new MetaException("Transaction tables not properly " +
"initialized, null record found in next_txn_id");
diff --git pom.xml pom.xml
index 3c46ad1..3ddec7a 100644
--- pom.xml
+++ pom.xml
@@ -99,7 +99,7 @@
2.4
2.4
2.4.3
- 2.19.1
+ 2.18.1
2.4
2.8
2.9
diff --git ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index da00bb3..f23992a 100644
--- ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -814,9 +814,11 @@ public static Directory getAcidState(Path directory,
// Add original files to obsolete list if any
for (HdfsFileStatusWithId fswid : original) {
obsolete.add(fswid.getFileStatus());
+ LOG.info("zw getChildState1 adds obsolete: " + fswid.getFileStatus().getPath().getName());
}
// Add original direcotries to obsolete list if any
obsolete.addAll(originalDirectories);
+ LOG.info("zw getChildState2 adds obsolete dirs: " + originalDirectories);
// remove the entries so we don't get confused later and think we should
// use them.
original.clear();
@@ -873,6 +875,7 @@ else if (prev != null && next.maxTransaction == prev.maxTransaction
}
else {
obsolete.add(next.path);
+ LOG.info("zw getChildState3 adds obsolete: " + next.getPath().getName());
}
}
@@ -953,11 +956,13 @@ private static void getChildState(FileStatus child, HdfsFileStatusWithId childWi
} else if (bestBase.txn < txn) {
if(isValidBase(txn, txnList)) {
obsolete.add(bestBase.status);
+ LOG.info("zw getChildState1 adds obsolete: " + bestBase.status.getPath().getName());
bestBase.status = child;
bestBase.txn = txn;
}
} else {
obsolete.add(child);
+ LOG.info("zw getChildState2 adds obsolete: " + bestBase.status.getPath().getName());
}
} else if ((fn.startsWith(DELTA_PREFIX) || fn.startsWith(DELETE_DELTA_PREFIX))
&& child.isDir()) {
diff --git ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
index 1aef7ac..8536fce 100644
--- ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
+++ ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
@@ -305,6 +305,7 @@ private void launchCompactionJob(JobConf job, Path baseDir, CompactionType compa
mrJob = job;
}
+ LOG.info("zw VALID_TXNS_KEY: " + job.get(ValidTxnList.VALID_TXNS_KEY));
LOG.info("Submitting " + compactionType + " compaction job '" +
job.getJobName() + "' to " + job.getQueueName() + " queue. " +
"(current delta dirs count=" + curDirNumber +
diff --git ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java
index e85f49c..50c5f2c 100644
--- ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java
+++ ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java
@@ -418,6 +418,10 @@ public void minorPartitionWithBase() throws Exception {
// There should still be four directories in the location.
FileSystem fs = FileSystem.get(conf);
FileStatus[] stat = fs.listStatus(new Path(p.getSd().getLocation()));
+ LOG.info("zw minorPartitionWithBase length: " + stat.length);
+ for (FileStatus aStat : stat) {
+ LOG.info("zw minorPartitionWithBase file: " + aStat.getPath().getName());
+ }
Assert.assertEquals(4, stat.length);
// Find the new delta file and make sure it has the right contents
@@ -469,6 +473,10 @@ public void minorTableNoBase() throws Exception {
if (stat[i].getPath().getName().equals(makeDeltaDirNameCompacted(1, 4))) {
sawNewDelta = true;
FileStatus[] buckets = fs.listStatus(stat[i].getPath());
+ LOG.info("zw minorTableNoBase length: " + stat.length);
+ for (FileStatus aStat : stat) {
+ LOG.info("zw minorTableNoBase file: " + aStat.getPath().getName());
+ }
Assert.assertEquals(2, buckets.length);
Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]"));
@@ -505,6 +513,10 @@ public void majorTableWithBase() throws Exception {
// There should still now be 5 directories in the location
FileSystem fs = FileSystem.get(conf);
FileStatus[] stat = fs.listStatus(new Path(t.getSd().getLocation()));
+ LOG.info("zw majorTableWithBase length: " + stat.length);
+ for (FileStatus aStat : stat) {
+ LOG.info("zw majorTableWithBase file: " + aStat.getPath().getName());
+ }
Assert.assertEquals(4, stat.length);
// Find the new delta file and make sure it has the right contents
@@ -653,6 +665,10 @@ public void majorPartitionWithBase() throws Exception {
// There should still be four directories in the location.
FileSystem fs = FileSystem.get(conf);
FileStatus[] stat = fs.listStatus(new Path(p.getSd().getLocation()));
+ LOG.info("zw majorPartitionWithBase length: " + stat.length);
+ for (FileStatus aStat : stat) {
+ LOG.info("zw majorPartitionWithBase file: " + aStat.getPath().getName());
+ }
Assert.assertEquals(4, stat.length);
// Find the new delta file and make sure it has the right contents
@@ -696,6 +712,10 @@ public void majorTableNoBase() throws Exception {
// There should now be 3 directories in the location
FileSystem fs = FileSystem.get(conf);
FileStatus[] stat = fs.listStatus(new Path(t.getSd().getLocation()));
+ LOG.info("zw majorTableNoBase length: " + stat.length);
+ for (FileStatus aStat : stat) {
+ LOG.info("zw majorTableNoBase file: " + aStat.getPath().getName());
+ }
Assert.assertEquals(3, stat.length);
// Find the new delta file and make sure it has the right contents
@@ -791,6 +811,10 @@ public void minorTableLegacy() throws Exception {
if (stat[i].getPath().getName().equals(makeDeltaDirNameCompacted(21, 24))) {
sawNewDelta = true;
FileStatus[] buckets = fs.listStatus(stat[i].getPath());
+ LOG.info("zw minorTableLegacy length: " + stat.length);
+ for (FileStatus aStat : stat) {
+ LOG.info("zw minorTableLegacy file: " + aStat.getPath().getName());
+ }
Assert.assertEquals(2, buckets.length);
Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]"));
@@ -828,6 +852,10 @@ public void majorPartitionWithBaseMissingBuckets() throws Exception {
// There should still be four directories in the location.
FileSystem fs = FileSystem.get(conf);
FileStatus[] stat = fs.listStatus(new Path(p.getSd().getLocation()));
+ LOG.info("zw majorPartitionWithBaseMissingBuckets length: " + stat.length);
+ for (FileStatus aStat : stat) {
+ LOG.info("zw majorPartitionWithBaseMissingBuckets file: " + aStat.getPath().getName());
+ }
Assert.assertEquals(4, stat.length);
// Find the new delta file and make sure it has the right contents