diff --git metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java index 9145fcc..fde1b54 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java @@ -726,6 +726,10 @@ public void purgeCompactionHistory() throws MetaException { } close(rs); + if (deleteSet.size() <= 0) { + return; + } + List queries = new ArrayList(); StringBuilder prefix = new StringBuilder(); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index 97d09df..382df1f 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -2844,11 +2844,11 @@ public void performTimeOuts() { List currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE); timedOutTxns.add(currentBatch); do { - currentBatch.add(rs.getLong(1)); if(currentBatch.size() == TIMED_OUT_TXN_ABORT_BATCH_SIZE) { currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE); timedOutTxns.add(currentBatch); } + currentBatch.add(rs.getLong(1)); } while(rs.next()); dbConn.commit(); close(rs, stmt, null); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java index 2ffa1da..e9c1890 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java @@ -31,6 +31,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.sql.SQLException; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -144,7 +145,10 @@ public static boolean isAcidTable(Table table) { */ public static void buildQueryWithINClause(HiveConf conf, List queries, StringBuilder prefix, StringBuilder suffix, List inList, - String inColumn, boolean addParens, boolean notIn) { + String inColumn, boolean addParens, boolean notIn) throws SQLException { + if (inList == null || inList.size() == 0) { + throw new SQLException("The IN List is empty!"); + } int batchSize = conf.getIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE); int numWholeBatches = inList.size() / batchSize; StringBuilder buf = new StringBuilder(); @@ -160,6 +164,11 @@ public static void buildQueryWithINClause(HiveConf conf, List queries, S } for (int i = 0; i <= numWholeBatches; i++) { + if (i * batchSize == inList.size()) { + // At this point we just realized we don't need another query + break; + } + if (needNewQuery(conf, buf)) { // Wrap up current query string if (addParens) { @@ -198,10 +207,6 @@ public static void buildQueryWithINClause(HiveConf conf, List queries, S } } - if (i * batchSize == inList.size()) { - // At this point we just realized we don't need another query - return; - } for (int j = i * batchSize; j < (i + 1) * batchSize && j < inList.size(); j++) { buf.append(inList.get(j)).append(","); } diff --git metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java index 8fada2c..4c3b824 100644 --- metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java +++ metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java @@ -70,24 +70,56 @@ public void testBuildQueryWithINClause() throws Exception { Assert.assertEquals(2, queries.size()); runAgainstDerby(queries); - // Case 3 - Max in list members: 1000; Max query string length: 5KB + // Case 3.1 - Max in list members: 1000, Max query string length: 1KB, and exact 1000 members in a single IN clause + conf.setIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_QUERY_LENGTH, 1); + conf.setIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE, 1000); + queries.clear(); + for (long i = 202; i <= 1000; i++) { + inList.add(i); + } + TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, false); + Assert.assertEquals(1, queries.size()); + runAgainstDerby(queries); + + // Case 3.2 - Max in list members: 1000, Max query string length: 10KB, and exact 1000 members in a single IN clause conf.setIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_QUERY_LENGTH, 10); conf.setIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE, 1000); queries.clear(); - for (long i = 202; i <= 4321; i++) { + TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, false); + Assert.assertEquals(1, queries.size()); + runAgainstDerby(queries); + + // Case 3.3 - Now with 2000 entries, try the above settings + for (long i = 1001; i <= 2000; i++) { + inList.add(i); + } + conf.setIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_QUERY_LENGTH, 1); + queries.clear(); + TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, false); + Assert.assertEquals(2, queries.size()); + runAgainstDerby(queries); + conf.setIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_QUERY_LENGTH, 10); + queries.clear(); + TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, false); + Assert.assertEquals(1, queries.size()); + runAgainstDerby(queries); + + // Case 4 - Max in list members: 1000; Max query string length: 10KB + queries.clear(); + for (long i = 2001; i <= 4321; i++) { inList.add(i); } TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, false); Assert.assertEquals(3, queries.size()); runAgainstDerby(queries); - // Case 4 - NOT IN list + // Case 5 - NOT IN list queries.clear(); TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, true); Assert.assertEquals(3, queries.size()); runAgainstDerby(queries); - // Case 5 - No parenthesis + // Case 6 - No parenthesis queries.clear(); suffix.setLength(0); suffix.append("");