diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java index da5dd61d08..2bc6e8ccc5 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java @@ -136,6 +136,7 @@ public static synchronized void prepDb(Configuration conf) throws Exception { " HL_BLOCKEDBY_INT_ID bigint," + " PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID))"); stmt.execute("CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID)"); + stmt.execute("CREATE INDEX HL_DTP_INDEX ON HIVE_LOCKS (HL_DB, HL_TABLE, HL_PARTITION)"); stmt.execute("CREATE TABLE NEXT_LOCK_ID (" + " NL_NEXT bigint NOT NULL)"); stmt.execute("INSERT INTO NEXT_LOCK_ID VALUES(1)"); @@ -395,16 +396,8 @@ public static void cleanDb(Configuration conf) throws Exception { stmt = conn.createStatement(); // We want to try these, whether they succeed or fail. - try { - stmt.execute("DROP INDEX HL_TXNID_INDEX"); - } catch (SQLException e) { - if(!("42X65".equals(e.getSQLState()) && 30000 == e.getErrorCode())) { - //42X65/3000 means index doesn't exist - LOG.error("Unable to drop index HL_TXNID_INDEX " + e.getMessage() + - "State=" + e.getSQLState() + " code=" + e.getErrorCode() + " retryCount=" + retryCount); - success = false; - } - } + success &= dropIndex(stmt, "HL_TXNID_INDEX", retryCount); + success &= dropIndex(stmt, "HL_DTP_INDEX", retryCount); success &= dropTable(stmt, "TXN_COMPONENTS", retryCount); success &= dropTable(stmt, "COMPLETED_TXN_COMPONENTS", retryCount); @@ -438,6 +431,20 @@ public static void cleanDb(Configuration conf) throws Exception { throw new RuntimeException("Failed to clean up txn tables"); } + private static boolean dropIndex(Statement stmt, String index, int retryCount) { + try { + stmt.execute("DROP INDEX " + index); + } catch (SQLException e) { + if (!("42X65".equals(e.getSQLState()) && 30000 == e.getErrorCode())) { + //42X65/3000 means index doesn't exist + LOG.error("Unable to drop index {} {} State={} code={} retryCount={}", + index, e.getMessage(), e.getSQLState(), e.getErrorCode(), retryCount); + return false; + } + } + return true; + } + private static boolean dropTable(Statement stmt, String name, int retryCount) throws SQLException { for (int i = 0; i < 3; i++) { try { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index aa62ca2120..d67e8be665 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -4335,17 +4335,10 @@ private LockResponse checkLock(Connection dbConn, long extLockId) LOG.debug("checkLock(): Setting savepoint. extLockId=" + JavaUtils.lockIdToString(extLockId)); Savepoint save = dbConn.setSavepoint(); - StringBuilder query = new StringBuilder("SELECT \"HL_LOCK_EXT_ID\", " + - "\"HL_LOCK_INT_ID\", \"HL_DB\", \"HL_TABLE\", \"HL_PARTITION\", \"HL_LOCK_STATE\", " + - "\"HL_LOCK_TYPE\", \"HL_TXNID\" FROM \"HIVE_LOCKS\" WHERE \"HL_DB\" IN ("); - - Set strings = new HashSet<>(locksBeingChecked.size()); //This the set of entities that the statement represented by extLockId wants to update List writeSet = new ArrayList<>(); - for (LockInfo info : locksBeingChecked) { - strings.add(info.db); if(!isPartOfDynamicPartitionInsert && info.type == LockType.SHARED_WRITE) { writeSet.add(info); } @@ -4403,66 +4396,29 @@ private LockResponse checkLock(Connection dbConn, long extLockId) close(rs, stmt, null); } - boolean first = true; - for (String s : strings) { - if (first) first = false; - else query.append(", "); - query.append('\''); - query.append(s); - query.append('\''); - } - query.append(")"); + StringBuilder query = new StringBuilder("SELECT \"LS\".* FROM (") + .append(" SELECT \"HL_LOCK_EXT_ID\", \"HL_LOCK_INT_ID\", \"HL_DB\", \"HL_TABLE\", \"HL_PARTITION\",") + .append(" \"HL_LOCK_STATE\", \"HL_LOCK_TYPE\", \"HL_TXNID\" FROM \"HIVE_LOCKS\"") + .append(" WHERE \"HL_LOCK_EXT_ID\" < ").append(extLockId).append(") \"LS\"") + .append(" LEFT JOIN (SELECT \"HL_DB\", \"HL_TABLE\", \"HL_PARTITION\" FROM \"HIVE_LOCKS\"") + .append(" WHERE \"HL_LOCK_EXT_ID\" = ").append(extLockId).append(") \"LBC\"") + .append(" ON \"LS\".\"HL_DB\" = \"LBC\".\"HL_DB\""); + StringBuilder whereClause = new StringBuilder(); // If any of the table requests are null, then I need to pull all the // table locks for this db. - boolean sawNull = false; - strings.clear(); - for (LockInfo info : locksBeingChecked) { - if (info.table == null) { - sawNull = true; - break; - } else { - strings.add(info.table); - } - } - if (!sawNull) { - query.append(" AND (\"HL_TABLE\" IS NULL OR \"HL_TABLE\" IN("); - first = true; - for (String s : strings) { - if (first) first = false; - else query.append(", "); - query.append('\''); - query.append(s); - query.append('\''); - } - query.append("))"); + if (locksBeingChecked.stream().noneMatch(info -> info.table == null)) { + query.append(" AND \"LS\".\"HL_TABLE\" = \"LBC\".\"HL_TABLE\""); + whereClause.append(" WHERE (\"LS\".\"HL_TABLE\" IS NULL OR \"LBC\".\"HL_DB\" IS NOT NULL)"); // If any of the partition requests are null, then I need to pull all // partition locks for this table. - sawNull = false; - strings.clear(); - for (LockInfo info : locksBeingChecked) { - if (info.partition == null) { - sawNull = true; - break; - } else { - strings.add(info.partition); - } - } - if (!sawNull) { - query.append(" AND (\"HL_PARTITION\" IS NULL OR \"HL_PARTITION\" IN("); - first = true; - for (String s : strings) { - if (first) first = false; - else query.append(", "); - query.append('\''); - query.append(s); - query.append('\''); - } - query.append("))"); + if (locksBeingChecked.stream().noneMatch(info -> info.partition == null)) { + query.append(" AND \"LS\".\"HL_PARTITION\" = \"LBC\".\"HL_PARTITION\""); + whereClause.append(" AND (\"LS\".\"HL_PARTITION\" IS NULL OR \"LBC\".\"HL_DB\" IS NOT NULL)"); } } - query.append(" AND \"HL_LOCK_EXT_ID\" < ").append(extLockId); + query.append(whereClause.toString()); LOG.debug("Going to execute query <" + query.toString() + ">"); stmt = dbConn.createStatement(); diff --git a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql index 6710271886..ca6087680f 100644 --- a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql +++ b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql @@ -568,6 +568,7 @@ CREATE TABLE HIVE_LOCKS ( ); CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID); +CREATE INDEX HL_DTP_INDEX ON HIVE_LOCKS (HL_DB, HL_TABLE, HL_PARTITION); CREATE TABLE NEXT_LOCK_ID ( NL_NEXT bigint NOT NULL diff --git a/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql b/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql index ae0a32541a..16ec71b95c 100644 --- a/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql +++ b/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql @@ -48,6 +48,8 @@ CREATE INDEX LASTUPDATETIMEINDEX ON APP.SCHEDULED_EXECUTIONS (LAST_UPDATE_TIME); CREATE INDEX SCHEDULED_EXECUTIONS_SCHQID ON APP.SCHEDULED_EXECUTIONS (SCHEDULED_QUERY_ID); CREATE UNIQUE INDEX SCHEDULED_EXECUTIONS_UNIQUE_ID ON APP.SCHEDULED_EXECUTIONS (SCHEDULED_EXECUTION_ID); +CREATE INDEX HL_DTP_INDEX ON HIVE_LOCKS (HL_DB, HL_TABLE, HL_PARTITION); + -- This needs to be the last thing done. Insert any changes above this line. UPDATE "APP".VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; diff --git a/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql b/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql index 221d4f1fff..06e9001ea7 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql @@ -1075,6 +1075,8 @@ PRIMARY KEY CLUSTERED ) ); +CREATE INDEX HL_DTP_IDX ON HIVE_LOCKS (HL_DB, HL_TABLE, HL_PARTITION); + CREATE TABLE NEXT_COMPACTION_QUEUE_ID( NCQ_NEXT bigint NOT NULL ); diff --git a/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql b/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql index bc98d5fc4a..39ce159e14 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql @@ -22,6 +22,8 @@ UPDATE TAB_COL_STATS SET ENGINE = 'hive' WHERE ENGINE IS NULL; ALTER TABLE PART_COL_STATS ADD ENGINE nvarchar(128); UPDATE PART_COL_STATS SET ENGINE = 'hive' WHERE ENGINE IS NULL; +CREATE INDEX HL_DTP_IDX ON HIVE_LOCKS (HL_DB, HL_TABLE, HL_PARTITION); + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.2.0 to 4.0.0' AS MESSAGE; diff --git a/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql b/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql index dd761a66db..24be0d4e1b 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql @@ -1051,6 +1051,7 @@ CREATE TABLE HIVE_LOCKS ( ) ENGINE=InnoDB DEFAULT CHARSET=latin1; CREATE INDEX HL_TXNID_IDX ON HIVE_LOCKS (HL_TXNID); +CREATE INDEX HL_DTP_IDX ON HIVE_LOCKS (HL_DB, HL_TABLE, HL_PARTITION) USING BTREE; CREATE TABLE NEXT_LOCK_ID ( NL_NEXT bigint NOT NULL diff --git a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql index 6a040a6a64..ecf3b75b98 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql @@ -52,6 +52,8 @@ CREATE INDEX IDX_SCHEDULED_EXECUTIONS_LAST_UPDATE_TIME ON SCHEDULED_EXECUTIONS ( CREATE INDEX IDX_SCHEDULED_EXECUTIONS_SCHEDULED_QUERY_ID ON SCHEDULED_EXECUTIONS (SCHEDULED_QUERY_ID); CREATE UNIQUE INDEX UNIQUE_SCHEDULED_EXECUTIONS_ID ON SCHEDULED_EXECUTIONS (SCHEDULED_EXECUTION_ID); +CREATE INDEX HL_DTP_IDX ON HIVE_LOCKS (HL_DB, HL_TABLE, HL_PARTITION) USING BTREE; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.2.0 to 4.0.0' AS MESSAGE; diff --git a/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql b/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql index f5ec1ba1af..da564e5847 100644 --- a/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql +++ b/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql @@ -1032,6 +1032,7 @@ CREATE TABLE HIVE_LOCKS ( ) ROWDEPENDENCIES; CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID); +CREATE INDEX HL_DTP_INDEX ON HIVE_LOCKS (HL_DB, HL_TABLE, HL_PARTITION); CREATE TABLE NEXT_LOCK_ID ( NL_NEXT NUMBER(19) NOT NULL diff --git a/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql b/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql index c7738be273..53dbcc5928 100644 --- a/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql +++ b/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql @@ -52,6 +52,8 @@ CREATE TABLE "SCHEDULED_EXECUTIONS" ( CREATE INDEX IDX_SCHEDULED_EX_LAST_UPDATE ON "SCHEDULED_EXECUTIONS" ("LAST_UPDATE_TIME"); CREATE INDEX IDX_SCHEDULED_EX_SQ_ID ON "SCHEDULED_EXECUTIONS" ("SCHEDULED_QUERY_ID"); +CREATE INDEX HL_DTP_INDEX ON HIVE_LOCKS (HL_DB, HL_TABLE, HL_PARTITION); + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.2.0 to 4.0.0' AS Status from dual; diff --git a/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql b/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql index 455f98b725..6b80ebdd35 100644 --- a/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql +++ b/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql @@ -1718,6 +1718,7 @@ CREATE TABLE "HIVE_LOCKS" ( ); CREATE INDEX HL_TXNID_INDEX ON "HIVE_LOCKS" USING hash ("HL_TXNID"); +CREATE INDEX HL_DTP_INDEX ON "HIVE_LOCKS" USING btree ("HL_DB", "HL_TABLE", "HL_PARTITION"); CREATE TABLE "NEXT_LOCK_ID" ( "NL_NEXT" bigint NOT NULL diff --git a/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql b/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql index 5c39b0d9f4..f4f9088080 100644 --- a/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql +++ b/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql @@ -183,6 +183,7 @@ ALTER TABLE "WRITE_SET" RENAME COLUMN ws_txnid TO "WS_TXNID"; ALTER TABLE "WRITE_SET" RENAME COLUMN ws_commit_id TO "WS_COMMIT_ID"; ALTER TABLE "WRITE_SET" RENAME COLUMN ws_operation_type TO "WS_OPERATION_TYPE"; +CREATE INDEX HL_DTP_INDEX ON "HIVE_LOCKS" USING btree ("HL_DB", "HL_TABLE", "HL_PARTITION"); -- These lines need to be last. Insert any changes above. UPDATE "VERSION" SET "SCHEMA_VERSION"='4.0.0', "VERSION_COMMENT"='Hive release version 4.0.0' where "VER_ID"=1;