diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 8fafd61..de941e4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -1403,6 +1403,10 @@ private boolean requiresLock() { if (!checkConcurrency()) { return false; } + // Lock operations themselves don't require the lock. + if (isExplicitLockOperation()){ + return false; + } if (!HiveConf.getBoolVar(conf, ConfVars.HIVE_LOCK_MAPRED_ONLY)) { return true; } @@ -1425,6 +1429,22 @@ private boolean requiresLock() { return false; } + private boolean isExplicitLockOperation() { + HiveOperation currentOpt = plan.getOperation(); + if (currentOpt != null) { + switch (currentOpt) { + case LOCKDB: + case UNLOCKDB: + case LOCKTABLE: + case UNLOCKTABLE: + return true; + default: + return false; + } + } + return false; + } + private CommandProcessorResponse createProcessorResponse(int ret) { return new CommandProcessorResponse(ret, errorMessage, SQLState, downstreamError); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index eea2fcc..07c4bbe 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -2448,9 +2448,11 @@ private void analyzeUnlockTable(ASTNode ast) private void analyzeLockDatabase(ASTNode ast) throws SemanticException { String dbName = unescapeIdentifier(ast.getChild(0).getText()); String mode = unescapeIdentifier(ast.getChild(1).getText().toUpperCase()); - - //inputs.add(new ReadEntity(dbName)); - //outputs.add(new WriteEntity(dbName)); + inputs.add(new ReadEntity(getDatabase(dbName))); + // Lock database operation is to acquire the lock explicitly, the operation + // itself doesn't need to be locked. Set the WriteEntity as WriteType: + // DDL_NO_LOCK here, otherwise it will conflict with Hive's transaction. + outputs.add(new WriteEntity(getDatabase(dbName), WriteType.DDL_NO_LOCK)); LockDatabaseDesc lockDatabaseDesc = new LockDatabaseDesc(dbName, mode, HiveConf.getVar(conf, ConfVars.HIVEQUERYID)); lockDatabaseDesc.setQueryStr(ctx.getCmd()); @@ -2461,7 +2463,12 @@ private void analyzeLockDatabase(ASTNode ast) throws SemanticException { private void analyzeUnlockDatabase(ASTNode ast) throws SemanticException { String dbName = unescapeIdentifier(ast.getChild(0).getText()); - + inputs.add(new ReadEntity(getDatabase(dbName))); + // Unlock database operation is to release the lock explicitly, the + // operation itself don't need to be locked. Set the WriteEntity as + // WriteType: DDL_NO_LOCK here, otherwise it will conflict with + // Hive's transaction. + outputs.add(new WriteEntity(getDatabase(dbName), WriteType.DDL_NO_LOCK)); UnlockDatabaseDesc unlockDatabaseDesc = new UnlockDatabaseDesc(dbName); DDLWork work = new DDLWork(getInputs(), getOutputs(), unlockDatabaseDesc); rootTasks.add(TaskFactory.get(work, conf)); diff --git a/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out b/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out index b8bbe93..e783251 100644 --- a/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out +++ b/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out @@ -10,4 +10,6 @@ POSTHOOK: type: CREATEDATABASE POSTHOOK: Output: database:drop_nodblock PREHOOK: query: lock database drop_nodblock shared PREHOOK: type: LOCKDATABASE +PREHOOK: Input: database:drop_nodblock +PREHOOK: Output: database:drop_nodblock FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Current transaction manager does not support explicit lock requests. Transaction manager: org.apache.hadoop.hive.ql.lockmgr.DbTxnManager diff --git a/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out b/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out index ea7a3eb..d7a39f0 100644 --- a/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out +++ b/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out @@ -10,4 +10,6 @@ POSTHOOK: type: CREATEDATABASE POSTHOOK: Output: database:drop_nodbunlock PREHOOK: query: unlock database drop_nodbunlock PREHOOK: type: UNLOCKDATABASE +PREHOOK: Input: database:drop_nodbunlock +PREHOOK: Output: database:drop_nodbunlock FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Current transaction manager does not support explicit lock requests. Transaction manager: org.apache.hadoop.hive.ql.lockmgr.DbTxnManager diff --git a/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out b/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out index 134a57b..2c46159 100644 --- a/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out +++ b/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out @@ -34,8 +34,12 @@ POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpar POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: lock database lockneg1 shared PREHOOK: type: LOCKDATABASE +PREHOOK: Input: database:lockneg1 +PREHOOK: Output: database:lockneg1 POSTHOOK: query: lock database lockneg1 shared POSTHOOK: type: LOCKDATABASE +POSTHOOK: Input: database:lockneg1 +POSTHOOK: Output: database:lockneg1 PREHOOK: query: show locks database lockneg1 PREHOOK: type: SHOWLOCKS POSTHOOK: query: show locks database lockneg1 @@ -53,4 +57,6 @@ POSTHOOK: Input: lockneg1@tstsrcpart@ds=2008-04-08/hr=11 500 PREHOOK: query: unlock database lockneg1 PREHOOK: type: UNLOCKDATABASE +PREHOOK: Input: database:lockneg1 +PREHOOK: Output: database:lockneg1 FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Database lockneg1 is not locked diff --git a/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out b/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out index 0a5d98a..fe12d83 100644 --- a/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out +++ b/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out @@ -6,9 +6,15 @@ POSTHOOK: type: CREATEDATABASE POSTHOOK: Output: database:lockneg4 PREHOOK: query: lock database lockneg4 exclusive PREHOOK: type: LOCKDATABASE +PREHOOK: Input: database:lockneg4 +PREHOOK: Output: database:lockneg4 POSTHOOK: query: lock database lockneg4 exclusive POSTHOOK: type: LOCKDATABASE +POSTHOOK: Input: database:lockneg4 +POSTHOOK: Output: database:lockneg4 PREHOOK: query: lock database lockneg4 shared PREHOOK: type: LOCKDATABASE +PREHOOK: Input: database:lockneg4 +PREHOOK: Output: database:lockneg4 Unable to acquire EXPLICIT, SHARED lock lockneg4 after 1 attempts. FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask diff --git a/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out b/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out index 5ac4e3c..e66965e 100644 --- a/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out +++ b/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out @@ -6,8 +6,12 @@ POSTHOOK: type: CREATEDATABASE POSTHOOK: Output: database:lockneg9 PREHOOK: query: lock database lockneg9 shared PREHOOK: type: LOCKDATABASE +PREHOOK: Input: database:lockneg9 +PREHOOK: Output: database:lockneg9 POSTHOOK: query: lock database lockneg9 shared POSTHOOK: type: LOCKDATABASE +POSTHOOK: Input: database:lockneg9 +POSTHOOK: Output: database:lockneg9 PREHOOK: query: show locks database lockneg9 PREHOOK: type: SHOWLOCKS POSTHOOK: query: show locks database lockneg9 diff --git a/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out b/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out index 5486151..e5c8f3e 100644 --- a/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out +++ b/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out @@ -34,13 +34,19 @@ POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpar POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: lock database lockneg2 shared PREHOOK: type: LOCKDATABASE +PREHOOK: Input: database:lockneg2 +PREHOOK: Output: database:lockneg2 POSTHOOK: query: lock database lockneg2 shared POSTHOOK: type: LOCKDATABASE +POSTHOOK: Input: database:lockneg2 +POSTHOOK: Output: database:lockneg2 PREHOOK: query: show locks PREHOOK: type: SHOWLOCKS POSTHOOK: query: show locks POSTHOOK: type: SHOWLOCKS PREHOOK: query: lock database lockneg2 exclusive PREHOOK: type: LOCKDATABASE +PREHOOK: Input: database:lockneg2 +PREHOOK: Output: database:lockneg2 Unable to acquire EXPLICIT, EXCLUSIVE lock lockneg2 after 1 attempts. FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask