diff --git ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java index fdf6676..00f0bbb 100644 --- ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java +++ ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.ql.lockmgr; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.common.ValidTxnList; @@ -26,6 +27,7 @@ import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryPlan; +import org.apache.hadoop.hive.ql.hooks.Entity; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.metadata.DummyPartition; @@ -126,26 +128,65 @@ public void acquireLocks(QueryPlan plan, Context ctx, String username) throws Lo for (WriteEntity output : plan.getOutputs()) { LOG.debug("Adding " + output.getName() + " to list of lock outputs"); - List lockObj = null; - if (output.getType() == WriteEntity.Type.DATABASE) { - lockObjects.addAll(getLockObjects(plan, output.getDatabase(), null, - null, - output.isComplete() ? HiveLockMode.EXCLUSIVE : HiveLockMode.SHARED)); - } else if (output.getTyp() == WriteEntity.Type.TABLE) { - lockObj = getLockObjects(plan, null, output.getTable(), null, - output.isComplete() ? HiveLockMode.EXCLUSIVE : HiveLockMode.SHARED); - } else if (output.getTyp() == WriteEntity.Type.PARTITION) { - lockObj = getLockObjects(plan, null, null, output.getPartition(), - HiveLockMode.EXCLUSIVE); + if (output.getType() == Entity.Type.DFS_DIR || output.getType() == Entity.Type.LOCAL_DIR) { + // We don't lock files or directories. + continue; } - // In case of dynamic queries, it is possible to have incomplete dummy partitions - else if (output.getTyp() == WriteEntity.Type.DUMMYPARTITION) { - lockObj = getLockObjects(plan, null, null, output.getPartition(), - HiveLockMode.SHARED); + + List lockObj; + Database lockDb = null; + Table lockTable = null; + Partition lockPt = null; + HiveLockMode mode; + Boolean isLockDb = false; + + switch (output.getWriteType()) { + case INSERT: + case DDL_EXCLUSIVE: + case INSERT_OVERWRITE: + mode = HiveLockMode.EXCLUSIVE; + break; + + case DDL_SHARED: + mode = HiveLockMode.SHARED; + break; + + case UPDATE: + case DELETE: + mode = HiveLockMode.SEMI_SHARED; + break; + + case DDL_NO_LOCK: + continue; // No lock required here + + default: + throw new RuntimeException("Unknown write type " + + output.getWriteType().toString()); + } + switch (output.getType()) { + case DATABASE: + lockDb = output.getDatabase(); + isLockDb = true; + break; + + case TABLE: + case DUMMYPARTITION: + lockTable = output.getTable(); + break; + + case PARTITION: + lockPt = output.getPartition(); + break; + + default: + // This is a file or something we don't hold locks for. + continue; } - if(lockObj != null) { - lockObjects.addAll(lockObj); + lockObj = getLockObjects(plan, lockDb, lockTable, lockPt, mode); + lockObjects.addAll(lockObj); + + if(!isLockDb) { ctx.getOutputLockObjects().put(output, lockObj); } } diff --git ql/src/test/queries/clientpositive/insert_into7.q ql/src/test/queries/clientpositive/insert_into7.q new file mode 100644 index 0000000..38b8838 --- /dev/null +++ ql/src/test/queries/clientpositive/insert_into7.q @@ -0,0 +1,9 @@ +set hive.lock.numretries=5; +set hive.lock.sleep.between.retries=5; + +create database test_lock; +use test_lock; + +lock database test_lock SHARED; + +create table test_lock(a int, b string); diff --git ql/src/test/results/clientpositive/insert_into7.q.out ql/src/test/results/clientpositive/insert_into7.q.out new file mode 100644 index 0000000..e97ee47 --- /dev/null +++ ql/src/test/results/clientpositive/insert_into7.q.out @@ -0,0 +1,24 @@ +PREHOOK: query: create database test_lock +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:test_lock +POSTHOOK: query: create database test_lock +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:test_lock +PREHOOK: query: use test_lock +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:test_lock +POSTHOOK: query: use test_lock +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:test_lock +PREHOOK: query: lock database test_lock SHARED +PREHOOK: type: LOCKDATABASE +POSTHOOK: query: lock database test_lock SHARED +POSTHOOK: type: LOCKDATABASE +PREHOOK: query: create table test_lock(a int, b string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:test_lock +PREHOOK: Output: test_lock@test_lock +POSTHOOK: query: create table test_lock(a int, b string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:test_lock +POSTHOOK: Output: test_lock@test_lock diff --git ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out index e598191..e090c88 100644 --- ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out +++ ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out @@ -1796,9 +1796,9 @@ STAGE PLANS: value expressions: key (type: string) auto parallelism: true Path -> Alias: - -mr-10002default.src{} [s1] + -mr-10003default.src{} [s1] Path -> Partition: - -mr-10002default.src{} + -mr-10003default.src{} Partition input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1842,7 +1842,7 @@ STAGE PLANS: name: default.src name: default.src Truncated Path -> Alias: - -mr-10002default.src{} [s1] + -mr-10003default.src{} [s1] Map 3 Map Operator Tree: TableScan @@ -1862,9 +1862,9 @@ STAGE PLANS: value expressions: key (type: string) auto parallelism: true Path -> Alias: - -mr-10003default.src{} [s2] + -mr-10002default.src{} [s2] Path -> Partition: - -mr-10003default.src{} + -mr-10002default.src{} Partition input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1908,7 +1908,7 @@ STAGE PLANS: name: default.src name: default.src Truncated Path -> Alias: - -mr-10003default.src{} [s2] + -mr-10002default.src{} [s2] Reducer 2 Needs Tagging: false Reduce Operator Tree: