commit 50f52c2761028170578af7b8a6ea3277675fc36a Author: Daniel Dai Date: Mon Dec 11 15:01:27 2017 -0800 HIVE-18227: Tez parallel execution fail diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 37079b7..7998d5f 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -478,7 +478,8 @@ minillap.query.files=acid_bucket_pruning.q,\ unionDistinct_1.q,\ load_fs2.q,\ llap_stats.q,\ - multi_count_distinct_null.q + multi_count_distinct_null.q,\ + multi_insert_parallel.q minillaplocal.query.files=\ dp_counter_non_mm.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java index 8795cfc..6994468 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java @@ -719,6 +719,11 @@ public void shutdown() { closeDagClientOnCancellation(dagClient); } + @Override + public boolean canExecuteInParallel(){ + return false; + } + /** DAG client that does dumb global sync on all the method calls; * Tez DAG client is not thread safe and getting the 2nd one is not recommended. */ public class SyncDagClient extends DAGClient { diff --git a/ql/src/test/queries/clientpositive/multi_insert_parallel.q b/ql/src/test/queries/clientpositive/multi_insert_parallel.q new file mode 100644 index 0000000..dd0bb6c --- /dev/null +++ b/ql/src/test/queries/clientpositive/multi_insert_parallel.q @@ -0,0 +1,11 @@ +create table src_multi1 like src; +create table src_multi2 like src; + +set hive.exec.parallel=true; +set hive.merge.tezfiles=true; +set tez.grouping.max-size=2000; +set tez.grouping.min-size=1000; + +from src +insert overwrite table src_multi1 select * where key < 10 +insert overwrite table src_multi2 select * where key > 10 and key < 20; diff --git a/ql/src/test/results/clientpositive/llap/multi_insert_parallel.q.out b/ql/src/test/results/clientpositive/llap/multi_insert_parallel.q.out new file mode 100644 index 0000000..d2dcb19 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/multi_insert_parallel.q.out @@ -0,0 +1,34 @@ +PREHOOK: query: create table src_multi1 like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_multi1 +POSTHOOK: query: create table src_multi1 like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_multi1 +PREHOOK: query: create table src_multi2 like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_multi2 +POSTHOOK: query: create table src_multi2 like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_multi2 +PREHOOK: query: from src +insert overwrite table src_multi1 select * where key < 10 +insert overwrite table src_multi2 select * where key > 10 and key < 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_multi1 +PREHOOK: Output: default@src_multi2 +POSTHOOK: query: from src +insert overwrite table src_multi1 select * where key < 10 +insert overwrite table src_multi2 select * where key > 10 and key < 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_multi1 +POSTHOOK: Output: default@src_multi2 +POSTHOOK: Lineage: src_multi1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_multi1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: src_multi2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_multi2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]