commit 099ece944b95ba99788a53c69c066f808dd6efab Author: Daniel Dai Date: Tue Dec 5 22:47:14 2017 +0000 HIVE-18227: Tez parallel execution fail diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index cca1055..1f2308a 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -478,7 +478,8 @@ minillap.query.files=acid_bucket_pruning.q,\ unionDistinct_1.q,\ load_fs2.q,\ llap_stats.q,\ - multi_count_distinct_null.q + multi_count_distinct_null.q,\ + multi_insert_parallel.q minillaplocal.query.files=\ dp_counter_non_mm.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java index af77f30..07d72ef 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java @@ -568,7 +568,11 @@ DAGClient submit(JobConf conf, DAG dag, Path scratchDir, try { // ready to start execution on the cluster sessionState.getSession().addAppMasterLocalFiles(resourceMap); - dagClient = sessionState.getSession().submitDAG(dag); + synchronized (sessionState) { + TezClient tezClient = sessionState.getSession(); + tezClient.waitTillReady(); + dagClient = tezClient.submitDAG(dag); + } } catch (SessionNotRunning nr) { console.printInfo("Tez session was closed. Reopening..."); diff --git a/ql/src/test/queries/clientpositive/multi_insert_parallel.q b/ql/src/test/queries/clientpositive/multi_insert_parallel.q new file mode 100644 index 0000000..dd0bb6c --- /dev/null +++ b/ql/src/test/queries/clientpositive/multi_insert_parallel.q @@ -0,0 +1,11 @@ +create table src_multi1 like src; +create table src_multi2 like src; + +set hive.exec.parallel=true; +set hive.merge.tezfiles=true; +set tez.grouping.max-size=2000; +set tez.grouping.min-size=1000; + +from src +insert overwrite table src_multi1 select * where key < 10 +insert overwrite table src_multi2 select * where key > 10 and key < 20; diff --git a/ql/src/test/results/clientpositive/llap/multi_insert_parallel.q.out b/ql/src/test/results/clientpositive/llap/multi_insert_parallel.q.out new file mode 100644 index 0000000..d2dcb19 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/multi_insert_parallel.q.out @@ -0,0 +1,34 @@ +PREHOOK: query: create table src_multi1 like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_multi1 +POSTHOOK: query: create table src_multi1 like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_multi1 +PREHOOK: query: create table src_multi2 like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_multi2 +POSTHOOK: query: create table src_multi2 like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_multi2 +PREHOOK: query: from src +insert overwrite table src_multi1 select * where key < 10 +insert overwrite table src_multi2 select * where key > 10 and key < 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_multi1 +PREHOOK: Output: default@src_multi2 +POSTHOOK: query: from src +insert overwrite table src_multi1 select * where key < 10 +insert overwrite table src_multi2 select * where key > 10 and key < 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_multi1 +POSTHOOK: Output: default@src_multi2 +POSTHOOK: Lineage: src_multi1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_multi1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: src_multi2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_multi2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]