From 8df9bc40cc30d7e6fbc4c1975d6199122c5246ab Mon Sep 17 00:00:00 2001 From: nareshpr Date: Tue, 26 May 2020 18:44:06 -0700 Subject: [PATCH] HIVE-23461 - Needs to capture input/output entities in explainRewrite --- .../hive/ql/hooks/EnforceReadOnlyTables.java | 2 +- .../ExplainSQRewriteSemanticAnalyzer.java | 2 ++ .../subquery_exists_explain_rewrite.q.out | 8 +++++ .../llap/subquery_in_explain_rewrite.q.out | 32 +++++++++++++++++++ 4 files changed, 43 insertions(+), 1 deletion(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java index d5c8a515d9..3b2c4a727a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java @@ -41,7 +41,7 @@ public void run(HookContext hookContext) throws Exception { Set outputs = hookContext.getOutputs(); UserGroupInformation ugi = hookContext.getUgi(); boolean isExplain = hookContext.getQueryPlan().isExplain(); - this.run(ss,inputs,outputs,ugi, isExplain); + this.run(ss, inputs, outputs, ugi, isExplain); } public void run(SessionState sess, Set inputs, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSQRewriteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSQRewriteSemanticAnalyzer.java index 20e4290bfd..6035efc190 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSQRewriteSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSQRewriteSemanticAnalyzer.java @@ -44,6 +44,8 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { SemanticAnalyzerFactory.get(queryState, input); sem.analyze(input, ctx); sem.validate(); + inputs = sem.getInputs(); + outputs = sem.getOutputs(); ctx.setResFile(ctx.getLocalTmpPath()); diff --git a/ql/src/test/results/clientpositive/llap/subquery_exists_explain_rewrite.q.out b/ql/src/test/results/clientpositive/llap/subquery_exists_explain_rewrite.q.out index 6bb95c898c..529ba4e6e6 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_exists_explain_rewrite.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_exists_explain_rewrite.q.out @@ -7,6 +7,8 @@ where exists where b.value = a.value and a.key = b.key and a.value > 'val_9' ) PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### POSTHOOK: query: explain rewrite select * from src b @@ -16,6 +18,8 @@ where exists where b.value = a.value and a.key = b.key and a.value > 'val_9' ) POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### Where Clause Rewritten SubQuery: (select a.key, a.value as sq_corr_0, a.key as sq_corr_1 @@ -43,6 +47,8 @@ from (select * where b.value = a.value and a.key = b.key and a.value > 'val_9') ) a PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### POSTHOOK: query: explain rewrite select * from (select * @@ -53,6 +59,8 @@ from (select * where b.value = a.value and a.key = b.key and a.value > 'val_9') ) a POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### Where Clause for Query Block 'a' Rewritten SubQuery: (select a.key, a.value as sq_corr_0, a.key as sq_corr_1 diff --git a/ql/src/test/results/clientpositive/llap/subquery_in_explain_rewrite.q.out b/ql/src/test/results/clientpositive/llap/subquery_in_explain_rewrite.q.out index f87ffff6a3..8482176368 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_in_explain_rewrite.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_in_explain_rewrite.q.out @@ -3,11 +3,15 @@ PREHOOK: query: explain rewrite from src where src.key in (select key from src s1 where s1.key > '9') PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### POSTHOOK: query: explain rewrite select * from src where src.key in (select key from src s1 where s1.key > '9') POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### Where Clause Rewritten SubQuery: (select key from src s1 where s1.key > '9') sq_1 @@ -28,6 +32,8 @@ where b.key in where b.value = a.value and a.key > '9' ) PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### POSTHOOK: query: explain rewrite select * from src b @@ -37,6 +43,8 @@ where b.key in where b.value = a.value and a.key > '9' ) POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### Where Clause Rewritten SubQuery: (select a.key, a.value as sq_corr_0 @@ -63,6 +71,8 @@ part where part.p_size in where r <= 2 ) PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### POSTHOOK: query: explain rewrite select p_name, p_size from @@ -72,6 +82,8 @@ part where part.p_size in where r <= 2 ) POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### Where Clause Rewritten SubQuery: (select avg(p_size) @@ -97,6 +109,8 @@ from part b where b.p_size in where r <= 2 and b.p_mfgr = a.p_mfgr ) PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### POSTHOOK: query: explain rewrite select p_mfgr, p_name, p_size from part b where b.p_size in @@ -105,6 +119,8 @@ from part b where b.p_size in where r <= 2 and b.p_mfgr = a.p_mfgr ) POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### Where Clause Rewritten SubQuery: (select min(p_size), a.p_mfgr as sq_corr_0 @@ -130,6 +146,8 @@ where b.key in where b.value = a.value and a.key > '9' ) PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### POSTHOOK: query: explain rewrite select * from src b @@ -139,6 +157,8 @@ where b.key in where b.value = a.value and a.key > '9' ) POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### Where Clause Rewritten SubQuery: (select distinct a.key, a.value as sq_corr_0 @@ -162,12 +182,16 @@ from part where part.p_size in (select first_value(p_size) over(partition by p_mfgr order by p_size) from part) PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### POSTHOOK: query: explain rewrite select p_mfgr, p_name, p_size from part where part.p_size in (select first_value(p_size) over(partition by p_mfgr order by p_size) from part) POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### Where Clause Rewritten SubQuery: (select first_value(p_size) over(partition by p_mfgr order by p_size) from part) sq_1 @@ -185,12 +209,16 @@ from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li o where li.l_linenumber = 1 and li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR') PREHOOK: type: QUERY +PREHOOK: Input: default@lineitem +#### A masked pattern was here #### POSTHOOK: query: explain rewrite select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey where li.l_linenumber = 1 and li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR') POSTHOOK: type: QUERY +POSTHOOK: Input: default@lineitem +#### A masked pattern was here #### Where Clause Rewritten SubQuery: (select l_orderkey from lineitem where l_shipmode = 'AIR') sq_1 @@ -209,12 +237,16 @@ from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li o where li.l_linenumber = 1 and li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR' and l_linenumber = li.l_linenumber) PREHOOK: type: QUERY +PREHOOK: Input: default@lineitem +#### A masked pattern was here #### POSTHOOK: query: explain rewrite select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey where li.l_linenumber = 1 and li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR' and l_linenumber = li.l_linenumber) POSTHOOK: type: QUERY +POSTHOOK: Input: default@lineitem +#### A masked pattern was here #### Where Clause Rewritten SubQuery: (select l_orderkey, l_linenumber as sq_corr_0 from lineitem where l_shipmode = 'AIR') sq_1 -- 2.20.1