diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index e8db920..ed0852e 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -405,6 +405,7 @@ minillap.query.files=acid_bucket_pruning.q,\ bucket5.q,\ bucket6.q,\ except_distinct.q,\ + explain_formatted_user_2.q,\ explainuser_2.q,\ empty_dir_in_table.q,\ intersect_all.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java index 74cec3e..4e8d65c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java @@ -376,6 +376,7 @@ public int execute(DriverContext driverContext) { out.print(jsonDependencies); } else { if (work.isUserLevelExplain()) { + boolean isOrigFormatted = work.isFormatted(); // Because of the implementation of the JsonParserFactory, we are sure // that we can get a TezJsonParser. JsonParser jsonParser = JsonParserFactory.getParser(conf); @@ -384,15 +385,19 @@ public int execute(DriverContext driverContext) { if (work.getCboInfo() != null) { jsonPlan.put("cboInfo", work.getCboInfo()); } - try { - jsonParser.print(jsonPlan, out); - } catch (Exception e) { - // if there is anything wrong happen, we bail out. - LOG.error("Running explain user level has problem: " + e.toString() - + ". Falling back to normal explain"); - work.getConfig().setFormatted(false); - work.getConfig().setUserLevelExplain(false); - jsonPlan = getJSONPlan(out, work); + if (isOrigFormatted) { + out.print(jsonPlan); + } else { + try { + jsonParser.print(jsonPlan, out); + } catch (Exception e) { + // if there is anything wrong happen, we bail out. + LOG.error("Running explain user level has problem: " + e.toString() + + ". Falling back to normal explain"); + work.getConfig().setFormatted(false); + work.getConfig().setUserLevelExplain(false); + jsonPlan = getJSONPlan(out, work); + } } } else { JSONObject jsonPlan = getJSONPlan(out, work); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java index f62cf9a..b16840d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java @@ -183,7 +183,6 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { } config.setUserLevelExplain(!config.isExtended() - && !config.isFormatted() && !config.isDependency() && !config.isLogical() && !config.isAuthorize() diff --git a/ql/src/test/queries/clientpositive/explain_formatted_user_2.q b/ql/src/test/queries/clientpositive/explain_formatted_user_2.q new file mode 100644 index 0000000..75c85c9 --- /dev/null +++ b/ql/src/test/queries/clientpositive/explain_formatted_user_2.q @@ -0,0 +1,20 @@ +set hive.explain.user=true; + +set hive.auto.convert.join=false; + +create table srcTable (key string, value string); + +explain formatted +SELECT x.key, z.value, y.value +FROM srcTable x JOIN srcTable y ON (x.key = y.key) +JOIN srcTable z ON (x.value = z.value); + +explain formatted +SELECT x.key, z.value, y.value +FROM srcTable x JOIN srcTable y ON (x.key = y.key) +JOIN (select * from srcTable union select * from srcTable)z ON (x.value = z.value) +union +SELECT x.key, z.value, y.value +FROM srcTable x JOIN srcTable y ON (x.key = y.key) +JOIN (select * from srcTable union select * from srcTable)z ON (x.value = z.value); + diff --git a/ql/src/test/results/clientpositive/llap/explain_formatted_user_2.q.out b/ql/src/test/results/clientpositive/llap/explain_formatted_user_2.q.out new file mode 100644 index 0000000..7a52d8e --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/explain_formatted_user_2.q.out @@ -0,0 +1,38 @@ +PREHOOK: query: create table srcTable (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@srcTable +POSTHOOK: query: create table srcTable (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srcTable +PREHOOK: query: explain formatted +SELECT x.key, z.value, y.value +FROM srcTable x JOIN srcTable y ON (x.key = y.key) +JOIN srcTable z ON (x.value = z.value) +PREHOOK: type: QUERY +POSTHOOK: query: explain formatted +SELECT x.key, z.value, y.value +FROM srcTable x JOIN srcTable y ON (x.key = y.key) +JOIN srcTable z ON (x.value = z.value) +POSTHOOK: type: QUERY +{"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"Edges:":{"Reducer 2":[{"parent":"Map 1","type":"SIMPLE_EDGE"},{"parent":"Map 4","type":"SIMPLE_EDGE"}],"Reducer 3":[{"parent":"Map 5","type":"SIMPLE_EDGE"},{"parent":"Reducer 2","type":"SIMPLE_EDGE"}]},"Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"Output:":["key","value"],"":"default@srctable,x,Tbl:PARTIAL,Col:NONE","Statistics:":"rows=1 width=0","OperatorId:":"TS_0","children":{"Filter Operator":{"predicate:":"(key is not null and value is not null)","Statistics:":"rows=1 width=0","OperatorId:":"FIL_22","children":{"Select Operator":{"Output:":["_col0","_col1"],"Statistics:":"rows=1 width=0","OperatorId:":"SEL_2","children":{"Reduce Output Operator":{"outputname:":"Reducer 2","PartitionCols:":"_col0","Statistics:":"rows=1 width=0","OperatorId:":"RS_9"}}}}}}}}],"Execution mode:":"llap","tag:":"0"},"Map 4":{"Map Operator Tree:":[{"TableScan":{"Output:":["key","value"],"":"default@srctable,y,Tbl:PARTIAL,Col:NONE","Statistics:":"rows=1 width=0","OperatorId:":"TS_3","children":{"Filter Operator":{"predicate:":"key is not null","Statistics:":"rows=1 width=0","OperatorId:":"FIL_23","children":{"Select Operator":{"Output:":["_col0","_col1"],"Statistics:":"rows=1 width=0","OperatorId:":"SEL_5","children":{"Reduce Output Operator":{"outputname:":"Reducer 2","PartitionCols:":"_col0","Statistics:":"rows=1 width=0","OperatorId:":"RS_10"}}}}}}}}],"Execution mode:":"llap","tag:":"0"},"Map 5":{"Map Operator Tree:":[{"TableScan":{"Output:":["value"],"":"default@srctable,z,Tbl:PARTIAL,Col:NONE","Statistics:":"rows=1 width=0","OperatorId:":"TS_6","children":{"Filter Operator":{"predicate:":"value is not null","Statistics:":"rows=1 width=0","OperatorId:":"FIL_24","children":{"Select Operator":{"Output:":["_col0"],"Statistics:":"rows=1 width=0","OperatorId:":"SEL_8","children":{"Reduce Output Operator":{"outputname:":"Reducer 3","PartitionCols:":"_col0","Statistics:":"rows=1 width=0","OperatorId:":"RS_13"}}}}}}}}],"Execution mode:":"llap","tag:":"0"},"Reducer 2":{"Execution mode:":"llap","Reduce Operator Tree:":{"Merge Join Operator":{"condition map:":[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],"keys:":{"0":"_col0","1":"_col0"},"Output:":["_col0","_col1","_col3"],"Statistics:":"rows=1 width=0","OperatorId:":"MERGEJOIN_25","children":{"Reduce Output Operator":{"outputname:":"Reducer 3","PartitionCols:":"_col1","Statistics:":"rows=1 width=0","OperatorId:":"RS_12"}}}},"tag:":"0","tagToInput:":{"0":"Map 1","1":"Map 4"}},"Reducer 3":{"Execution mode:":"llap","Reduce Operator Tree:":{"Merge Join Operator":{"condition map:":[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],"keys:":{"0":"_col1","1":"_col0"},"Output:":["_col0","_col3","_col4"],"Statistics:":"rows=1 width=0","OperatorId:":"MERGEJOIN_26","children":{"Select Operator":{"Output:":["_col0","_col1","_col2"],"Statistics:":"rows=1 width=0","OperatorId:":"SEL_15","children":{"File Output Operator":{"Statistics:":"rows=1 width=0","OperatorId:":"FS_16"}}}}}},"tag:":"0","tagToInput:":{"0":"Reducer 2","1":"Map 5"}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1"}}},"cboInfo":"Plan optimized by CBO."} +PREHOOK: query: explain formatted +SELECT x.key, z.value, y.value +FROM srcTable x JOIN srcTable y ON (x.key = y.key) +JOIN (select * from srcTable union select * from srcTable)z ON (x.value = z.value) +union +SELECT x.key, z.value, y.value +FROM srcTable x JOIN srcTable y ON (x.key = y.key) +JOIN (select * from srcTable union select * from srcTable)z ON (x.value = z.value) +PREHOOK: type: QUERY +POSTHOOK: query: explain formatted +SELECT x.key, z.value, y.value +FROM srcTable x JOIN srcTable y ON (x.key = y.key) +JOIN (select * from srcTable union select * from srcTable)z ON (x.value = z.value) +union +SELECT x.key, z.value, y.value +FROM srcTable x JOIN srcTable y ON (x.key = y.key) +JOIN (select * from srcTable union select * from srcTable)z ON (x.value = z.value) +POSTHOOK: type: QUERY +{"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"Edges:":{"Map 10":{"parent":"Union 8","type":"CONTAINS"},"Map 15":{"parent":"Union 16","type":"CONTAINS"},"Map 18":{"parent":"Union 16","type":"CONTAINS"},"Map 7":{"parent":"Union 8","type":"CONTAINS"},"Reducer 12":[{"parent":"Map 11","type":"SIMPLE_EDGE"},{"parent":"Map 14","type":"SIMPLE_EDGE"}],"Reducer 13":[{"parent":"Reducer 12","type":"SIMPLE_EDGE"},{"parent":"Reducer 17","type":"SIMPLE_EDGE"},{"parent":"Union 4","type":"CONTAINS"}],"Reducer 17":{"parent":"Union 16","type":"SIMPLE_EDGE"},"Reducer 2":[{"parent":"Map 1","type":"SIMPLE_EDGE"},{"parent":"Map 6","type":"SIMPLE_EDGE"}],"Reducer 3":[{"parent":"Reducer 2","type":"SIMPLE_EDGE"},{"parent":"Reducer 9","type":"SIMPLE_EDGE"},{"parent":"Union 4","type":"CONTAINS"}],"Reducer 5":{"parent":"Union 4","type":"SIMPLE_EDGE"},"Reducer 9":{"parent":"Union 8","type":"SIMPLE_EDGE"}},"Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"Output:":["key","value"],"":"default@srctable,x,Tbl:PARTIAL,Col:NONE","Statistics:":"rows=1 width=0","OperatorId:":"TS_0","children":{"Filter Operator":{"predicate:":"(key is not null and value is not null)","Statistics:":"rows=1 width=0","OperatorId:":"FIL_72","children":{"Select Operator":{"Output:":["_col0","_col1"],"Statistics:":"rows=1 width=0","OperatorId:":"SEL_2","children":{"Reduce Output Operator":{"outputname:":"Reducer 2","PartitionCols:":"_col0","Statistics:":"rows=1 width=0","OperatorId:":"RS_19"}}}}}}}}],"Execution mode:":"llap","tag:":"0"},"Map 10":{"Map Operator Tree:":[{"TableScan":{"Output:":["key","value"],"Statistics:":"rows=1 width=0","OperatorId:":"TS_9","children":{"Filter Operator":{"predicate:":"value is not null","Statistics:":"rows=1 width=0","OperatorId:":"FIL_75","children":{"Select Operator":{"Output:":["_col0","_col1"],"Statistics:":"rows=1 width=0","OperatorId:":"SEL_11","children":{"Group By Operator":{"keys:":"_col1, _col0","Output:":["_col0","_col1"],"Statistics:":"rows=2 width=0","OperatorId:":"GBY_15","children":{"Reduce Output Operator":{"outputname:":"Reducer 9","PartitionCols:":"_col0, _col1","Statistics:":"rows=2 width=0","OperatorId:":"RS_16"}}}}}}}}}}],"Execution mode:":"llap","tag:":"0"},"Map 11":{"Map Operator Tree:":[{"TableScan":{"Output:":["key","value"],"":"default@srctable,x,Tbl:PARTIAL,Col:NONE","Statistics:":"rows=1 width=0","OperatorId:":"TS_26","children":{"Filter Operator":{"predicate:":"(key is not null and value is not null)","Statistics:":"rows=1 width=0","OperatorId:":"FIL_76","children":{"Select Operator":{"Output:":["_col0","_col1"],"Statistics:":"rows=1 width=0","OperatorId:":"SEL_28","children":{"Reduce Output Operator":{"outputname:":"Reducer 12","PartitionCols:":"_col0","Statistics:":"rows=1 width=0","OperatorId:":"RS_45"}}}}}}}}],"Execution mode:":"llap","tag:":"0"},"Map 14":{"Map Operator Tree:":[{"TableScan":{"Output:":["key"],"":"default@srctable,y,Tbl:PARTIAL,Col:NONE","Statistics:":"rows=1 width=0","OperatorId:":"TS_29","children":{"Filter Operator":{"predicate:":"key is not null","Statistics:":"rows=1 width=0","OperatorId:":"FIL_77","children":{"Select Operator":{"Output:":["_col0"],"Statistics:":"rows=1 width=0","OperatorId:":"SEL_31","children":{"Reduce Output Operator":{"outputname:":"Reducer 12","PartitionCols:":"_col0","Statistics:":"rows=1 width=0","OperatorId:":"RS_46"}}}}}}}}],"Execution mode:":"llap","tag:":"0"},"Map 15":{"Map Operator Tree:":[{"TableScan":{"Output:":["key","value"],"Statistics:":"rows=1 width=0","OperatorId:":"TS_32","children":{"Filter Operator":{"predicate:":"value is not null","Statistics:":"rows=1 width=0","OperatorId:":"FIL_78","children":{"Select Operator":{"Output:":["_col0","_col1"],"Statistics:":"rows=1 width=0","OperatorId:":"SEL_34","children":{"Group By Operator":{"keys:":"_col1, _col0","Output:":["_col0","_col1"],"Statistics:":"rows=2 width=0","OperatorId:":"GBY_41","children":{"Reduce Output Operator":{"outputname:":"Reducer 17","PartitionCols:":"_col0, _col1","Statistics:":"rows=2 width=0","OperatorId:":"RS_42"}}}}}}}}}}],"Execution mode:":"llap","tag:":"0"},"Map 18":{"Map Operator Tree:":[{"TableScan":{"Output:":["key","value"],"Statistics:":"rows=1 width=0","OperatorId:":"TS_35","children":{"Filter Operator":{"predicate:":"value is not null","Statistics:":"rows=1 width=0","OperatorId:":"FIL_79","children":{"Select Operator":{"Output:":["_col0","_col1"],"Statistics:":"rows=1 width=0","OperatorId:":"SEL_37","children":{"Group By Operator":{"keys:":"_col1, _col0","Output:":["_col0","_col1"],"Statistics:":"rows=2 width=0","OperatorId:":"GBY_41","children":{"Reduce Output Operator":{"outputname:":"Reducer 17","PartitionCols:":"_col0, _col1","Statistics:":"rows=2 width=0","OperatorId:":"RS_42"}}}}}}}}}}],"Execution mode:":"llap","tag:":"0"},"Map 6":{"Map Operator Tree:":[{"TableScan":{"Output:":["key"],"":"default@srctable,y,Tbl:PARTIAL,Col:NONE","Statistics:":"rows=1 width=0","OperatorId:":"TS_3","children":{"Filter Operator":{"predicate:":"key is not null","Statistics:":"rows=1 width=0","OperatorId:":"FIL_73","children":{"Select Operator":{"Output:":["_col0"],"Statistics:":"rows=1 width=0","OperatorId:":"SEL_5","children":{"Reduce Output Operator":{"outputname:":"Reducer 2","PartitionCols:":"_col0","Statistics:":"rows=1 width=0","OperatorId:":"RS_20"}}}}}}}}],"Execution mode:":"llap","tag:":"0"},"Map 7":{"Map Operator Tree:":[{"TableScan":{"Output:":["key","value"],"Statistics:":"rows=1 width=0","OperatorId:":"TS_6","children":{"Filter Operator":{"predicate:":"value is not null","Statistics:":"rows=1 width=0","OperatorId:":"FIL_74","children":{"Select Operator":{"Output:":["_col0","_col1"],"Statistics:":"rows=1 width=0","OperatorId:":"SEL_8","children":{"Group By Operator":{"keys:":"_col1, _col0","Output:":["_col0","_col1"],"Statistics:":"rows=2 width=0","OperatorId:":"GBY_15","children":{"Reduce Output Operator":{"outputname:":"Reducer 9","PartitionCols:":"_col0, _col1","Statistics:":"rows=2 width=0","OperatorId:":"RS_16"}}}}}}}}}}],"Execution mode:":"llap","tag:":"0"},"Reducer 12":{"Execution mode:":"llap","Reduce Operator Tree:":{"Merge Join Operator":{"condition map:":[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],"keys:":{"0":"_col0","1":"_col0"},"Output:":["_col0","_col1"],"Statistics:":"rows=1 width=0","OperatorId:":"MERGEJOIN_81","children":{"Reduce Output Operator":{"outputname:":"Reducer 13","PartitionCols:":"_col1","Statistics:":"rows=1 width=0","OperatorId:":"RS_48"}}}},"tag:":"0","tagToInput:":{"0":"Map 11","1":"Map 14"}},"Reducer 13":{"Execution mode:":"llap","Reduce Operator Tree:":{"Merge Join Operator":{"condition map:":[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],"keys:":{"0":"_col1","1":"_col1"},"Output:":["_col0","_col4"],"Statistics:":"rows=1 width=0","OperatorId:":"MERGEJOIN_83","children":{"Select Operator":{"Output:":["_col0","_col1"],"Statistics:":"rows=1 width=0","OperatorId:":"SEL_51","children":{"Group By Operator":{"keys:":"_col0, _col1","Output:":["_col0","_col1"],"Statistics:":"rows=2 width=0","OperatorId:":"GBY_55","children":{"Reduce Output Operator":{"outputname:":"Reducer 5","PartitionCols:":"_col0, _col1","Statistics:":"rows=2 width=0","OperatorId:":"RS_56"}}}}}}}},"tag:":"0","tagToInput:":{"0":"Reducer 12","1":"Reducer 17"}},"Reducer 17":{"Execution mode:":"llap","Reduce Operator Tree:":{"Group By Operator":{"keys:":"KEY._col0, KEY._col1","Output:":["_col0","_col1"],"Statistics:":"rows=1 width=0","OperatorId:":"GBY_43","children":{"Select Operator":{"Output:":["_col1"],"Statistics:":"rows=1 width=0","OperatorId:":"SEL_44","children":{"Reduce Output Operator":{"outputname:":"Reducer 13","PartitionCols:":"_col1","Statistics:":"rows=1 width=0","OperatorId:":"RS_49"}}}}}},"tag:":"0","tagToInput:":{"0":"Union 16"}},"Reducer 2":{"Execution mode:":"llap","Reduce Operator Tree:":{"Merge Join Operator":{"condition map:":[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],"keys:":{"0":"_col0","1":"_col0"},"Output:":["_col0","_col1"],"Statistics:":"rows=1 width=0","OperatorId:":"MERGEJOIN_80","children":{"Reduce Output Operator":{"outputname:":"Reducer 3","PartitionCols:":"_col1","Statistics:":"rows=1 width=0","OperatorId:":"RS_22"}}}},"tag:":"0","tagToInput:":{"0":"Map 1","1":"Map 6"}},"Reducer 3":{"Execution mode:":"llap","Reduce Operator Tree:":{"Merge Join Operator":{"condition map:":[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],"keys:":{"0":"_col1","1":"_col1"},"Output:":["_col0","_col4"],"Statistics:":"rows=1 width=0","OperatorId:":"MERGEJOIN_82","children":{"Select Operator":{"Output:":["_col0","_col1"],"Statistics:":"rows=1 width=0","OperatorId:":"SEL_25","children":{"Group By Operator":{"keys:":"_col0, _col1","Output:":["_col0","_col1"],"Statistics:":"rows=2 width=0","OperatorId:":"GBY_55","children":{"Reduce Output Operator":{"outputname:":"Reducer 5","PartitionCols:":"_col0, _col1","Statistics:":"rows=2 width=0","OperatorId:":"RS_56"}}}}}}}},"tag:":"0","tagToInput:":{"0":"Reducer 2","1":"Reducer 9"}},"Reducer 5":{"Execution mode:":"llap","Reduce Operator Tree:":{"Group By Operator":{"keys:":"KEY._col0, KEY._col1","Output:":["_col0","_col1"],"Statistics:":"rows=1 width=0","OperatorId:":"GBY_57","children":{"File Output Operator":{"Statistics:":"rows=1 width=0","OperatorId:":"FS_59"}}}},"tag:":"0","tagToInput:":{"0":"Union 4"}},"Reducer 9":{"Execution mode:":"llap","Reduce Operator Tree:":{"Group By Operator":{"keys:":"KEY._col0, KEY._col1","Output:":["_col0","_col1"],"Statistics:":"rows=1 width=0","OperatorId:":"GBY_17","children":{"Select Operator":{"Output:":["_col1"],"Statistics:":"rows=1 width=0","OperatorId:":"SEL_18","children":{"Reduce Output Operator":{"outputname:":"Reducer 3","PartitionCols:":"_col1","Statistics:":"rows=1 width=0","OperatorId:":"RS_23"}}}}}},"tag:":"0","tagToInput:":{"0":"Union 8"}},"Union 16":{"Vertex:":"Union 16","tag:":"0"},"Union 4":{"Vertex:":"Union 4","tag:":"0"},"Union 8":{"Vertex:":"Union 8","tag:":"0"}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1"}}},"cboInfo":"Plan optimized by CBO."}