Index: ql/src/test/results/clientpositive/input4.q.out =================================================================== --- ql/src/test/results/clientpositive/input4.q.out (revision 1197403) +++ ql/src/test/results/clientpositive/input4.q.out (working copy) @@ -19,8 +19,8 @@ STAGE PLANS: Stage: Stage-0 Copy - source: file:/Users/krishnak/Projects/hdp/sources/hive-git-apache/data/files/kv1.txt - destination: pfile:/Users/krishnak/Projects/hdp/sources/hive-git-apache/build/ql/scratchdir/hive_2011-03-20_23-49-16_008_2588436210508896099/-ext-10000 + source: file:/data/users/njain/hive_commit1/data/files/kv1.txt + destination: pfile:/data/users/njain/hive_commit1/build/ql/scratchdir/hive_2011-11-03_16-47-14_258_1757057361379982466/-ext-10000 Stage: Stage-1 Move Operator @@ -39,14 +39,21 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/kv1.txt' INTO TABLE INPUT4 POSTHOOK: type: LOAD POSTHOOK: Output: default@input4 +PREHOOK: query: EXPLAIN FORMATTED +SELECT INPUT4.VALUE, INPUT4.KEY FROM INPUT4 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN FORMATTED +SELECT INPUT4.VALUE, INPUT4.KEY FROM INPUT4 +POSTHOOK: type: QUERY +{"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Alias -> Map Operator Tree:":{"input4":{"TS_0":{"SEL_1":{"FS_2":{"File Output Operator":{"GlobalTableId:":"0","compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}}}}}}},"Percentage Sample:":{}}},"Stage-0":{"Fetch Operator":{"limit:":"-1"}}},"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"ROOT STAGE":"TRUE"}},"ABSTRACT SYNTAX TREE":"(TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME INPUT4))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL INPUT4) VALUE)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL INPUT4) KEY)))))"} PREHOOK: query: SELECT INPUT4.VALUE, INPUT4.KEY FROM INPUT4 PREHOOK: type: QUERY PREHOOK: Input: default@input4 -PREHOOK: Output: file:/var/folders/67/67R3POPtF90VG63KSmCbcU++F0U/-Tmp-/krishnak/hive_2011-03-20_23-49-17_477_5960644229997823855/-mr-10000 +PREHOOK: Output: file:/tmp/njain/hive_2011-11-03_16-47-15_010_2383066747887433640/-mr-10000 POSTHOOK: query: SELECT INPUT4.VALUE, INPUT4.KEY FROM INPUT4 POSTHOOK: type: QUERY POSTHOOK: Input: default@input4 -POSTHOOK: Output: file:/var/folders/67/67R3POPtF90VG63KSmCbcU++F0U/-Tmp-/krishnak/hive_2011-03-20_23-49-17_477_5960644229997823855/-mr-10000 +POSTHOOK: Output: file:/tmp/njain/hive_2011-11-03_16-47-15_010_2383066747887433640/-mr-10000 val_238 238 val_86 86 val_311 311 Index: ql/src/test/results/clientpositive/join0.q.out =================================================================== --- ql/src/test/results/clientpositive/join0.q.out (revision 1197403) +++ ql/src/test/results/clientpositive/join0.q.out (working copy) @@ -100,7 +100,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/sdong/hive_2011-02-10_16-01-47_793_8777845351054616084/-mr-10002 + file:/var/folders/bZ/bZe+iKfoFTuPoShRd6dy6-tOU9Y/-Tmp-/njain/hive_2011-11-03_16-13-27_794_5039563178190087904/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -136,6 +136,23 @@ limit: -1 +PREHOOK: query: EXPLAIN FORMATTED +SELECT src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (SELECT * FROM src WHERE src.key < 10) src1 + JOIN + (SELECT * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN FORMATTED +SELECT src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (SELECT * FROM src WHERE src.key < 10) src1 + JOIN + (SELECT * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2 +POSTHOOK: type: QUERY +{"STAGE PLANS":{"Stage-2":{"Map Reduce":{"Reduce Operator Tree:":{"OP_11":{"FS_12":{"File Output Operator":{"GlobalTableId:":"0","compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}}}}},"Alias -> Map Operator Tree:":{"file:/var/folders/bZ/bZe+iKfoFTuPoShRd6dy6-tOU9Y/-Tmp-/njain/hive_2011-11-03_16-13-28_171_2888498791907064539/-mr-10002":{"TS_16":{"RS_10":{"Reduce Output Operator":{"Map-reduce partition columns:":"","sort order:":"++++","tag:":"-1","value expressions:":"{\"type:\":\"string\",\"expr:\":\"_col0\"}, {\"type:\":\"string\",\"expr:\":\"_col1\"}, {\"type:\":\"string\",\"expr:\":\"_col2\"}, {\"type:\":\"string\",\"expr:\":\"_col3\"}","key expressions:":"{\"type:\":\"string\",\"expr:\":\"_col0\"}, {\"type:\":\"string\",\"expr:\":\"_col1\"}, {\"type:\":\"string\",\"expr:\":\"_col2\"}, {\"type:\":\"string\",\"expr:\":\"_col3\"}"}}}}},"Percentage Sample:":{}}},"Stage-1":{"Map Reduce":{"Reduce Operator Tree:":{"JOIN_8":{"SEL_9":{"FS_15":{"File Output Operator":{"GlobalTableId:":"0","compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat"}}}}}},"Alias -> Map Operator Tree:":{"src2:src":{"TS_0":{"FIL_13":{"SEL_2":{"RS_7":{"Reduce Output Operator":{"Map-reduce partition columns:":"","sort order:":"","tag:":"1","value expressions:":"{\"type:\":\"string\",\"expr:\":\"_col0\"}, {\"type:\":\"string\",\"expr:\":\"_col1\"}","key expressions:":""}}}}}},"src1:src":{"TS_3":{"FIL_14":{"SEL_5":{"RS_6":{"Reduce Output Operator":{"Map-reduce partition columns:":"","sort order:":"","tag:":"0","value expressions:":"{\"type:\":\"string\",\"expr:\":\"_col0\"}, {\"type:\":\"string\",\"expr:\":\"_col1\"}","key expressions:":""}}}}}}},"Percentage Sample:":{}}},"Stage-0":{"Fetch Operator":{"limit:":"-1"}}},"STAGE DEPENDENCIES":{"Stage-2":{"DEPENDENT STAGES":"Stage-1"},"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"ROOT STAGE":"TRUE"}},"ABSTRACT SYNTAX TREE":"(TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (< (. (TOK_TABLE_OR_COL src) key) 10)))) src1) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (< (. (TOK_TABLE_OR_COL src) key) 10)))) src2))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) key) k1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) value) v1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) key) k2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) value) v2)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL k1)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL v1)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL k2)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL v2)))))"} PREHOOK: query: SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 FROM (SELECT * FROM src WHERE src.key < 10) src1 @@ -144,7 +161,7 @@ SORT BY k1, v1, k2, v2 PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_16-01-47_898_1006010715784348203/-mr-10000 +PREHOOK: Output: file:/var/folders/bZ/bZe+iKfoFTuPoShRd6dy6-tOU9Y/-Tmp-/njain/hive_2011-11-03_16-13-28_323_1828259293437597011/-mr-10000 POSTHOOK: query: SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 FROM (SELECT * FROM src WHERE src.key < 10) src1 @@ -153,7 +170,7 @@ SORT BY k1, v1, k2, v2 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_16-01-47_898_1006010715784348203/-mr-10000 +POSTHOOK: Output: file:/var/folders/bZ/bZe+iKfoFTuPoShRd6dy6-tOU9Y/-Tmp-/njain/hive_2011-11-03_16-13-28_323_1828259293437597011/-mr-10000 0 val_0 0 val_0 0 val_0 0 val_0 0 val_0 0 val_0 Index: ql/src/test/queries/clientpositive/input4.q =================================================================== --- ql/src/test/queries/clientpositive/input4.q (revision 1197403) +++ ql/src/test/queries/clientpositive/input4.q (working copy) @@ -2,5 +2,7 @@ EXPLAIN LOAD DATA LOCAL INPATH '../data/files/kv1.txt' INTO TABLE INPUT4; LOAD DATA LOCAL INPATH '../data/files/kv1.txt' INTO TABLE INPUT4; +EXPLAIN FORMATTED SELECT INPUT4.VALUE, INPUT4.KEY FROM INPUT4; +SELECT INPUT4.VALUE, INPUT4.KEY FROM INPUT4; Index: ql/src/test/queries/clientpositive/join0.q =================================================================== --- ql/src/test/queries/clientpositive/join0.q (revision 1197403) +++ ql/src/test/queries/clientpositive/join0.q (working copy) @@ -6,6 +6,7 @@ (SELECT * FROM src WHERE src.key < 10) src2 SORT BY k1, v1, k2, v2; +EXPLAIN FORMATTED SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 FROM (SELECT * FROM src WHERE src.key < 10) src1 @@ -13,3 +14,10 @@ (SELECT * FROM src WHERE src.key < 10) src2 SORT BY k1, v1, k2, v2; +SELECT src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (SELECT * FROM src WHERE src.key < 10) src1 + JOIN + (SELECT * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2; + Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java (revision 1197403) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java (working copy) @@ -40,8 +40,9 @@ import org.apache.hadoop.hive.ql.plan.api.StageType; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.StringUtils; +import org.json.JSONException; +import org.json.JSONObject; - /** * ExplainTask implementation. * @@ -53,6 +54,47 @@ super(); } + static public JSONObject getJSONPlan(PrintStream out, ExplainWork work) + throws Exception { + // If the user asked for a formatted output, dump the json output + // in the output stream + JSONObject outJSONObject = new JSONObject(); + boolean jsonOutput = work.isFormatted(); + if (jsonOutput) { + out = null; + } + + // Print out the parse AST + String jsonAST = outputAST(work.getAstStringTree(), out, jsonOutput, 0); + if (out != null) { + out.println(); + } + + if (jsonOutput) { + outJSONObject.put("ABSTRACT SYNTAX TREE", jsonAST); + } + + JSONObject jsonDependencies = outputDependencies(out, jsonOutput, + work.getRootTasks(), 0); + + if (out != null) { + out.println(); + } + + if (jsonOutput) { + outJSONObject.put("STAGE DEPENDENCIES", jsonDependencies); + } + + // Go over all the tasks and dump out the plans + JSONObject jsonPlan = outputStagePlans(out, work, work.getRootTasks(), 0); + + if (jsonOutput) { + outJSONObject.put("STAGE PLANS", jsonPlan); + } + + return jsonOutput ? outJSONObject : null; + } + @Override public int execute(DriverContext driverContext) { @@ -62,29 +104,27 @@ OutputStream outS = resFile.getFileSystem(conf).create(resFile); out = new PrintStream(outS); - // Print out the parse AST - outputAST(work.getAstStringTree(), out, 0); - out.println(); + JSONObject jsonPlan = getJSONPlan(out, work); - outputDependencies(out, work.getRootTasks(), 0); - out.println(); + if (work.isFormatted()) { + out.print(jsonPlan); + } - // Go over all the tasks and dump out the plans - outputStagePlans(out, work.getRootTasks(), 0); out.close(); out = null; - return (0); - } catch (Exception e) { - console.printError("Failed with exception " + e.getMessage(), "\n" - + StringUtils.stringifyException(e)); + } + catch (Exception e) { + console.printError("Failed with exception " + e.getMessage(), + "\n" + StringUtils.stringifyException(e)); return (1); - } finally { + } + finally { IOUtils.closeStream(out); } } - private String indentString(int indent) { + private static String indentString(int indent) { StringBuilder sb = new StringBuilder(); for (int i = 0; i < indent; ++i) { sb.append(" "); @@ -93,73 +133,112 @@ return sb.toString(); } - private void outputMap(Map mp, String header, PrintStream out, - boolean extended, int indent) throws Exception { + private static JSONObject outputMap(Map mp, String header, PrintStream out, + boolean extended, boolean jsonOutput, int indent) throws Exception { boolean first_el = true; TreeMap tree = new TreeMap(); tree.putAll(mp); + JSONObject json = jsonOutput ? new JSONObject() : null; for (Entry ent : tree.entrySet()) { - if (first_el) { + if (first_el && (out != null)) { out.println(header); } first_el = false; // Print the key - out.print(indentString(indent)); - out.printf("%s ", ent.getKey().toString()); + if (out != null) { + out.print(indentString(indent)); + out.printf("%s ", ent.getKey().toString()); + } + // Print the value if (isPrintable(ent.getValue())) { - out.print(ent.getValue()); - out.println(); - } else if (ent.getValue() instanceof List - || ent.getValue() instanceof Map) { - out.print(ent.getValue().toString()); - out.println(); - } else if (ent.getValue() instanceof Serializable) { - out.println(); - outputPlan((Serializable) ent.getValue(), out, extended, indent + 2); - } else { - out.println(); + if (out != null) { + out.print(ent.getValue()); + out.println(); + } + if (jsonOutput) { + json.put(ent.getKey().toString(), ent.getValue().toString()); + } } + else if (ent.getValue() instanceof List || ent.getValue() instanceof Map) { + if (out != null) { + out.print(ent.getValue().toString()); + out.println(); + } + if (jsonOutput) { + json.put(ent.getKey().toString(), ent.getValue().toString()); + } + } + else if (ent.getValue() instanceof Serializable) { + if (out != null) { + out.println(); + } + JSONObject jsonOut = outputPlan((Serializable) ent.getValue(), out, + extended, jsonOutput, jsonOutput ? 0 : indent + 2); + if (jsonOutput) { + json.put(ent.getKey().toString(), jsonOut); + } + } + else { + if (out != null) { + out.println(); + } + } } + + return jsonOutput ? json : null; } - private void outputList(List l, String header, PrintStream out, - boolean extended, int indent) throws Exception { + private static String outputList(List l, String header, PrintStream out, + boolean extended, boolean jsonOutput, int indent) throws Exception { boolean first_el = true; boolean nl = false; + StringBuffer s = new StringBuffer(); for (Object o : l) { - if (first_el) { + if (first_el && (out != null)) { out.print(header); } if (isPrintable(o)) { - if (!first_el) { - out.print(", "); - } else { - out.print(" "); + String delim = first_el ? " " : ", "; + if (out != null) { + out.print(delim); + out.print(o); } - out.print(o); + if (jsonOutput) { + s.append(delim); + s.append(o); + } nl = true; - } else if (o instanceof Serializable) { - if (first_el) { + } + else if (o instanceof Serializable) { + if (first_el && (out != null)) { out.println(); } - outputPlan((Serializable) o, out, extended, indent + 2); + JSONObject jsonOut = outputPlan((Serializable) o, out, extended, + jsonOutput, jsonOutput ? 0 : indent + 2); + if (jsonOutput) { + if (!first_el) { + s.append(", "); + } + s.append(jsonOut); + } } first_el = false; } - if (nl) { + if (nl && (out != null)) { out.println(); } + return jsonOutput ? s.toString() : null; } - private boolean isPrintable(Object val) { + private static boolean isPrintable(Object val) { if (val instanceof Boolean || val instanceof String || val instanceof Integer || val instanceof Byte || val instanceof Float || val instanceof Double) { @@ -173,33 +252,56 @@ return false; } - private void outputPlan(Serializable work, PrintStream out, boolean extended, - int indent) throws Exception { + private static JSONObject outputPlan(Serializable work, PrintStream out, + boolean extended, boolean jsonOutput, int indent) throws Exception { // Check if work has an explain annotation Annotation note = work.getClass().getAnnotation(Explain.class); + String keyJSONObject = null; + if (note instanceof Explain) { Explain xpl_note = (Explain) note; if (extended || xpl_note.normalExplain()) { - out.print(indentString(indent)); - out.println(xpl_note.displayName()); + keyJSONObject = xpl_note.displayName(); + if (out != null) { + out.print(indentString(indent)); + out.println(xpl_note.displayName()); + } } } + JSONObject json = jsonOutput ? new JSONObject() : null; // If this is an operator then we need to call the plan generation on the - // conf and then - // the children + // conf and then the children if (work instanceof Operator) { Operator operator = (Operator) work; if (operator.getConf() != null) { - outputPlan(operator.getConf(), out, extended, indent); + JSONObject jsonOut = outputPlan(operator.getConf(), out, extended, + jsonOutput, jsonOutput ? 0 : indent); + if (jsonOutput) { + json.put(operator.getOperatorId(), jsonOut); + } } + if (operator.getChildOperators() != null) { for (Operator op : operator.getChildOperators()) { - outputPlan(op, out, extended, indent + 2); + JSONObject jsonOut = outputPlan(op, out, extended, jsonOutput, jsonOutput ? 0 : indent + 2); + if (jsonOutput) { + json.put(operator.getOperatorId(), jsonOut); + } } } - return; + + if (jsonOutput) { + if (keyJSONObject != null) { + JSONObject ret = new JSONObject(); + ret.put(keyJSONObject, json); + return ret; + } + + return json; + } + return null; } // We look at all methods that generate values for explain @@ -207,7 +309,7 @@ Arrays.sort(methods, new MethodComparator()); for (Method m : methods) { - int prop_indents = indent + 2; + int prop_indents = jsonOutput ? 0 : indent + 2; note = m.getAnnotation(Explain.class); if (note instanceof Explain) { @@ -224,179 +326,286 @@ String header = null; if (!xpl_note.displayName().equals("")) { header = indentString(prop_indents) + xpl_note.displayName() + ":"; - } else { + } + else { prop_indents = indent; header = indentString(prop_indents); } if (isPrintable(val)) { - - out.printf("%s ", header); - out.println(val); + if (out != null) { + out.printf("%s ", header); + out.println(val); + } + if (jsonOutput) { + json.put(header, val.toString()); + } continue; } // Try this as a map try { // Go through the map and print out the stuff Map mp = (Map) val; - outputMap(mp, header, out, extended, prop_indents + 2); + JSONObject jsonOut = outputMap(mp, header, out, extended, jsonOutput, + jsonOutput ? 0 : prop_indents + 2); + if (jsonOutput) { + json.put(header, jsonOut); + } continue; - } catch (ClassCastException ce) { + } + catch (ClassCastException ce) { // Ignore - all this means is that this is not a map } // Try this as a list try { List l = (List) val; - outputList(l, header, out, extended, prop_indents + 2); - + String jsonOut = outputList(l, header, out, extended, jsonOutput, + jsonOutput ? 0 : prop_indents + 2); + if (jsonOutput) { + json.put(header, jsonOut); + } continue; - } catch (ClassCastException ce) { + } + catch (ClassCastException ce) { // Ignore } // Finally check if it is serializable try { Serializable s = (Serializable) val; - out.println(header); - outputPlan(s, out, extended, prop_indents + 2); - + if (out != null) { + out.println(header); + } + JSONObject jsonOut = outputPlan(s, out, extended, jsonOutput, + jsonOutput ? 0 : prop_indents + 2); + if (jsonOutput) { + json.put(header, jsonOut); + } continue; - } catch (ClassCastException ce) { + } + catch (ClassCastException ce) { // Ignore } } } } + + if (jsonOutput) { + if (keyJSONObject != null) { + JSONObject ret = new JSONObject(); + ret.put(keyJSONObject, json); + return ret; + } + + return json; + } + + return null; } - private void outputPlan(Task task, PrintStream out, - boolean extended, HashSet> displayedSet, + private static JSONObject outputPlan(Task task, + PrintStream out, JSONObject parentJSON, boolean extended, + boolean jsonOutput, HashSet> displayedSet, int indent) throws Exception { if (displayedSet.contains(task)) { - return; + return null; } displayedSet.add(task); - out.print(indentString(indent)); - out.printf("Stage: %s\n", task.getId()); + if (out != null) { + out.print(indentString(indent)); + out.printf("Stage: %s\n", task.getId()); + } + // Start by getting the work part of the task and call the output plan for // the work - outputPlan(task.getWork(), out, extended, indent + 2); - out.println(); + JSONObject jsonOutputPlan = outputPlan(task.getWork(), out, extended, + jsonOutput, jsonOutput ? 0 : indent + 2); + + if (out != null) { + out.println(); + } + + if (jsonOutput) { + parentJSON.put(task.getId(), jsonOutputPlan); + } + if (task instanceof ConditionalTask && ((ConditionalTask) task).getListTasks() != null) { - for (Task con : ((ConditionalTask) task) - .getListTasks()) { - outputPlan(con, out, extended, displayedSet, indent); + for (Task con : ((ConditionalTask) task).getListTasks()) { + outputPlan(con, out, parentJSON, extended, jsonOutput, displayedSet, + jsonOutput ? 0 : indent); } } if (task.getChildTasks() != null) { for (Task child : task.getChildTasks()) { - outputPlan(child, out, extended, displayedSet, indent); + outputPlan(child, out, parentJSON, extended, jsonOutput, displayedSet, + jsonOutput ? 0 : indent); } } + return null; } - private final Set> dependeciesTaskSet = - new HashSet>(); + private static JSONObject outputDependencies(Task task, + Set> dependeciesTaskSet, PrintStream out, + JSONObject parentJson, boolean jsonOutput, int indent, + boolean rootTskCandidate) throws Exception { - private void outputDependencies(Task task, - PrintStream out, int indent, boolean rootTskCandidate) throws Exception { - if (dependeciesTaskSet.contains(task)) { - return; + return null; } dependeciesTaskSet.add(task); boolean first = true; - out.print(indentString(indent)); - out.printf("%s", task.getId()); + JSONObject json = jsonOutput ? new JSONObject() : null; + if (out != null) { + out.print(indentString(indent)); + out.printf("%s", task.getId()); + } + if ((task.getParentTasks() == null || task.getParentTasks().isEmpty())) { if (rootTskCandidate) { - out.print(" is a root stage"); + if (out != null) { + out.print(" is a root stage"); + } + + if (jsonOutput) { + json.put("ROOT STAGE", "TRUE"); + } } - } else { - out.print(" depends on stages: "); + } + else { + StringBuffer s = new StringBuffer(); first = true; for (Task parent : task.getParentTasks()) { if (!first) { - out.print(", "); + s.append(", "); } first = false; - out.print(parent.getId()); + s.append(parent.getId()); } + + if (out != null) { + out.print(" depends on stages: "); + out.print(s.toString()); + } + if (jsonOutput) { + json.put("DEPENDENT STAGES", s.toString()); + } } - Task cuurBackupTask = task.getBackupTask(); - if(cuurBackupTask != null) { - out.print(" has a backup stage: "); - if (!first) { - out.print(", "); + Task currBackupTask = task.getBackupTask(); + if (currBackupTask != null) { + if (out != null) { + out.print(" has a backup stage: "); + out.print(currBackupTask.getId()); } - first = false; - out.print(cuurBackupTask.getId()); + if (jsonOutput) { + json.put("BACKUP STAGE", currBackupTask.getId()); + } } - if (task instanceof ConditionalTask && ((ConditionalTask) task).getListTasks() != null) { - out.print(" , consists of "); + StringBuffer s = new StringBuffer(); first = true; - for (Task con : ((ConditionalTask) task) - .getListTasks()) { + for (Task con : ((ConditionalTask) task).getListTasks()) { if (!first) { - out.print(", "); + s.append(", "); } first = false; - out.print(con.getId()); + s.append(con.getId()); } + + if (out != null) { + out.print(" , consists of "); + out.print(s.toString()); + } + if (jsonOutput) { + json.put("CONDITIONAL CHILD TASKS", s.toString()); + } } + if (out != null) { + out.println(); + } - out.println(); - if (task instanceof ConditionalTask && ((ConditionalTask) task).getListTasks() != null) { - for (Task con : ((ConditionalTask) task) - .getListTasks()) { - outputDependencies(con, out, indent, false); + for (Task con : ((ConditionalTask) task).getListTasks()) { + JSONObject jsonOut = outputDependencies(con, dependeciesTaskSet, out, + parentJson, jsonOutput, jsonOutput ? 0 : indent, false); + if (jsonOutput && (jsonOut != null)) { + parentJson.put(con.getId(), jsonOut); + } } } if (task.getChildTasks() != null) { for (Task child : task.getChildTasks()) { - outputDependencies(child, out, indent, true); + JSONObject jsonOut = outputDependencies(child, dependeciesTaskSet, out, + parentJson, jsonOutput, jsonOutput ? 0 : indent, true); + if (jsonOutput && (jsonOut != null)) { + parentJson.put(child.getId(), jsonOut); + } } } - + return jsonOutput ? json : null; } - public void outputAST(String treeString, PrintStream out, int indent) { - out.print(indentString(indent)); - out.println("ABSTRACT SYNTAX TREE:"); - out.print(indentString(indent + 2)); - out.println(treeString); + public static String outputAST(String treeString, PrintStream out, + boolean jsonOutput, int indent) throws JSONException { + if (out != null) { + out.print(indentString(indent)); + out.println("ABSTRACT SYNTAX TREE:"); + out.print(indentString(indent + 2)); + out.println(treeString); + } + + return jsonOutput ? treeString : null; } - public void outputDependencies(PrintStream out, - List> rootTasks, int indent) throws Exception { - out.print(indentString(indent)); - out.println("STAGE DEPENDENCIES:"); + public static JSONObject outputDependencies(PrintStream out, boolean jsonOutput, + List> rootTasks, int indent) + throws Exception { + if (out != null) { + out.print(indentString(indent)); + out.println("STAGE DEPENDENCIES:"); + } + + JSONObject json = jsonOutput ? new JSONObject() : null; + Set> dependenciesTaskSet = + new HashSet>(); + for (Task rootTask : rootTasks) { - outputDependencies(rootTask, out, indent + 2, true); + JSONObject jsonOut = outputDependencies(rootTask, + dependenciesTaskSet, out, json, jsonOutput, + jsonOutput ? 0 : indent + 2, true); + if (jsonOutput && (jsonOut != null)) { + json.put(rootTask.getId(), jsonOut); + } } + + return jsonOutput ? json : null; } - public void outputStagePlans(PrintStream out, - List> rootTasks, int indent) throws Exception { - out.print(indentString(indent)); - out.println("STAGE PLANS:"); - HashSet> displayedSet = - new HashSet>(); + public static JSONObject outputStagePlans(PrintStream out, ExplainWork work, + List> rootTasks, int indent) + throws Exception { + boolean jsonOutput = work.isFormatted(); + if (out != null) { + out.print(indentString(indent)); + out.println("STAGE PLANS:"); + } + + JSONObject json = jsonOutput ? new JSONObject() : null; + HashSet> displayedSet = new HashSet>(); for (Task rootTask : rootTasks) { - outputPlan(rootTask, out, work.getExtended(), displayedSet, indent + 2); + outputPlan(rootTask, out, json, work.getExtended(), jsonOutput, + displayedSet, jsonOutput ? 0 : indent + 2); } + return jsonOutput ? json : null; } /** @@ -425,7 +634,7 @@ protected void localizeMRTmpFilesImpl(Context ctx) { // explain task has nothing to localize // we don't expect to enter this code path at all - throw new RuntimeException ("Unexpected call"); + throw new RuntimeException("Unexpected call"); } } Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java (revision 1197403) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java (working copy) @@ -35,17 +35,21 @@ private ArrayList> rootTasks; private String astStringTree; boolean extended; + boolean formatted; public ExplainWork() { } public ExplainWork(String resFile, - List> rootTasks, String astStringTree, - boolean extended) { + List> rootTasks, + String astStringTree, + boolean extended, + boolean formatted) { this.resFile = resFile; this.rootTasks = new ArrayList>(rootTasks); this.astStringTree = astStringTree; this.extended = extended; + this.formatted = formatted; } public String getResFile() { @@ -79,4 +83,12 @@ public void setExtended(boolean extended) { this.extended = extended; } + + public boolean isFormatted() { + return formatted; + } + + public void setFormatted(boolean formatted) { + this.formatted = formatted; + } } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (revision 1197403) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (working copy) @@ -285,7 +285,8 @@ explainStatement @init { msgs.push("explain statement"); } @after { msgs.pop(); } - : KW_EXPLAIN (isExtended=KW_EXTENDED)? execStatement -> ^(TOK_EXPLAIN execStatement $isExtended?) + : KW_EXPLAIN (explainOptions=KW_EXTENDED|explainOptions=KW_FORMATTED)? execStatement + -> ^(TOK_EXPLAIN execStatement $explainOptions?) ; execStatement @@ -639,7 +640,6 @@ : Identifier KW_CHANGE KW_COLUMN? oldName=Identifier newName=Identifier colType (KW_COMMENT comment=StringLiteral)? alterStatementChangeColPosition? ->^(TOK_ALTERTABLE_RENAMECOL Identifier $oldName $newName colType $comment? alterStatementChangeColPosition?) ; - alterStatementChangeColPosition : first=KW_FIRST|KW_AFTER afterCol=Identifier Index: ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java (revision 1197403) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java (working copy) @@ -47,10 +47,13 @@ .getChild(0)); sem.analyze((ASTNode) ast.getChild(0), ctx); sem.validate(); - + boolean extended = false; - if (ast.getChildCount() > 1) { - extended = true; + boolean formatted = false; + if (ast.getChildCount() == 2) { + int explainOptions = ast.getChild(1).getType(); + formatted = (explainOptions == HiveParser.KW_FORMATTED); + extended = (explainOptions == HiveParser.KW_EXTENDED); } ctx.setResFile(new Path(ctx.getLocalTmpFileURI())); @@ -65,7 +68,12 @@ tasks.add(fetchTask); } - rootTasks.add(TaskFactory.get(new ExplainWork(ctx.getResFile().toString(), - tasks, ((ASTNode) ast.getChild(0)).toStringTree(), extended), conf)); + rootTasks.add( + TaskFactory.get(new ExplainWork(ctx.getResFile().toString(), + tasks, + ((ASTNode) ast.getChild(0)).toStringTree(), + extended, + formatted), + conf)); } }