diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Attr.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Attr.java deleted file mode 100644 index 5d355d2..0000000 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Attr.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.common.jsonexplain.tez; - -public final class Attr implements Comparable { - public final String name; - public final String value; - - public Attr(String name, String value) { - super(); - this.name = name; - this.value = value; - } - - @Override - public int compareTo(Attr o) { - return this.name.compareToIgnoreCase(o.name); - } - - public String toString() { - return this.name + this.value; - } -} diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Op.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Op.java index d0c1037..61c714d 100644 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Op.java +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Op.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.common.jsonexplain.tez; import java.util.ArrayList; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -34,7 +33,7 @@ public final String operatorId; public Op parent; public final List children; - public final List attrs; + public final Map attrs; // the jsonObject for this operator public final JSONObject opObject; // the vertex that this operator belongs to @@ -43,8 +42,9 @@ // ReduceOutputOperator public final String outputVertexName; - public Op(String name, String id, String outputVertexName, List children, List attrs, - JSONObject opObject, Vertex vertex, TezJsonParser tezJsonParser) throws JSONException { + public Op(String name, String id, String outputVertexName, List children, + Map attrs, JSONObject opObject, Vertex vertex, TezJsonParser tezJsonParser) + throws JSONException { super(); this.name = name; this.operatorId = id; @@ -62,14 +62,14 @@ private void inlineJoinOp() throws Exception { JSONObject mapjoinObj = opObject.getJSONObject("Map Join Operator"); // get the map for posToVertex JSONObject verticeObj = mapjoinObj.getJSONObject("input vertices:"); - Map posToVertex = new LinkedHashMap<>(); + Map posToVertex = new LinkedHashMap<>(); for (String pos : JSONObject.getNames(verticeObj)) { String vertexName = verticeObj.getString(pos); - posToVertex.put(pos, vertexName); // update the connection Connection c = null; for (Connection connection : vertex.parentConnections) { if (connection.from.name.equals(vertexName)) { + posToVertex.put(pos, connection.from); c = connection; break; } @@ -79,22 +79,30 @@ private void inlineJoinOp() throws Exception { } } // update the attrs - removeAttr("input vertices:"); - // update the keys to use vertex name + this.attrs.remove("input vertices:"); + // update the keys to use operator name JSONObject keys = mapjoinObj.getJSONObject("keys:"); if (keys.length() != 0) { JSONObject newKeys = new JSONObject(new LinkedHashMap<>()); for (String key : JSONObject.getNames(keys)) { - String vertexName = posToVertex.get(key); - if (vertexName != null) { - newKeys.put(vertexName, keys.get(key)); + Vertex vertex = posToVertex.get(key); + if (vertex != null) { + if (vertex.rootOps.size() > 1) { + throw new Exception("There are more than one root operators in a single vertex " + + vertex.name + " when hive explain user is trying to identify the operator id."); + } + newKeys.put(vertex.rootOps.get(0).operatorId, keys.get(key)); } else { - newKeys.put(this.vertex.name, keys.get(key)); + if (parent == null) { + throw new Exception( + "Can not find the source operator on one of the branches of join."); + } + newKeys.put(this.parent.operatorId, keys.get(key)); } } // update the attrs - removeAttr("keys:"); - this.attrs.add(new Attr("keys:", newKeys.toString())); + this.attrs.remove("keys:"); + this.attrs.put("keys:", newKeys.toString()); } } // inline merge join operator in a self-join @@ -107,12 +115,17 @@ private void inlineJoinOp() throws Exception { } } - private String getNameWithOpId() { + private String getNameWithOpIdStats() { + StringBuffer sb = new StringBuffer(); + sb.append(TezJsonParserUtils.renameReduceOutputOperator(name, vertex)); if (operatorId != null) { - return this.name + " [" + operatorId + "]"; - } else { - return this.name; + sb.append(" [" + operatorId + "]"); + } + if (!TezJsonParserUtils.OperatorNoStats.contains(name) && attrs.containsKey("Statistics:")) { + sb.append(" (" + attrs.get("Statistics:") + ")"); } + attrs.remove("Statistics:"); + return sb.toString(); } /** @@ -123,19 +136,19 @@ private String getNameWithOpId() { * operator so that we can decide the corresponding indent. * @throws Exception */ - public void print(Printer printer, List indentFlag, boolean branchOfJoinOp) + public void print(Printer printer, int indentFlag, boolean branchOfJoinOp) throws Exception { // print name if (parser.printSet.contains(this)) { printer.println(TezJsonParser.prefixString(indentFlag) + " Please refer to the previous " - + this.getNameWithOpId()); + + this.getNameWithOpIdStats()); return; } parser.printSet.add(this); if (!branchOfJoinOp) { - printer.println(TezJsonParser.prefixString(indentFlag) + this.getNameWithOpId()); + printer.println(TezJsonParser.prefixString(indentFlag) + this.getNameWithOpIdStats()); } else { - printer.println(TezJsonParser.prefixString(indentFlag, "|<-") + this.getNameWithOpId()); + printer.println(TezJsonParser.prefixString(indentFlag, "<-") + this.getNameWithOpIdStats()); } branchOfJoinOp = false; // if this operator is a Map Join Operator or a Merge Join Operator @@ -156,71 +169,28 @@ public void print(Printer printer, List indentFlag, boolean branchOfJoi } } // print attr - List attFlag = new ArrayList<>(); - attFlag.addAll(indentFlag); - // should print | if (1) it is branchOfJoinOp or (2) it is the last op and - // has following non-inlined vertex - if (branchOfJoinOp || (this.parent == null && !noninlined.isEmpty())) { - attFlag.add(true); - } else { - attFlag.add(false); - } - Collections.sort(attrs); - for (Attr attr : attrs) { - printer.println(TezJsonParser.prefixString(attFlag) + attr.toString()); + indentFlag++; + if (!attrs.isEmpty()) { + printer + .println(TezJsonParser.prefixString(indentFlag) + TezJsonParserUtils.attrsToString(attrs)); } // print inline vertex if (parser.inlineMap.containsKey(this)) { for (int index = 0; index < parser.inlineMap.get(this).size(); index++) { Connection connection = parser.inlineMap.get(this).get(index); - List vertexFlag = new ArrayList<>(); - vertexFlag.addAll(indentFlag); - if (branchOfJoinOp) { - vertexFlag.add(true); - } - // if there is an inline vertex but the operator itself is not on a join - // branch, - // then it means it is from a vertex created by an operator tree, - // e.g., fetch operator, etc. - else { - vertexFlag.add(false); - } - connection.from.print(printer, vertexFlag, connection.type, this.vertex); + connection.from.print(printer, indentFlag, connection.type, this.vertex); } } // print parent op, i.e., where data comes from if (this.parent != null) { - List parentFlag = new ArrayList<>(); - parentFlag.addAll(indentFlag); - parentFlag.add(false); - this.parent.print(printer, parentFlag, branchOfJoinOp); + this.parent.print(printer, indentFlag, branchOfJoinOp); } // print next vertex else { for (int index = 0; index < noninlined.size(); index++) { Vertex v = noninlined.get(index).from; - List vertexFlag = new ArrayList<>(); - vertexFlag.addAll(indentFlag); - if (index != noninlined.size() - 1) { - vertexFlag.add(true); - } else { - vertexFlag.add(false); - } - v.print(printer, vertexFlag, noninlined.get(index).type, this.vertex); - } - } - } - - public void removeAttr(String name) { - int removeIndex = -1; - for (int index = 0; index < attrs.size(); index++) { - if (attrs.get(index).name.equals(name)) { - removeIndex = index; - break; + v.print(printer, indentFlag, noninlined.get(index).type, this.vertex); } } - if (removeIndex != -1) { - attrs.remove(removeIndex); - } } } diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Stage.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Stage.java index 455d59f..e942a76 100644 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Stage.java +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Stage.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.Map.Entry; import org.apache.hadoop.fs.Path; import org.json.JSONArray; @@ -43,7 +44,7 @@ // downstream stages. public final List childStages = new ArrayList<>(); public final Map vertexs =new LinkedHashMap<>(); - public final List attrs = new ArrayList<>(); + public final Map attrs = new TreeMap<>(); Map> tezStageDependency; // some stage may contain only a single operator, e.g., create table operator, // fetch operator. @@ -112,6 +113,17 @@ public void extractVertex(JSONObject object) throws Exception { // for union vertex, we reverse the dependency relationship if (!"CONTAINS".equals(type)) { v.addDependency(new Connection(type, parentVertex)); + switch (type) { + case "BROADCAST_EDGE": + parentVertex.type = Vertex.Type.BROADCAST; + break; + case "SHUFFLE_EDGE": + parentVertex.type = Vertex.Type.SHUFFLE; + break; + default: + parentVertex.type = Vertex.Type.OTHER; + break; + } parentVertex.children.add(v); } else { parentVertex.addDependency(new Connection(type, v)); @@ -133,6 +145,17 @@ public void extractVertex(JSONObject object) throws Exception { String type = obj.getString("type"); if (!"CONTAINS".equals(type)) { v.addDependency(new Connection(type, parentVertex)); + switch (type) { + case "BROADCAST_EDGE": + parentVertex.type = Vertex.Type.BROADCAST; + break; + case "SHUFFLE_EDGE": + parentVertex.type = Vertex.Type.SHUFFLE; + break; + default: + parentVertex.type = Vertex.Type.OTHER; + break; + } parentVertex.children.add(v); } else { parentVertex.addDependency(new Connection(type, v)); @@ -162,7 +185,9 @@ public void extractVertex(JSONObject object) throws Exception { if (name.contains("Operator")) { this.op = extractOp(name, object.getJSONObject(name)); } else { - attrs.add(new Attr(name, object.get(name).toString())); + if (!object.get(name).toString().isEmpty()) { + attrs.put(name, object.get(name).toString()); + } } } } @@ -178,14 +203,14 @@ public void extractVertex(JSONObject object) throws Exception { * etc */ Op extractOp(String opName, JSONObject opObj) throws Exception { - List attrs = new ArrayList<>(); + Map attrs = new TreeMap<>(); Vertex v = null; if (opObj.length() > 0) { String[] names = JSONObject.getNames(opObj); for (String name : names) { Object o = opObj.get(name); - if (isPrintable(o)) { - attrs.add(new Attr(name, o.toString())); + if (isPrintable(o) && !o.toString().isEmpty()) { + attrs.put(name, o.toString()); } else if (o instanceof JSONObject) { JSONObject attrObj = (JSONObject) o; if (attrObj.length() > 0) { @@ -196,7 +221,9 @@ Op extractOp(String opName, JSONObject opObj) throws Exception { v.extractOpTree(); } else { for (String attrName : JSONObject.getNames(attrObj)) { - attrs.add(new Attr(attrName, attrObj.get(attrName).toString())); + if (!attrObj.get(attrName).toString().isEmpty()) { + attrs.put(attrName, attrObj.get(attrName).toString()); + } } } } @@ -224,7 +251,7 @@ private boolean isPrintable(Object val) { return false; } - public void print(Printer printer, List indentFlag) throws Exception { + public void print(Printer printer, int indentFlag) throws Exception { // print stagename if (parser.printSet.contains(this)) { printer.println(TezJsonParser.prefixString(indentFlag) + " Please refer to the previous " @@ -234,27 +261,23 @@ public void print(Printer printer, List indentFlag) throws Exception { parser.printSet.add(this); printer.println(TezJsonParser.prefixString(indentFlag) + externalName); // print vertexes - List nextIndentFlag = new ArrayList<>(); - nextIndentFlag.addAll(indentFlag); - nextIndentFlag.add(false); + indentFlag++; for (Vertex candidate : this.vertexs.values()) { if (!parser.isInline(candidate) && candidate.children.isEmpty()) { - candidate.print(printer, nextIndentFlag, null, null); + candidate.print(printer, indentFlag, null, null); } } if (!attrs.isEmpty()) { - Collections.sort(attrs); - for (Attr attr : attrs) { - printer.println(TezJsonParser.prefixString(nextIndentFlag) + attr.toString()); - } + printer.println(TezJsonParser.prefixString(indentFlag) + + TezJsonParserUtils.attrsToString(attrs)); } if (op != null) { - op.print(printer, nextIndentFlag, false); + op.print(printer, indentFlag, false); } - nextIndentFlag.add(false); + indentFlag++; // print dependent stages for (Stage stage : this.parentStages) { - stage.print(printer, nextIndentFlag); + stage.print(printer, indentFlag); } } } diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java index b193fef..9228315 100644 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java @@ -75,13 +75,10 @@ public void extractStagesAndPlans(JSONObject inputObject) throws Exception { * help to generate correct indent * @return */ - public static String prefixString(List indentFlag) { + public static String prefixString(int indentFlag) { StringBuilder sb = new StringBuilder(); - for (int index = 0; index < indentFlag.size(); index++) { - if (indentFlag.get(index)) - sb.append("| "); - else - sb.append(" "); + for (int index = 0; index < indentFlag; index++) { + sb.append(" "); } return sb.toString(); } @@ -92,13 +89,10 @@ public static String prefixString(List indentFlag) { * help to generate correct indent with a specific tail * @return */ - public static String prefixString(List indentFlag, String tail) { + public static String prefixString(int indentFlag, String tail) { StringBuilder sb = new StringBuilder(); - for (int index = 0; index < indentFlag.size(); index++) { - if (indentFlag.get(index)) - sb.append("| "); - else - sb.append(" "); + for (int index = 0; index < indentFlag; index++) { + sb.append(" "); } int len = sb.length(); return sb.replace(len - tail.length(), len, tail).toString(); @@ -136,11 +130,10 @@ public void print(JSONObject inputObject, PrintStream outputStream) throws Excep printer.println(); } } - List indentFlag = new ArrayList<>(); // print out all the stages that have no childStages. for (Stage candidate : this.stages.values()) { if (candidate.childStages.isEmpty()) { - candidate.print(printer, indentFlag); + candidate.print(printer, 0); } } outputStream.println(printer.toString()); diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParserUtils.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParserUtils.java new file mode 100644 index 0000000..6ae4efa --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParserUtils.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.jsonexplain.tez; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + + +public class TezJsonParserUtils { + + public static List OperatorNoStats = Arrays.asList(new String[] { "File Output Operator", + "Reduce Output Operator" }); + + public static String renameReduceOutputOperator(String operatorName, Vertex vertex) { + if (operatorName.equals("Reduce Output Operator")) { + return vertex.type.name(); + } else { + return operatorName; + } + } + + public static String attrsToString(Map attrs) { + StringBuffer sb = new StringBuffer(); + boolean first = true; + for (Entry entry : attrs.entrySet()) { + if (first) { + first = false; + } else { + sb.append(","); + } + sb.append(entry.getKey() + entry.getValue()); + } + return sb.toString(); + } +} diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Vertex.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Vertex.java index be01b8b..2e2d57f 100644 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Vertex.java +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Vertex.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.TreeMap; import org.codehaus.jackson.JsonParseException; import org.codehaus.jackson.map.JsonMappingException; @@ -53,6 +55,12 @@ // execution mode public String executionMode = ""; + public static enum Type { + BROADCAST, SHUFFLE, OTHER + }; + // type for broadcast or shuffle only. + public Type type; + public Vertex(String name, JSONObject vertexObject, TezJsonParser tezJsonParser) { super(); this.name = name; @@ -134,7 +142,7 @@ Op extractOp(JSONObject operator) throws JSONException, JsonParseException, Json } else { String opName = names[0]; JSONObject attrObj = (JSONObject) operator.get(opName); - List attrs = new ArrayList<>(); + Map attrs = new TreeMap<>(); List children = new ArrayList<>(); String id = null; String outputVertexName = null; @@ -162,7 +170,9 @@ Op extractOp(JSONObject operator) throws JSONException, JsonParseException, Json } else if (attrName.equals("outputname:")) { outputVertexName = attrObj.get(attrName).toString(); } else { - attrs.add(new Attr(attrName, attrObj.get(attrName).toString())); + if (!attrObj.get(attrName).toString().isEmpty()) { + attrs.put(attrName, attrObj.get(attrName).toString()); + } } } } @@ -178,22 +188,22 @@ Op extractOp(JSONObject operator) throws JSONException, JsonParseException, Json } } - public void print(Printer printer, List indentFlag, String type, Vertex callingVertex) + public void print(Printer printer, int indentFlag, String type, Vertex callingVertex) throws JSONException, Exception { // print vertexname if (parser.printSet.contains(this) && !hasMultiReduceOp) { if (type != null) { - printer.println(TezJsonParser.prefixString(indentFlag, "|<-") + printer.println(TezJsonParser.prefixString(indentFlag, "<-") + " Please refer to the previous " + this.name + " [" + type + "]"); } else { - printer.println(TezJsonParser.prefixString(indentFlag, "|<-") + printer.println(TezJsonParser.prefixString(indentFlag, "<-") + " Please refer to the previous " + this.name); } return; } parser.printSet.add(this); if (type != null) { - printer.println(TezJsonParser.prefixString(indentFlag, "|<-") + this.name + " [" + type + "]" + printer.println(TezJsonParser.prefixString(indentFlag, "<-") + this.name + " [" + type + "]" + this.executionMode); } else if (this.name != null) { printer.println(TezJsonParser.prefixString(indentFlag) + this.name + this.executionMode); @@ -226,14 +236,7 @@ public void print(Printer printer, List indentFlag, String type, Vertex // print dependent vertexs for (int index = 0; index < this.parentConnections.size(); index++) { Connection connection = this.parentConnections.get(index); - List unionFlag = new ArrayList<>(); - unionFlag.addAll(indentFlag); - if (index != this.parentConnections.size() - 1) { - unionFlag.add(true); - } else { - unionFlag.add(false); - } - connection.from.print(printer, unionFlag, connection.type, this); + connection.from.print(printer, indentFlag++, connection.type, this); } } } diff --git a/data/scripts/q_test_cleanup.sql b/data/scripts/q_test_cleanup.sql index 4620dcd..257eac0 100644 --- a/data/scripts/q_test_cleanup.sql +++ b/data/scripts/q_test_cleanup.sql @@ -1,25 +1,3 @@ DROP TABLE IF EXISTS src; DROP TABLE IF EXISTS src1; -DROP TABLE IF EXISTS src_json; -DROP TABLE IF EXISTS src_sequencefile; -DROP TABLE IF EXISTS src_thrift; -DROP TABLE IF EXISTS srcbucket; -DROP TABLE IF EXISTS srcbucket2; DROP TABLE IF EXISTS srcpart; -DROP TABLE IF EXISTS primitives; -DROP TABLE IF EXISTS dest1; -DROP TABLE IF EXISTS dest2; -DROP TABLE IF EXISTS dest3; -DROP TABLE IF EXISTS dest4; -DROP TABLE IF EXISTS dest4_sequencefile; -DROP TABLE IF EXISTS dest_j1; -DROP TABLE IF EXISTS dest_g1; -DROP TABLE IF EXISTS dest_g2; -DROP TABLE IF EXISTS fetchtask_ioexception; -DROP TABLE IF EXISTS alltypesorc; -DROP TABLE IF EXISTS cbo_t1; -DROP TABLE IF EXISTS cbo_t2; -DROP TABLE IF EXISTS cbo_t3; -DROP TABLE IF EXISTS src_cbo; -DROP TABLE IF EXISTS part; -DROP TABLE IF EXISTS lineitem; diff --git a/data/scripts/q_test_init.sql b/data/scripts/q_test_init.sql index 0c8668c..12ea6f2 100644 --- a/data/scripts/q_test_init.sql +++ b/data/scripts/q_test_init.sql @@ -26,82 +26,6 @@ ANALYZE TABLE src1 COMPUTE STATISTICS; ANALYZE TABLE src1 COMPUTE STATISTICS FOR COLUMNS key,value; -- --- Table src_json --- -DROP TABLE IF EXISTS src_json; - -CREATE TABLE src_json (json STRING COMMENT 'default') STORED AS TEXTFILE; - -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/json.txt" INTO TABLE src_json; - -ANALYZE TABLE src_json COMPUTE STATISTICS; - -ANALYZE TABLE src_json COMPUTE STATISTICS FOR COLUMNS json; - --- --- Table src_sequencefile --- -DROP TABLE IF EXISTS src_sequencefile; - -CREATE TABLE src_sequencefile (key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS SEQUENCEFILE; - -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.seq" INTO TABLE src_sequencefile; - -ANALYZE TABLE src_sequencefile COMPUTE STATISTICS; - -ANALYZE TABLE src_sequencefile COMPUTE STATISTICS FOR COLUMNS key,value; - --- --- Table src_thrift --- -DROP TABLE IF EXISTS src_thrift; - -CREATE TABLE src_thrift -ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.thrift.ThriftDeserializer' -WITH SERDEPROPERTIES ( - 'serialization.class' = 'org.apache.hadoop.hive.serde2.thrift.test.Complex', - 'serialization.format' = 'org.apache.thrift.protocol.TBinaryProtocol') -STORED AS SEQUENCEFILE; - -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/complex.seq" INTO TABLE src_thrift; - -ANALYZE TABLE src_thrift COMPUTE STATISTICS; - --- --- Table srcbucket --- -DROP TABLE IF EXISTS srcbucket; - -CREATE TABLE srcbucket (key INT, value STRING) -CLUSTERED BY (key) INTO 2 BUCKETS -STORED AS TEXTFILE; - -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket0.txt" INTO TABLE srcbucket; -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket1.txt" INTO TABLE srcbucket; - -ANALYZE TABLE srcbucket COMPUTE STATISTICS; - -ANALYZE TABLE srcbucket COMPUTE STATISTICS FOR COLUMNS key,value; - --- --- Table srcbucket2 --- -DROP TABLE IF EXISTS srcbucket2; - -CREATE TABLE srcbucket2 (key INT, value STRING) -CLUSTERED BY (key) INTO 4 BUCKETS -STORED AS TEXTFILE; - -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket20.txt" INTO TABLE srcbucket2; -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket21.txt" INTO TABLE srcbucket2; -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket22.txt" INTO TABLE srcbucket2; -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket23.txt" INTO TABLE srcbucket2; - -ANALYZE TABLE srcbucket2 COMPUTE STATISTICS; - -ANALYZE TABLE srcbucket2 COMPUTE STATISTICS FOR COLUMNS key,value; - --- -- Table srcpart -- DROP TABLE IF EXISTS srcpart; @@ -126,196 +50,3 @@ ANALYZE TABLE srcpart PARTITION(ds, hr) COMPUTE STATISTICS; ANALYZE TABLE srcpart PARTITION(ds, hr) COMPUTE STATISTICS FOR COLUMNS key,value; --- --- Table alltypesorc --- -DROP TABLE IF EXISTS alltypesorc; -CREATE TABLE alltypesorc( - ctinyint TINYINT, - csmallint SMALLINT, - cint INT, - cbigint BIGINT, - cfloat FLOAT, - cdouble DOUBLE, - cstring1 STRING, - cstring2 STRING, - ctimestamp1 TIMESTAMP, - ctimestamp2 TIMESTAMP, - cboolean1 BOOLEAN, - cboolean2 BOOLEAN) - STORED AS ORC; - -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/alltypesorc" -OVERWRITE INTO TABLE alltypesorc; - -ANALYZE TABLE alltypesorc COMPUTE STATISTICS; - -ANALYZE TABLE alltypesorc COMPUTE STATISTICS FOR COLUMNS ctinyint,csmallint,cint,cbigint,cfloat,cdouble,cstring1,cstring2,ctimestamp1,ctimestamp2,cboolean1,cboolean2; - --- --- Table primitives --- -DROP TABLE IF EXISTS primitives; -CREATE TABLE primitives ( - id INT COMMENT 'default', - bool_col BOOLEAN COMMENT 'default', - tinyint_col TINYINT COMMENT 'default', - smallint_col SMALLINT COMMENT 'default', - int_col INT COMMENT 'default', - bigint_col BIGINT COMMENT 'default', - float_col FLOAT COMMENT 'default', - double_col DOUBLE COMMENT 'default', - date_string_col STRING COMMENT 'default', - string_col STRING COMMENT 'default', - timestamp_col TIMESTAMP COMMENT 'default') -PARTITIONED BY (year INT COMMENT 'default', month INT COMMENT 'default') -ROW FORMAT DELIMITED - FIELDS TERMINATED BY ',' - ESCAPED BY '\\' -STORED AS TEXTFILE; - -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/types/primitives/090101.txt" -OVERWRITE INTO TABLE primitives PARTITION(year=2009, month=1); - -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/types/primitives/090201.txt" -OVERWRITE INTO TABLE primitives PARTITION(year=2009, month=2); - -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/types/primitives/090301.txt" -OVERWRITE INTO TABLE primitives PARTITION(year=2009, month=3); - -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/types/primitives/090401.txt" -OVERWRITE INTO TABLE primitives PARTITION(year=2009, month=4); - --- --- Function qtest_get_java_boolean --- -DROP FUNCTION IF EXISTS qtest_get_java_boolean; -CREATE FUNCTION qtest_get_java_boolean AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaBoolean'; - --- --- Table dest1 --- -DROP TABLE IF EXISTS dest1; - -CREATE TABLE dest1 (key STRING COMMENT 'default', value STRING COMMENT 'default') -STORED AS -INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' -OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'; - --- --- Table dest2 --- -DROP TABLE IF EXISTS dest2; - -CREATE TABLE dest2 (key STRING COMMENT 'default', value STRING COMMENT 'default') -STORED AS -INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' -OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'; - --- --- Table dest3 --- -DROP TABLE IF EXISTS dest3; - -CREATE TABLE dest3 (key STRING COMMENT 'default', value STRING COMMENT 'default') -PARTITIONED BY (ds STRING, hr STRING) -STORED AS -INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' -OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'; -ALTER TABLE dest3 ADD PARTITION (ds='2008-04-08',hr='12'); - --- --- Table dest4 --- -DROP TABLE IF EXISTS dest4; - -CREATE TABLE dest4 (key STRING COMMENT 'default', value STRING COMMENT 'default') -STORED AS -INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' -OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'; - --- --- Table dest4_sequencefile --- -DROP TABLE IF EXISTS dest4_sequencefile; - -CREATE TABLE dest4_sequencefile (key STRING COMMENT 'default', value STRING COMMENT 'default') -STORED AS -INPUTFORMAT 'org.apache.hadoop.mapred.SequenceFileInputFormat' -OUTPUTFORMAT 'org.apache.hadoop.mapred.SequenceFileOutputFormat'; - - --- --- CBO tables --- - -drop table if exists cbo_t1; -drop table if exists cbo_t2; -drop table if exists cbo_t3; -drop table if exists src_cbo; -drop table if exists part; -drop table if exists lineitem; - -set hive.cbo.enable=true; - -create table cbo_t1(key string, value string, c_int int, c_float float, c_boolean boolean) partitioned by (dt string) row format delimited fields terminated by ',' STORED AS TEXTFILE; -create table cbo_t2(key string, value string, c_int int, c_float float, c_boolean boolean) partitioned by (dt string) row format delimited fields terminated by ',' STORED AS TEXTFILE; -create table cbo_t3(key string, value string, c_int int, c_float float, c_boolean boolean) row format delimited fields terminated by ',' STORED AS TEXTFILE; - -load data local inpath '../../data/files/cbo_t1.txt' into table cbo_t1 partition (dt='2014'); -load data local inpath '../../data/files/cbo_t2.txt' into table cbo_t2 partition (dt='2014'); -load data local inpath '../../data/files/cbo_t3.txt' into table cbo_t3; - -CREATE TABLE part( - p_partkey INT, - p_name STRING, - p_mfgr STRING, - p_brand STRING, - p_type STRING, - p_size INT, - p_container STRING, - p_retailprice DOUBLE, - p_comment STRING -); - -LOAD DATA LOCAL INPATH '../../data/files/part_tiny.txt' overwrite into table part; - -CREATE TABLE lineitem (L_ORDERKEY INT, - L_PARTKEY INT, - L_SUPPKEY INT, - L_LINENUMBER INT, - L_QUANTITY DOUBLE, - L_EXTENDEDPRICE DOUBLE, - L_DISCOUNT DOUBLE, - L_TAX DOUBLE, - L_RETURNFLAG STRING, - L_LINESTATUS STRING, - l_shipdate STRING, - L_COMMITDATE STRING, - L_RECEIPTDATE STRING, - L_SHIPINSTRUCT STRING, - L_SHIPMODE STRING, - L_COMMENT STRING) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|'; - -LOAD DATA LOCAL INPATH '../../data/files/lineitem.txt' OVERWRITE INTO TABLE lineitem; - -create table src_cbo as select * from src; - - -analyze table cbo_t1 partition (dt) compute statistics; -analyze table cbo_t1 compute statistics for columns key, value, c_int, c_float, c_boolean; -analyze table cbo_t2 partition (dt) compute statistics; -analyze table cbo_t2 compute statistics for columns key, value, c_int, c_float, c_boolean; -analyze table cbo_t3 compute statistics; -analyze table cbo_t3 compute statistics for columns key, value, c_int, c_float, c_boolean; -analyze table src_cbo compute statistics; -analyze table src_cbo compute statistics for columns; -analyze table part compute statistics; -analyze table part compute statistics for columns; -analyze table lineitem compute statistics; -analyze table lineitem compute statistics for columns; - -reset; -set hive.stats.dbclass=fs; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java index f48db6a..4116141 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java @@ -763,7 +763,7 @@ private JSONObject outputPlan(Object work, PrintStream out, out.println(header); } JSONObject jsonOut = outputPlan(val, out, extended, jsonOutput, ind); - if (jsonOutput) { + if (jsonOutput && jsonOut != null && jsonOut.length() != 0) { if (!skipHeader) { json.put(header, jsonOut); } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java index bc67e5a..adec5c7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java @@ -33,11 +33,16 @@ protected long memNeeded = 0; @Override - @Explain(skipHeader = true, displayName = "Statistics", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(skipHeader = true, displayName = "Statistics") public Statistics getStatistics() { return statistics; } + @Explain(skipHeader = true, displayName = "Statistics", explainLevels = { Level.USER }) + public String getUserLevelStatistics() { + return statistics.toUserLevelExplainString(); + } + @Override public void setStatistics(Statistics statistics) { this.statistics = statistics; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java index 40a8477..cc462be 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java @@ -181,7 +181,7 @@ public void setTableInfo(final TableDesc tableInfo) { this.tableInfo = tableInfo; } - @Explain(displayName = "compressed", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(displayName = "compressed") public boolean getCompressed() { return compressed; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java index d04cb78..abb2025 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java @@ -18,7 +18,9 @@ package org.apache.hadoop.hive.ql.plan; +import java.util.Arrays; import java.util.List; + import org.apache.hadoop.hive.ql.plan.Explain.Level; @@ -105,11 +107,14 @@ public FilterDesc( this.sampleDescr = sampleDescr; } - @Explain(displayName = "predicate", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(displayName = "predicate") public String getPredicateString() { - StringBuilder sb = new StringBuilder(); - PlanUtils.addExprToStringBuffer(predicate, sb); - return sb.toString(); + return PlanUtils.getExprListString(Arrays.asList(predicate)); + } + + @Explain(displayName = "predicate", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public String getUserLevelExplainPredicateString() { + return PlanUtils.getExprListString(Arrays.asList(predicate), true); } public org.apache.hadoop.hive.ql.plan.ExprNodeDesc getPredicate() { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java index 0f2855e..67a3fab 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java @@ -158,11 +158,16 @@ public void setMode(final Mode mode) { this.mode = mode; } - @Explain(displayName = "keys", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(displayName = "keys") public String getKeyString() { return PlanUtils.getExprListString(keys); } + @Explain(displayName = "keys", explainLevels = { Level.USER }) + public String getUserLevelExplainKeyString() { + return PlanUtils.getExprListString(keys, true); + } + public ArrayList getKeys() { return keys; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java index 7c8eee2..0b1e95d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java @@ -303,7 +303,7 @@ public void setRetainList(Map> retainList) { /** * @return the keys in string form */ - @Explain(displayName = "keys", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(displayName = "keys") public Map getKeysString() { Map keyMap = new LinkedHashMap(); for (Map.Entry> k: getKeys().entrySet()) { @@ -312,6 +312,15 @@ public void setRetainList(Map> retainList) { return keyMap; } + @Explain(displayName = "keys", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public Map getUserLevelExplainKeysString() { + Map keyMap = new LinkedHashMap(); + for (Map.Entry> k: getKeys().entrySet()) { + keyMap.put(k.getKey(), PlanUtils.getExprListString(k.getValue(), true)); + } + return keyMap; + } + /** * @return the keys */ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java index eb83fd6..4ed2a0d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java @@ -211,7 +211,7 @@ public void setReversedExprs(Map reversedExprs) { /** * @return the keys in string form */ - @Explain(displayName = "keys", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(displayName = "keys") public Map getKeysString() { if (joinKeys == null) { return null; @@ -224,6 +224,19 @@ public void setReversedExprs(Map reversedExprs) { return keyMap; } + @Explain(displayName = "keys", explainLevels = { Level.USER }) + public Map getUserLevelExplainKeysString() { + if (joinKeys == null) { + return null; + } + + Map keyMap = new LinkedHashMap(); + for (byte i = 0; i < joinKeys.length; i++) { + keyMap.put(i, PlanUtils.getExprListString(Arrays.asList(joinKeys[i]), true)); + } + return keyMap; + } + public void setExprs(final Map> exprs) { this.exprs = exprs; } @@ -235,7 +248,7 @@ public void setExprs(final Map> exprs) { * * @return Map from alias to filters on the alias. */ - @Explain(displayName = "filter predicates", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(displayName = "filter predicates") public Map getFiltersStringMap() { if (getFilters() == null || getFilters().size() == 0) { return null; @@ -281,10 +294,15 @@ public void setFilters(Map> filters) { this.filters = filters; } - @Explain(displayName = "outputColumnNames", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(displayName = "outputColumnNames") public List getOutputColumnNames() { return outputColumnNames; } + + @Explain(displayName = "Output", explainLevels = { Level.USER }) + public List getUserLevelExplainOutputColumnNames() { + return outputColumnNames; + } public void setOutputColumnNames( List outputColumnNames) { @@ -299,7 +317,7 @@ public void setNoOuterJoin(final boolean noOuterJoin) { this.noOuterJoin = noOuterJoin; } - @Explain(displayName = "condition map", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(displayName = "condition map") public List getCondsList() { if (conds == null) { return null; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java index 4b93e7c..ec35860 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java @@ -29,8 +29,6 @@ import java.util.Set; import org.apache.hadoop.hive.ql.plan.Explain.Level; -import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; - /** * Map Join operator Descriptor implementation. * @@ -217,7 +215,7 @@ public void setDumpFilePrefix(String dumpFilePrefix) { * @return the keys in string form */ @Override - @Explain(displayName = "keys", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(displayName = "keys") public Map getKeysString() { Map keyMap = new LinkedHashMap(); for (Map.Entry> k: getKeys().entrySet()) { @@ -226,6 +224,16 @@ public void setDumpFilePrefix(String dumpFilePrefix) { return keyMap; } + @Override + @Explain(displayName = "keys", explainLevels = { Level.USER }) + public Map getUserLevelExplainKeysString() { + Map keyMap = new LinkedHashMap(); + for (Map.Entry> k: getKeys().entrySet()) { + keyMap.put(k.getKey(), PlanUtils.getExprListString(k.getValue(), true)); + } + return keyMap; + } + /** * @return the keys */ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java index 5bea6fb..04d26f3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java @@ -926,7 +926,11 @@ public static ReadEntity addInput(Set inputs, ReadEntity newInput) { return null; } - public static String getExprListString(Collection exprs) { + public static String getExprListString(Collection exprs) { + return getExprListString(exprs, false); + } + + public static String getExprListString(Collection exprs, boolean userLevelExplain) { StringBuilder sb = new StringBuilder(); boolean first = true; for (ExprNodeDesc expr: exprs) { @@ -935,18 +939,20 @@ public static String getExprListString(Collection exprs } else { first = false; } - addExprToStringBuffer(expr, sb); + addExprToStringBuffer(expr, sb, userLevelExplain); } return sb.length() == 0 ? null : sb.toString(); } - public static void addExprToStringBuffer(ExprNodeDesc expr, Appendable sb) { + public static void addExprToStringBuffer(ExprNodeDesc expr, Appendable sb, boolean userLevelExplain) { try { sb.append(expr.getExprString()); - sb.append(" (type: "); - sb.append(expr.getTypeString()); - sb.append(")"); + if (!userLevelExplain) { + sb.append(" (type: "); + sb.append(expr.getTypeString()); + sb.append(")"); + } } catch (IOException e) { throw new RuntimeException(e); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java index 2f69b7f..41d9ffe 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java @@ -213,7 +213,7 @@ public void setOutputValueColumnNames( this.outputValueColumnNames = outputValueColumnNames; } - @Explain(displayName = "key expressions", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(displayName = "key expressions") public String getKeyColString() { return PlanUtils.getExprListString(keyCols); } @@ -234,7 +234,7 @@ public void setNumDistributionKeys(int numKeys) { this.numDistributionKeys = numKeys; } - @Explain(displayName = "value expressions", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(displayName = "value expressions") public String getValueColsString() { return PlanUtils.getExprListString(valueCols); } @@ -247,11 +247,16 @@ public void setValueCols(final java.util.ArrayList valueCols) { this.valueCols = valueCols; } - @Explain(displayName = "Map-reduce partition columns", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(displayName = "Map-reduce partition columns") public String getParitionColsString() { return PlanUtils.getExprListString(partitionCols); } + @Explain(displayName = "PartitionCols", explainLevels = { Level.USER }) + public String getUserLevelExplainParitionColsString() { + return PlanUtils.getExprListString(partitionCols, true); + } + public java.util.ArrayList getPartitionCols() { return partitionCols; } @@ -356,7 +361,7 @@ public void setValueSerializeInfo(TableDesc valueSerializeInfo) { * of the same length as key columns, that consists of only "+" * (ascending order) and "-" (descending order). */ - @Explain(displayName = "sort order", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(displayName = "sort order") public String getOrder() { return keySerializeInfo.getProperties().getProperty( org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_SORT_ORDER); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java index e7bbab4..67a8327 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java @@ -82,11 +82,16 @@ public void setColList( this.colList = colList; } - @Explain(displayName = "outputColumnNames", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(displayName = "outputColumnNames") public List getOutputColumnNames() { return outputColumnNames; } + @Explain(displayName = "Output", explainLevels = { Level.USER }) + public List getUserLevelExplainOutputColumnNames() { + return outputColumnNames; + } + public void setOutputColumnNames( List outputColumnNames) { this.outputColumnNames = outputColumnNames; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java index 4e52bac..029043f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java @@ -102,7 +102,7 @@ public void setColumnStatsState(State columnStatsState) { } @Override - @Explain(displayName = "Statistics", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(displayName = "Statistics") public String toString() { StringBuilder sb = new StringBuilder(); sb.append("Num rows: "); @@ -116,6 +116,21 @@ public String toString() { return sb.toString(); } + @Explain(displayName = "Statistics", explainLevels = { Level.USER }) + public String toUserLevelExplainString() { + StringBuilder sb = new StringBuilder(); + sb.append("rows="); + sb.append(numRows); + sb.append(" width="); + // just to be safe about numRows + if (numRows != 0) { + sb.append(dataSize / numRows); + } else { + sb.append("-1"); + } + return sb.toString(); + } + public String extendedToString() { StringBuilder sb = new StringBuilder(); sb.append(" numRows: "); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java index 2fdb0a1..1da8e91 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java @@ -140,7 +140,7 @@ public void setJobProperties(Map jobProperties) { /** * @return the serdeClassName */ - @Explain(displayName = "serde", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(displayName = "serde") public String getSerdeClassName() { return properties.getProperty(serdeConstants.SERIALIZATION_LIB); } @@ -151,12 +151,12 @@ public String getTableName() { .getProperty(hive_metastoreConstants.META_TABLE_NAME); } - @Explain(displayName = "input format", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(displayName = "input format") public String getInputFileFormatClassName() { return getInputFileFormatClass().getName(); } - @Explain(displayName = "output format", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(displayName = "output format") public String getOutputFileFormatClassName() { return getOutputFileFormatClass().getName(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java index 098aa89..59d20ef 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java @@ -20,6 +20,7 @@ import java.io.Serializable; import java.util.ArrayList; +import java.util.Arrays; import java.util.BitSet; import java.util.List; import java.util.Map; @@ -131,21 +132,38 @@ public Object clone() { return new TableScanDesc(getAlias(), vcs, this.tableMetadata); } - @Explain(displayName = "alias", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(displayName = "alias") public String getAlias() { return alias; } + + @Explain(explainLevels = { Level.USER }) + public String getTbl() { + StringBuffer sb = new StringBuffer(); + sb.append(this.tableMetadata.getCompleteName()); + sb.append("," + alias); + if (isAcidTable()) { + sb.append(", ACID table"); + } + sb.append(",Tbl:"); + sb.append(this.statistics.getBasicStatsState()); + sb.append(",Col:"); + sb.append(this.statistics.getColumnStatsState()); + return sb.toString(); + } - @Explain(displayName = "ACID table", explainLevels = { Level.USER }, displayOnlyOnTrue = true) public boolean isAcidTable() { return SemanticAnalyzer.isAcidTable(this.tableMetadata); } + @Explain(displayName = "Output", explainLevels = { Level.USER }) + public List getOutputColumnNames() { + return this.neededColumns; + } + @Explain(displayName = "filterExpr") public String getFilterExprString() { - StringBuilder sb = new StringBuilder(); - PlanUtils.addExprToStringBuffer(filterExpr, sb); - return sb.toString(); + return PlanUtils.getExprListString(Arrays.asList(filterExpr)); } public ExprNodeGenericFuncDesc getFilterExpr() { diff --git a/ql/src/test/queries/clientpositive/e.q b/ql/src/test/queries/clientpositive/e.q new file mode 100644 index 0000000..3b7f362 --- /dev/null +++ b/ql/src/test/queries/clientpositive/e.q @@ -0,0 +1,11 @@ +set hive.mapred.mode=nonstrict; +set hive.explain.user=true; +set hive.auto.convert.join=true; +set hive.auto.convert.join.noconditionaltask=true; +set hive.auto.convert.join.noconditionaltask.size=10000; + +EXPLAIN +SELECT x.key, z.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11); + diff --git a/ql/src/test/results/clientpositive/tez/e.q.out b/ql/src/test/results/clientpositive/tez/e.q.out new file mode 100644 index 0000000..9ccf3f0 --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/e.q.out @@ -0,0 +1,52 @@ +PREHOOK: query: EXPLAIN +SELECT x.key, z.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT x.key, z.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Map 1 + File Output Operator [FS_16] + Select Operator [SEL_15] (rows=605 width=10) + Output:["_col0","_col1","_col2"] + Map Join Operator [MAPJOIN_26] (rows=605 width=10) + HybridGraceHashJoin:true,Output:["_col0","_col3","_col6"],keys:{"MAPJOIN_25":"_col3","RS_13":"_col0"} + <-Map 3 [BROADCAST_EDGE] + BROADCAST [RS_13] + PartitionCols:_col0 + Select Operator [SEL_8] (rows=500 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_24] (rows=500 width=10) + predicate:key is not null + TableScan [TS_6] (rows=500 width=10) + default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Map Join Operator [MAPJOIN_25] (rows=550 width=10) + HybridGraceHashJoin:true,Output:["_col0","_col3"],keys:{"SEL_2":"_col0","RS_10":"_col1"} + <-Map 2 [BROADCAST_EDGE] + BROADCAST [RS_10] + PartitionCols:_col1 + Select Operator [SEL_5] (rows=25 width=7) + Output:["_col0","_col1"] + Filter Operator [FIL_23] (rows=25 width=7) + predicate:(key is not null and value is not null) + TableScan [TS_3] (rows=25 width=7) + default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Select Operator [SEL_2] (rows=500 width=10) + Output:["_col0"] + Filter Operator [FIL_22] (rows=500 width=10) + predicate:value is not null + TableScan [TS_0] (rows=500 width=10) + default@srcpart,z,Tbl:COMPLETE,Col:NONE,Output:["value"] +