diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/DagJsonParser.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/DagJsonParser.java index 1f01685003..61746c8f3a 100644 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/DagJsonParser.java +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/DagJsonParser.java @@ -40,6 +40,7 @@ // the vertex that should be inlined. public final Map> inlineMap = new LinkedHashMap<>(); + public boolean rewriteObject; public DagJsonParser() { super(); @@ -101,7 +102,12 @@ public static String prefixString(int indentFlag, String tail) { @Override public void print(JSONObject inputObject, PrintStream outputStream) throws Exception { LOG.info("JsonParser is parsing:" + inputObject.toString()); + this.rewriteObject = outputStream == null; + // outputStream == null means we need to process it for explain formatted this.extractStagesAndPlans(inputObject); + if (rewriteObject) { + return; + } Printer printer = new Printer(); // print out the cbo info if (inputObject.has("cboInfo")) { diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Op.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Op.java index 03c5981320..1d9a681db2 100644 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Op.java +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Op.java @@ -36,7 +36,7 @@ public final String name; // tezJsonParser public final DagJsonParser parser; - public final String operatorId; + public String operatorId; public Op parent; public final List children; public final Map attrs; @@ -45,15 +45,15 @@ // the vertex that this operator belongs to public final Vertex vertex; // the vertex that this operator output to - public final String outputVertexName; + public String outputVertexName; // the Operator type - public final OpType type; + public OpType type; public enum OpType { MAPJOIN, MERGEJOIN, RS, OTHERS }; - public Op(String name, String id, String outputVertexName, List children, + public Op(String name, String id, String outputVertexName, Op parent, List children, Map attrs, JSONObject opObject, Vertex vertex, DagJsonParser tezJsonParser) throws JSONException { super(); @@ -61,6 +61,7 @@ public Op(String name, String id, String outputVertexName, List children, this.operatorId = id; this.type = deriveOpType(operatorId); this.outputVertexName = outputVertexName; + this.parent = parent; this.children = children; this.attrs = attrs; this.opObject = opObject; @@ -87,11 +88,10 @@ private OpType deriveOpType(String operatorId) { private void inlineJoinOp() throws Exception { // inline map join operator if (this.type == OpType.MAPJOIN) { - JSONObject joinObj = opObject.getJSONObject(this.name); // get the map for posToVertex Map posToVertex = new LinkedHashMap<>(); - if (joinObj.has("input vertices:")) { - JSONObject verticeObj = joinObj.getJSONObject("input vertices:"); + if (opObject.has("input vertices:")) { + JSONObject verticeObj = opObject.getJSONObject("input vertices:"); for (String pos : JSONObject.getNames(verticeObj)) { String vertexName = verticeObj.getString(pos); // update the connection @@ -111,7 +111,7 @@ private void inlineJoinOp() throws Exception { this.attrs.remove("input vertices:"); } // update the keys to use operator name - JSONObject keys = joinObj.getJSONObject("keys:"); + JSONObject keys = opObject.getJSONObject("keys:"); // find out the vertex for the big table Set parentVertexes = new HashSet<>(); for (Connection connection : vertex.parentConnections) { @@ -124,9 +124,9 @@ private void inlineJoinOp() throws Exception { // first search from the posToVertex if (posToVertex.containsKey(key)) { Vertex v = posToVertex.get(key); - if (v.rootOps.size() == 1) { - posToOpId.put(key, v.rootOps.get(0).operatorId); - } else if ((v.rootOps.size() == 0 && v.vertexType == VertexType.UNION)) { + if (v.outputOps.size() == 1) { + posToOpId.put(key, v.outputOps.get(0).operatorId); + } else if ((v.outputOps.size() == 0 && v.vertexType == VertexType.UNION)) { posToOpId.put(key, v.name); } else { Op joinRSOp = v.getJoinRSOp(vertex); @@ -147,9 +147,9 @@ else if (parent != null) { else if (parentVertexes.size() == 1) { Vertex v = parentVertexes.iterator().next(); parentVertexes.clear(); - if (v.rootOps.size() == 1) { - posToOpId.put(key, v.rootOps.get(0).operatorId); - } else if ((v.rootOps.size() == 0 && v.vertexType == VertexType.UNION)) { + if (v.outputOps.size() == 1) { + posToOpId.put(key, v.outputOps.get(0).operatorId); + } else if ((v.outputOps.size() == 0 && v.vertexType == VertexType.UNION)) { posToOpId.put(key, v.name); } else { Op joinRSOp = v.getJoinRSOp(vertex); @@ -171,7 +171,7 @@ else if (parentVertexes.size() == 1) { } this.attrs.remove("keys:"); StringBuffer sb = new StringBuffer(); - JSONArray conditionMap = joinObj.getJSONArray("condition map:"); + JSONArray conditionMap = opObject.getJSONArray("condition map:"); for (int index = 0; index < conditionMap.length(); index++) { JSONObject cond = conditionMap.getJSONObject(index); String k = (String) cond.keys().next(); @@ -203,9 +203,9 @@ else if (parentVertexes.size() == 1) { for (Connection connection : vertex.parentConnections) { if (connection.from.name.equals(entry.getValue())) { Vertex v = connection.from; - if (v.rootOps.size() == 1) { - posToOpId.put(entry.getKey(), v.rootOps.get(0).operatorId); - } else if ((v.rootOps.size() == 0 && v.vertexType == VertexType.UNION)) { + if (v.outputOps.size() == 1) { + posToOpId.put(entry.getKey(), v.outputOps.get(0).operatorId); + } else if ((v.outputOps.size() == 0 && v.vertexType == VertexType.UNION)) { posToOpId.put(entry.getKey(), v.name); } else { Op joinRSOp = v.getJoinRSOp(vertex); @@ -229,16 +229,15 @@ else if (parentVertexes.size() == 1) { } else { posToOpId.put(vertex.tag, this.parent.operatorId); for (Vertex v : vertex.mergeJoinDummyVertexs) { - if (v.rootOps.size() != 1) { + if (v.outputOps.size() != 1) { throw new Exception("Can not find a single root operators in a single vertex " + v.name + " when hive explain user is trying to identify the operator id."); } - posToOpId.put(v.tag, v.rootOps.get(0).operatorId); + posToOpId.put(v.tag, v.outputOps.get(0).operatorId); } } - JSONObject joinObj = opObject.getJSONObject(this.name); // update the keys to use operator name - JSONObject keys = joinObj.getJSONObject("keys:"); + JSONObject keys = opObject.getJSONObject("keys:"); if (keys.length() != 0) { for (String key : JSONObject.getNames(keys)) { if (!posToOpId.containsKey(key)) { @@ -256,7 +255,7 @@ else if (parentVertexes.size() == 1) { // update the attrs this.attrs.remove("keys:"); StringBuffer sb = new StringBuffer(); - JSONArray conditionMap = joinObj.getJSONArray("condition map:"); + JSONArray conditionMap = opObject.getJSONArray("condition map:"); for (int index = 0; index < conditionMap.length(); index++) { JSONObject cond = conditionMap.getJSONObject(index); String k = (String) cond.keys().next(); @@ -355,4 +354,9 @@ public void print(Printer printer, int indentFlag, boolean branchOfJoinOp) throw } } } + + public void setOperatorId(String operatorId) { + this.operatorId = operatorId; + this.type = deriveOpType(operatorId); + } } diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Stage.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Stage.java index d21a565f67..f0bd5f1cea 100644 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Stage.java +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Stage.java @@ -93,7 +93,7 @@ public void extractVertex(JSONObject object) throws Exception { JSONObject edges = tez.getJSONObject("Edges:"); // iterate for the first time to get all the vertices for (String to : JSONObject.getNames(edges)) { - vertexs.put(to, new Vertex(to, vertices.getJSONObject(to), parser)); + vertexs.put(to, new Vertex(to, vertices.getJSONObject(to), this, parser)); } // iterate for the second time to get all the vertex dependency for (String to : JSONObject.getNames(edges)) { @@ -105,7 +105,7 @@ public void extractVertex(JSONObject object) throws Exception { String parent = obj.getString("parent"); Vertex parentVertex = vertexs.get(parent); if (parentVertex == null) { - parentVertex = new Vertex(parent, vertices.getJSONObject(parent), parser); + parentVertex = new Vertex(parent, vertices.getJSONObject(parent), this, parser); vertexs.put(parent, parentVertex); } String type = obj.getString("type"); @@ -128,7 +128,7 @@ public void extractVertex(JSONObject object) throws Exception { String parent = obj.getString("parent"); Vertex parentVertex = vertexs.get(parent); if (parentVertex == null) { - parentVertex = new Vertex(parent, vertices.getJSONObject(parent), parser); + parentVertex = new Vertex(parent, vertices.getJSONObject(parent), this, parser); vertexs.put(parent, parentVertex); } String type = obj.getString("type"); @@ -147,16 +147,20 @@ public void extractVertex(JSONObject object) throws Exception { } } else { for (String vertexName : JSONObject.getNames(vertices)) { - vertexs.put(vertexName, new Vertex(vertexName, vertices.getJSONObject(vertexName), parser)); + vertexs.put(vertexName, new Vertex(vertexName, vertices.getJSONObject(vertexName), this, parser)); } } - // The opTree in vertex is extracted + + // iterate for the first time to extract opTree in vertex for (Vertex v : vertexs.values()) { if (v.vertexType == VertexType.MAP || v.vertexType == VertexType.REDUCE) { v.extractOpTree(); - v.checkMultiReduceOperator(); } } + // iterate for the second time to rewrite object + for (Vertex v : vertexs.values()) { + v.checkMultiReduceOperator(parser.rewriteObject); + } } else { String[] names = JSONObject.getNames(object); if (names != null) { @@ -196,7 +200,7 @@ Op extractOp(String opName, JSONObject opObj) throws Exception { if (name.equals("Processor Tree:")) { JSONObject object = new JSONObject(new LinkedHashMap<>()); object.put(name, attrObj); - v = new Vertex(null, object, parser); + v = new Vertex(null, object, this, parser); v.extractOpTree(); } else { for (String attrName : JSONObject.getNames(attrObj)) { @@ -211,7 +215,7 @@ Op extractOp(String opName, JSONObject opObj) throws Exception { } } } - Op op = new Op(opName, null, null, null, attrs, null, v, parser); + Op op = new Op(opName, null, null, null, null, attrs, null, v, parser); if (v != null) { parser.addInline(op, new Connection(null, v)); } diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java index c93059d6d3..18dd053ac8 100644 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java @@ -34,6 +34,8 @@ public final class Vertex implements Comparable{ public final String name; + // the stage that this vertex belongs to + public final Stage stage; //tezJsonParser public final DagJsonParser parser; // vertex's parent connections. @@ -45,8 +47,10 @@ // whether this vertex is dummy (which does not really exists but is created), // e.g., a dummy vertex for a mergejoin branch public boolean dummy; - // the rootOps in this vertex - public final List rootOps = new ArrayList<>(); + // the outputOps in this vertex. + public final List outputOps= new ArrayList<>(); + // the inputOps in this vertex. + public final List inputOps= new ArrayList<>(); // we create a dummy vertex for a mergejoin branch for a self join if this // vertex is a mergejoin public final List mergeJoinDummyVertexs = new ArrayList<>(); @@ -69,7 +73,7 @@ }; public String edgeType; - public Vertex(String name, JSONObject vertexObject, DagJsonParser dagJsonParser) { + public Vertex(String name, JSONObject vertexObject, Stage stage, DagJsonParser dagJsonParser) { super(); this.name = name; if (this.name != null) { @@ -87,7 +91,8 @@ public Vertex(String name, JSONObject vertexObject, DagJsonParser dagJsonParser) } this.dummy = false; this.vertexObject = vertexObject; - parser = dagJsonParser; + this.stage = stage; + this.parser = dagJsonParser; } public void addDependency(Connection connection) throws JSONException { @@ -108,16 +113,16 @@ public void extractOpTree() throws JSONException, JsonParseException, JsonMappin if (vertexObject.length() != 0) { for (String key : JSONObject.getNames(vertexObject)) { if (key.equals("Map Operator Tree:")) { - extractOp(vertexObject.getJSONArray(key).getJSONObject(0)); + extractOp(vertexObject.getJSONArray(key).getJSONObject(0), null); } else if (key.equals("Reduce Operator Tree:") || key.equals("Processor Tree:")) { - extractOp(vertexObject.getJSONObject(key)); + extractOp(vertexObject.getJSONObject(key), null); } else if (key.equals("Join:")) { // this is the case when we have a map-side SMB join // one input of the join is treated as a dummy vertex JSONArray array = vertexObject.getJSONArray(key); for (int index = 0; index < array.length(); index++) { JSONObject mpOpTree = array.getJSONObject(index); - Vertex v = new Vertex(null, mpOpTree, parser); + Vertex v = new Vertex(null, mpOpTree, this.stage, parser); v.extractOpTree(); v.dummy = true; mergeJoinDummyVertexs.add(v); @@ -125,7 +130,7 @@ public void extractOpTree() throws JSONException, JsonParseException, JsonMappin } else if (key.equals("Merge File Operator")) { JSONObject opTree = vertexObject.getJSONObject(key); if (opTree.has("Map Operator Tree:")) { - extractOp(opTree.getJSONArray("Map Operator Tree:").getJSONObject(0)); + extractOp(opTree.getJSONArray("Map Operator Tree:").getJSONObject(0), null); } else { throw new Exception("Merge File Operator does not have a Map Operator Tree"); } @@ -139,7 +144,7 @@ public void extractOpTree() throws JSONException, JsonParseException, JsonMappin } else if (key.equals("tag:")) { this.tag = vertexObject.getString(key); } else if (key.equals("Local Work:")) { - extractOp(vertexObject.getJSONObject(key)); + extractOp(vertexObject.getJSONObject(key), null); } else { throw new Exception("Unsupported operator tree in vertex " + this.name); } @@ -148,7 +153,8 @@ public void extractOpTree() throws JSONException, JsonParseException, JsonMappin } /** - * @param operator + * @param object + * @param isInput * @param parent * @return * @throws JSONException @@ -159,31 +165,31 @@ public void extractOpTree() throws JSONException, JsonParseException, JsonMappin * assumption: each operator only has one parent but may have many * children */ - Op extractOp(JSONObject operator) throws JSONException, JsonParseException, JsonMappingException, + Op extractOp(JSONObject object, Op parent) throws JSONException, JsonParseException, JsonMappingException, IOException, Exception { - String[] names = JSONObject.getNames(operator); + String[] names = JSONObject.getNames(object); if (names.length != 1) { - throw new Exception("Expect only one operator in " + operator.toString()); + throw new Exception("Expect only one operator in " + object.toString()); } else { String opName = names[0]; - JSONObject attrObj = (JSONObject) operator.get(opName); + JSONObject attrObj = (JSONObject) object.get(opName); Map attrs = new TreeMap<>(); List children = new ArrayList<>(); - String id = null; - String outputVertexName = null; + Op op = new Op(opName, null, null, parent, children, attrs, attrObj, this, parser); + if (JSONObject.getNames(attrObj) != null) { for (String attrName : JSONObject.getNames(attrObj)) { if (attrName.equals("children")) { Object childrenObj = attrObj.get(attrName); if (childrenObj instanceof JSONObject) { if (((JSONObject) childrenObj).length() != 0) { - children.add(extractOp((JSONObject) childrenObj)); + children.add(extractOp((JSONObject) childrenObj, op)); } } else if (childrenObj instanceof JSONArray) { if (((JSONArray) childrenObj).length() != 0) { JSONArray array = ((JSONArray) childrenObj); for (int index = 0; index < array.length(); index++) { - children.add(extractOp(array.getJSONObject(index))); + children.add(extractOp(array.getJSONObject(index), op)); } } } else { @@ -192,9 +198,9 @@ Op extractOp(JSONObject operator) throws JSONException, JsonParseException, Json } } else { if (attrName.equals("OperatorId:")) { - id = attrObj.get(attrName).toString(); + op.setOperatorId(attrObj.get(attrName).toString()); } else if (attrName.equals("outputname:")) { - outputVertexName = attrObj.get(attrName).toString(); + op.outputVertexName = attrObj.get(attrName).toString(); } else { if (!attrObj.get(attrName).toString().isEmpty()) { attrs.put(attrName, attrObj.get(attrName).toString()); @@ -203,13 +209,11 @@ Op extractOp(JSONObject operator) throws JSONException, JsonParseException, Json } } } - Op op = new Op(opName, id, outputVertexName, children, attrs, operator, this, parser); - if (!children.isEmpty()) { - for (Op child : children) { - child.parent = op; - } - } else { - this.rootOps.add(op); + if (parent == null) { + this.inputOps.add(op); + } + if (children.isEmpty()) { + this.outputOps.add(op); } return op; } @@ -239,7 +243,7 @@ public void print(Printer printer, int indentFlag, String type, Vertex callingVe if (numReduceOp > 1 && !(callingVertex.vertexType == VertexType.UNION)) { // find the right op Op choose = null; - for (Op op : this.rootOps) { + for (Op op : this.outputOps) { if (op.outputVertexName.equals(callingVertex.name)) { choose = op; } @@ -250,7 +254,7 @@ public void print(Printer printer, int indentFlag, String type, Vertex callingVe throw new Exception("Can not find the right reduce output operator for vertex " + this.name); } } else { - for (Op op : this.rootOps) { + for (Op op : this.outputOps) { // dummy vertex is treated as a branch of a join operator if (this.dummy) { op.print(printer, indentFlag, true); @@ -271,15 +275,24 @@ public void print(Printer printer, int indentFlag, String type, Vertex callingVe /** * We check if a vertex has multiple reduce operators. + * @throws JSONException */ - public void checkMultiReduceOperator() { + public void checkMultiReduceOperator(boolean rewriteObject) throws JSONException { // check if it is a reduce vertex and its children is more than 1; - if (this.rootOps.size() < 2) { - return; - } // check if all the child ops are reduce output operators - for (Op op : this.rootOps) { + numReduceOp = 0; + for (Op op : this.outputOps) { if (op.type == OpType.RS) { + if (rewriteObject) { + Vertex outputVertex = this.stage.vertexs.get(op.outputVertexName); + if (outputVertex != null && outputVertex.inputOps.size() > 0) { + JSONArray array = new JSONArray(); + for (Op inputOp : outputVertex.inputOps) { + array.put(inputOp.operatorId); + } + op.opObject.put("outputOperator:", array); + } + } numReduceOp++; } } @@ -301,16 +314,16 @@ public int compareTo(Vertex o) { } public Op getJoinRSOp(Vertex joinVertex) { - if (rootOps.size() == 0) { + if (outputOps.size() == 0) { return null; - } else if (rootOps.size() == 1) { - if (rootOps.get(0).type == OpType.RS) { - return rootOps.get(0); + } else if (outputOps.size() == 1) { + if (outputOps.get(0).type == OpType.RS) { + return outputOps.get(0); } else { return null; } } else { - for (Op op : rootOps) { + for (Op op : outputOps) { if (op.type == OpType.RS) { if (op.outputVertexName.equals(joinVertex.name)) { return op; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java index 4c24ab4cde..e0833dc247 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java @@ -54,11 +54,9 @@ import org.apache.hadoop.hive.ql.exec.spark.SparkTask; import org.apache.hadoop.hive.ql.exec.tez.TezTask; import org.apache.hadoop.hive.ql.exec.vector.VectorGroupByOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression; -import org.apache.hadoop.hive.ql.exec.vector.reducesink.VectorReduceSinkCommonOperator; import org.apache.hadoop.hive.ql.plan.MapJoinDesc; import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; import org.apache.hadoop.hive.ql.hooks.ReadEntity; @@ -117,7 +115,6 @@ public class ExplainTask extends Task implements Serializable { private static final long serialVersionUID = 1L; public static final String EXPL_COLUMN_NAME = "Explain"; - public static final String OUTPUT_OPERATORS = "OutputOperators:"; private final Set> visitedOps = new HashSet>(); private boolean isLogical = false; protected final Logger LOG; @@ -400,6 +397,11 @@ public int execute(DriverContext driverContext) { } else { JSONObject jsonPlan = getJSONPlan(out, work); if (work.isFormatted()) { + // use the parser to get the output operators of RS + JsonParser jsonParser = JsonParserFactory.getParser(conf); + if (jsonParser != null) { + jsonParser.print(jsonPlan, null); + } out.print(jsonPlan); } } @@ -797,17 +799,11 @@ private JSONObject outputPlan(Object work, PrintStream out, if (jsonOut != null && jsonOut.length() > 0) { ((JSONObject) jsonOut.get(JSONObject.getNames(jsonOut)[0])).put("OperatorId:", operator.getOperatorId()); - if (!this.work.isUserLevelExplain() - && this.work.isFormatted() - && (operator instanceof ReduceSinkOperator - || operator instanceof VectorReduceSinkOperator || operator instanceof VectorReduceSinkCommonOperator)) { - List outputOperators = ((ReduceSinkDesc) operator.getConf()) - .getOutputOperators(); - if (outputOperators != null) { - ((JSONObject) jsonOut.get(JSONObject.getNames(jsonOut)[0])).put(OUTPUT_OPERATORS, - Arrays.toString(outputOperators.toArray())); - } - } + } + if (!this.work.isUserLevelExplain() && this.work.isFormatted() + && operator.getConf() instanceof ReduceSinkDesc ) { + ((JSONObject) jsonOut.get(JSONObject.getNames(jsonOut)[0])).put("outputname:", + ((ReduceSinkDesc) operator.getConf()).getOutputName()); } } if (jsonOutput) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java index afe14841ba..af1fa66b6b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java @@ -38,7 +38,6 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorSparkHashTableSinkOperator; import org.apache.hadoop.hive.ql.exec.vector.VectorSparkPartitionPruningSinkOperator; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; -import org.apache.hadoop.hive.ql.exec.vector.reducesink.VectorReduceSinkCommonOperator; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.optimizer.spark.SparkPartitionPruningSinkDesc; import org.apache.hadoop.hive.ql.parse.spark.SparkPartitionPruningSinkOperator; @@ -144,17 +143,13 @@ public static Operator getVectorOperator( Class> opClass, CompilationOpContext cContext, T conf, - VectorizationContext vContext, Operator originalOp) throws HiveException { + VectorizationContext vContext) throws HiveException { try { VectorDesc vectorDesc = ((AbstractOperatorDesc) conf).getVectorDesc(); vectorDesc.setVectorOp(opClass); - Operator op = (Operator) opClass.getDeclaredConstructor(CompilationOpContext.class, - VectorizationContext.class, OperatorDesc.class).newInstance(cContext, vContext, conf); - op.setOperatorId(originalOp.getOperatorId()); - if (op instanceof VectorReduceSinkOperator || op instanceof VectorReduceSinkCommonOperator) { - ((ReduceSinkDesc) op.getConf()).setOutputOperators(((ReduceSinkDesc) originalOp.getConf()) - .getOutputOperators()); - } + Operator op = (Operator) opClass.getDeclaredConstructor( + CompilationOpContext.class, VectorizationContext.class, OperatorDesc.class) + .newInstance(cContext, vContext, conf); return op; } catch (Exception e) { e.printStackTrace(); @@ -163,12 +158,11 @@ } public static Operator getVectorOperator( - CompilationOpContext cContext, T conf, VectorizationContext vContext, - Operator originalOp) throws HiveException { + CompilationOpContext cContext, T conf, VectorizationContext vContext) throws HiveException { Class descClass = (Class) conf.getClass(); - Class> opClass = vectorOpvec.get(descClass); + Class opClass = vectorOpvec.get(descClass); if (opClass != null) { - return getVectorOperator(opClass, cContext, conf, vContext, originalOp); + return getVectorOperator(vectorOpvec.get(descClass), cContext, conf, vContext); } throw new HiveException("No vector operator for descriptor class " + descClass.getName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AnnotateReduceSinkOutputOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AnnotateReduceSinkOutputOperator.java deleted file mode 100644 index 0b61f4baa0..0000000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AnnotateReduceSinkOutputOperator.java +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.optimizer; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Stack; - -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; -import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; -import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; -import org.apache.hadoop.hive.ql.lib.GraphWalker; -import org.apache.hadoop.hive.ql.lib.Node; -import org.apache.hadoop.hive.ql.lib.NodeProcessor; -import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; -import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; -import org.apache.hadoop.hive.ql.parse.ParseContext; -import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.OperatorDesc; - -/** - * Implementation of AnnotateReduceSinkOutputOperator optimization step. - */ -public class AnnotateReduceSinkOutputOperator extends Transform { - @Override - public ParseContext transform(ParseContext pctx) throws SemanticException { - - // 1. We apply the transformation - Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", - "(" + ReduceSinkOperator.getOperatorName() + "%)"), new ReduceSinkOutputOperatorAnnotator()); - GraphWalker ogw = new DefaultGraphWalker(new DefaultRuleDispatcher(null, opRules, null)); - ArrayList topNodes = new ArrayList(); - topNodes.addAll(pctx.getTopOps().values()); - ogw.startWalking(topNodes, null); - return pctx; - } - - private static class ReduceSinkOutputOperatorAnnotator implements NodeProcessor { - @Override - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - ReduceSinkOperator rs = (ReduceSinkOperator) nd; - List> children = rs.getChildOperators(); - List outputOperators = new ArrayList<>(); - for (Operator operator : children) { - outputOperators.add(operator.getOperatorId()); - } - rs.getConf().setOutputOperators(outputOperators); - return null; - } - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java index e1a69526bc..3dc71f013e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java @@ -662,14 +662,6 @@ private boolean generateSemiJoinOperatorPlan(DynamicListContext ctx, ParseContex groupByDescFinal, new RowSchema(rsOp.getSchema()), rsOp); groupByOpFinal.setColumnExprMap(new HashMap()); - // for explain purpose - if (parseContext.getContext().getExplainConfig() != null - && parseContext.getContext().getExplainConfig().isFormatted()) { - List outputOperators = new ArrayList<>(); - outputOperators.add(groupByOpFinal.getOperatorId()); - rsOp.getConf().setOutputOperators(outputOperators); - } - createFinalRsForSemiJoinOp(parseContext, ts, groupByOpFinal, key, keyBaseAlias, ctx.parent.getChildren().get(0), sjHint != null); @@ -712,16 +704,6 @@ private void createFinalRsForSemiJoinOp( SemiJoinBranchInfo sjInfo = new SemiJoinBranchInfo(ts, isHint); parseContext.getRsToSemiJoinBranchInfo().put(rsOpFinal, sjInfo); - // for explain purpose - if (parseContext.getContext().getExplainConfig() != null && - parseContext.getContext().getExplainConfig().isFormatted()) { - List outputOperators = rsOpFinal.getConf().getOutputOperators(); - if (outputOperators == null) { - outputOperators = new ArrayList<>(); - } - outputOperators.add(ts.getOperatorId()); - } - // Save the info that is required at query time to resolve dynamic/runtime values. RuntimeValuesInfo runtimeValuesInfo = new RuntimeValuesInfo(); TableDesc rsFinalTableDesc = PlanUtils.getReduceValueTableDesc( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java index 92225acd58..7dace9076f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java @@ -232,10 +232,6 @@ public void initialize(HiveConf hiveConf) { transformations.add(new SimpleFetchAggregation()); } - if (pctx.getContext().getExplainConfig() != null - && pctx.getContext().getExplainConfig().isFormatted()) { - transformations.add(new AnnotateReduceSinkOutputOperator()); - } } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index f0df2e9890..243ef105cc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -2530,7 +2530,7 @@ private boolean isBigTableOnlyResults(MapJoinDesc desc) { vectorDesc.setVectorMapJoinInfo(vectorMapJoinInfo); vectorOp = OperatorFactory.getVectorOperator( - opClass, op.getCompilationOpContext(), op.getConf(), vContext, op); + opClass, op.getCompilationOpContext(), op.getConf(), vContext); LOG.info("Vectorizer vectorizeOperator map join class " + vectorOp.getClass().getSimpleName()); return vectorOp; @@ -2995,7 +2995,7 @@ private boolean canSpecializeMapJoin(Operator op, MapJoi Operator vectorOp = null; try { vectorOp = OperatorFactory.getVectorOperator( - opClass, op.getCompilationOpContext(), op.getConf(), vContext, op); + opClass, op.getCompilationOpContext(), op.getConf(), vContext); } catch (Exception e) { LOG.info("Vectorizer vectorizeOperator reduce sink class exception " + opClass.getSimpleName() + " exception " + e); @@ -3273,7 +3273,7 @@ private boolean usesVectorUDFAdaptor(VectorExpression[] vecExprs) { vContext.getVectorExpression(predicateExpr, VectorExpressionDescriptor.Mode.FILTER); vectorFilterDesc.setPredicateExpression(vectorPredicateExpr); return OperatorFactory.getVectorOperator( - filterOp.getCompilationOpContext(), filterDesc, vContext, filterOp); + filterOp.getCompilationOpContext(), filterDesc, vContext); } /* @@ -3301,7 +3301,7 @@ private boolean usesVectorUDFAdaptor(VectorExpression[] vecExprs) { vectorGroupByDesc.setAggregators(vecAggregators); vectorGroupByDesc.setProjectedOutputColumns(projectedOutputColumns); return OperatorFactory.getVectorOperator( - groupByOp.getCompilationOpContext(), groupByDesc, vContext, groupByOp); + groupByOp.getCompilationOpContext(), groupByDesc, vContext); } public static Operator vectorizeSelectOperator( @@ -3331,7 +3331,7 @@ private boolean usesVectorUDFAdaptor(VectorExpression[] vecExprs) { vectorSelectDesc.setSelectExpressions(vectorSelectExprs); vectorSelectDesc.setProjectedOutputColumns(projectedOutputColumns); return OperatorFactory.getVectorOperator( - selectOp.getCompilationOpContext(), selectDesc, vContext, selectOp); + selectOp.getCompilationOpContext(), selectDesc, vContext); } public Operator vectorizeOperator(Operator op, @@ -3367,7 +3367,7 @@ private boolean usesVectorUDFAdaptor(VectorExpression[] vecExprs) { } vectorOp = OperatorFactory.getVectorOperator( - opClass, op.getCompilationOpContext(), op.getConf(), vContext, op); + opClass, op.getCompilationOpContext(), op.getConf(), vContext); isNative = false; } else { @@ -3393,7 +3393,7 @@ private boolean usesVectorUDFAdaptor(VectorExpression[] vecExprs) { VectorSMBJoinDesc vectorSMBJoinDesc = new VectorSMBJoinDesc(); smbJoinSinkDesc.setVectorDesc(vectorSMBJoinDesc); vectorOp = OperatorFactory.getVectorOperator( - op.getCompilationOpContext(), smbJoinSinkDesc, vContext, op); + op.getCompilationOpContext(), smbJoinSinkDesc, vContext); isNative = false; } } @@ -3408,7 +3408,7 @@ private boolean usesVectorUDFAdaptor(VectorExpression[] vecExprs) { if (!specialize) { vectorOp = OperatorFactory.getVectorOperator( - op.getCompilationOpContext(), op.getConf(), vContext, op); + op.getCompilationOpContext(), op.getConf(), vContext); isNative = false; } else { @@ -3484,7 +3484,7 @@ private boolean usesVectorUDFAdaptor(VectorExpression[] vecExprs) { VectorFileSinkDesc vectorFileSinkDesc = new VectorFileSinkDesc(); fileSinkDesc.setVectorDesc(vectorFileSinkDesc); vectorOp = OperatorFactory.getVectorOperator( - op.getCompilationOpContext(), fileSinkDesc, vContext, op); + op.getCompilationOpContext(), fileSinkDesc, vContext); isNative = false; } break; @@ -3494,7 +3494,7 @@ private boolean usesVectorUDFAdaptor(VectorExpression[] vecExprs) { VectorLimitDesc vectorLimitDesc = new VectorLimitDesc(); limitDesc.setVectorDesc(vectorLimitDesc); vectorOp = OperatorFactory.getVectorOperator( - op.getCompilationOpContext(), limitDesc, vContext, op); + op.getCompilationOpContext(), limitDesc, vContext); isNative = true; } break; @@ -3504,7 +3504,7 @@ private boolean usesVectorUDFAdaptor(VectorExpression[] vecExprs) { VectorAppMasterEventDesc vectorEventDesc = new VectorAppMasterEventDesc(); eventDesc.setVectorDesc(vectorEventDesc); vectorOp = OperatorFactory.getVectorOperator( - op.getCompilationOpContext(), eventDesc, vContext, op); + op.getCompilationOpContext(), eventDesc, vContext); isNative = true; } break; @@ -3514,7 +3514,7 @@ private boolean usesVectorUDFAdaptor(VectorExpression[] vecExprs) { VectorSparkHashTableSinkDesc vectorSparkHashTableSinkDesc = new VectorSparkHashTableSinkDesc(); sparkHashTableSinkDesc.setVectorDesc(vectorSparkHashTableSinkDesc); vectorOp = OperatorFactory.getVectorOperator( - op.getCompilationOpContext(), sparkHashTableSinkDesc, vContext, op); + op.getCompilationOpContext(), sparkHashTableSinkDesc, vContext); isNative = true; } break; @@ -3524,7 +3524,7 @@ private boolean usesVectorUDFAdaptor(VectorExpression[] vecExprs) { VectorSparkPartitionPruningSinkDesc vectorSparkPartitionPruningSinkDesc = new VectorSparkPartitionPruningSinkDesc(); sparkPartitionPruningSinkDesc.setVectorDesc(vectorSparkPartitionPruningSinkDesc); vectorOp = OperatorFactory.getVectorOperator( - op.getCompilationOpContext(), sparkPartitionPruningSinkDesc, vContext, op); + op.getCompilationOpContext(), sparkPartitionPruningSinkDesc, vContext); isNative = true; } break; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java index 79d19b5c71..6f355d865c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java @@ -78,12 +78,6 @@ private String outputName; /** - * Holds the name of the output operators - * that this reduce sink is outputing to. - */ - private List outputOperators; - - /** * The partition columns (CLUSTER BY or DISTRIBUTE BY in Hive language). * Partition columns decide the reducer that the current row goes to. * Partition columns are not passed to reducer. @@ -188,6 +182,7 @@ public Object clone() { throw new RuntimeException("Clone with vectorization desc not supported"); } desc.vectorDesc = null; + desc.outputName = outputName; return desc; } @@ -592,12 +587,4 @@ public ReduceSinkOperatorExplainVectorization getReduceSinkVectorization() { } return new ReduceSinkOperatorExplainVectorization(this, vectorDesc); } - - public List getOutputOperators() { - return outputOperators; - } - - public void setOutputOperators(List outputOperators) { - this.outputOperators = outputOperators; - } } diff --git a/ql/src/test/queries/clientpositive/explain_formatted_oid.q b/ql/src/test/queries/clientpositive/explain_formatted_oid.q deleted file mode 100644 index 932f1196ee..0000000000 --- a/ql/src/test/queries/clientpositive/explain_formatted_oid.q +++ /dev/null @@ -1,18 +0,0 @@ -set hive.auto.convert.join=false; - -create table srcTable (key string, value string); - -explain formatted -SELECT x.key, z.value, y.value -FROM srcTable x JOIN srcTable y ON (x.key = y.key) -JOIN srcTable z ON (x.value = z.value); - -explain formatted -SELECT x.key, z.value, y.value -FROM srcTable x JOIN srcTable y ON (x.key = y.key) -JOIN (select * from srcTable union select * from srcTable)z ON (x.value = z.value) -union -SELECT x.key, z.value, y.value -FROM srcTable x JOIN srcTable y ON (x.key = y.key) -JOIN (select * from srcTable union select * from srcTable)z ON (x.value = z.value); - diff --git a/ql/src/test/results/clientpositive/explain_formatted_oid.q.out b/ql/src/test/results/clientpositive/explain_formatted_oid.q.out deleted file mode 100644 index 46c2090cf6..0000000000 --- a/ql/src/test/results/clientpositive/explain_formatted_oid.q.out +++ /dev/null @@ -1,38 +0,0 @@ -PREHOOK: query: create table srcTable (key string, value string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@srcTable -POSTHOOK: query: create table srcTable (key string, value string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@srcTable -PREHOOK: query: explain formatted -SELECT x.key, z.value, y.value -FROM srcTable x JOIN srcTable y ON (x.key = y.key) -JOIN srcTable z ON (x.value = z.value) -PREHOOK: type: QUERY -POSTHOOK: query: explain formatted -SELECT x.key, z.value, y.value -FROM srcTable x JOIN srcTable y ON (x.key = y.key) -JOIN srcTable z ON (x.value = z.value) -POSTHOOK: type: QUERY -{"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-2":{"DEPENDENT STAGES":"Stage-1"},"Stage-0":{"DEPENDENT STAGES":"Stage-2"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"x","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_0","children":{"Filter Operator":{"predicate:":"(key is not null and value is not null) (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_17","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_2","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","value expressions:":"_col1 (type: string)","OperatorId:":"RS_9","OutputOperators:":"[JOIN_11]"}}}}}}}},{"TableScan":{"alias:":"y","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_3","children":{"Filter Operator":{"predicate:":"key is not null (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_18","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_5","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","value expressions:":"_col1 (type: string)","OperatorId:":"RS_10","OutputOperators:":"[JOIN_11]"}}}}}}}}],"Reduce Operator Tree:":{"Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{"0":"_col0 (type: string)","1":"_col0 (type: string)"},"outputColumnNames:":["_col0","_col1","_col3"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"JOIN_11","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_20"}}}}}},"Stage-2":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"OperatorId:":"TS_21","children":{"Reduce Output Operator":{"key expressions:":"_col1 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col1 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","value expressions:":"_col0 (type: string), _col3 (type: string)","OperatorId:":"RS_12","OutputOperators:":"[JOIN_14]"}}}},{"TableScan":{"alias:":"z","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_6","children":{"Filter Operator":{"predicate:":"value is not null (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_19","children":{"Select Operator":{"expressions:":"value (type: string)","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_8","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_13","OutputOperators:":"[JOIN_14]"}}}}}}}}],"Reduce Operator Tree:":{"Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"outputColumnNames:":["_col0","_col3","_col4"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"JOIN_14","children":{"Select Operator":{"expressions:":"_col0 (type: string), _col4 (type: string), _col3 (type: string)","outputColumnNames:":["_col0","_col1","_col2"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_15","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_16"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_22"}}}}}} -PREHOOK: query: explain formatted -SELECT x.key, z.value, y.value -FROM srcTable x JOIN srcTable y ON (x.key = y.key) -JOIN (select * from srcTable union select * from srcTable)z ON (x.value = z.value) -union -SELECT x.key, z.value, y.value -FROM srcTable x JOIN srcTable y ON (x.key = y.key) -JOIN (select * from srcTable union select * from srcTable)z ON (x.value = z.value) -PREHOOK: type: QUERY -POSTHOOK: query: explain formatted -SELECT x.key, z.value, y.value -FROM srcTable x JOIN srcTable y ON (x.key = y.key) -JOIN (select * from srcTable union select * from srcTable)z ON (x.value = z.value) -union -SELECT x.key, z.value, y.value -FROM srcTable x JOIN srcTable y ON (x.key = y.key) -JOIN (select * from srcTable union select * from srcTable)z ON (x.value = z.value) -POSTHOOK: type: QUERY -{"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-2":{"DEPENDENT STAGES":"Stage-1, Stage-5"},"Stage-3":{"DEPENDENT STAGES":"Stage-2, Stage-8"},"Stage-5":{"ROOT STAGE":"TRUE"},"Stage-7":{"ROOT STAGE":"TRUE"},"Stage-8":{"DEPENDENT STAGES":"Stage-7, Stage-10"},"Stage-10":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"x","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_0","children":{"Filter Operator":{"predicate:":"(key is not null and value is not null) (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_60","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_2","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","value expressions:":"_col1 (type: string)","OperatorId:":"RS_19","OutputOperators:":"[JOIN_21]"}}}}}}}},{"TableScan":{"alias:":"y","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_3","children":{"Filter Operator":{"predicate:":"key is not null (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_61","children":{"Select Operator":{"expressions:":"key (type: string)","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_5","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_20","OutputOperators:":"[JOIN_21]"}}}}}}}}],"Reduce Operator Tree:":{"Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{"0":"_col0 (type: string)","1":"_col0 (type: string)"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"JOIN_21","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_68"}}}}}},"Stage-2":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"OperatorId:":"TS_69","children":{"Reduce Output Operator":{"key expressions:":"_col1 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col1 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","value expressions:":"_col0 (type: string)","OperatorId:":"RS_22","OutputOperators:":"[JOIN_24]"}}}},{"TableScan":{"OperatorId:":"TS_73","children":{"Reduce Output Operator":{"key expressions:":"_col1 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col1 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_23","OutputOperators:":"[JOIN_24]"}}}}],"Reduce Operator Tree:":{"Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col1 (type: string)"},"outputColumnNames:":["_col0","_col4"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"JOIN_24","children":{"Select Operator":{"expressions:":"_col0 (type: string), _col4 (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_25","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_70"}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"OperatorId:":"TS_71","children":{"Union":{"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"UNION_52","children":{"Group By Operator":{"keys:":"_col0 (type: string), _col1 (type: string)","mode:":"hash","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"GBY_55","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string), _col1 (type: string)","sort order:":"++","Map-reduce partition columns:":"_col0 (type: string), _col1 (type: string)","Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_56","OutputOperators:":"[GBY_57]"}}}}}}}},{"TableScan":{"OperatorId:":"TS_77","children":{"Union":{"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"UNION_52","children":{"Group By Operator":{"keys:":"_col0 (type: string), _col1 (type: string)","mode:":"hash","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"GBY_55","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string), _col1 (type: string)","sort order:":"++","Map-reduce partition columns:":"_col0 (type: string), _col1 (type: string)","Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_56","OutputOperators:":"[GBY_57]"}}}}}}}}],"Reduce Operator Tree:":{"Group By Operator":{"keys:":"KEY._col0 (type: string), KEY._col1 (type: string)","mode:":"mergepartial","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"GBY_57","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_59"}}}}}},"Stage-5":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"srctable","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_6","children":{"Filter Operator":{"predicate:":"value is not null (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_62","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_8","children":{"Union":{"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"UNION_12","children":{"Group By Operator":{"keys:":"_col1 (type: string), _col0 (type: string)","mode:":"hash","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"GBY_15","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string), _col1 (type: string)","sort order:":"++","Map-reduce partition columns:":"_col0 (type: string), _col1 (type: string)","Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_16","OutputOperators:":"[GBY_17]"}}}}}}}}}}}},{"TableScan":{"alias:":"srctable","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_9","children":{"Filter Operator":{"predicate:":"value is not null (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_63","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_11","children":{"Union":{"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"UNION_12","children":{"Group By Operator":{"keys:":"_col1 (type: string), _col0 (type: string)","mode:":"hash","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"GBY_15","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string), _col1 (type: string)","sort order:":"++","Map-reduce partition columns:":"_col0 (type: string), _col1 (type: string)","Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_16","OutputOperators:":"[GBY_17]"}}}}}}}}}}}}],"Reduce Operator Tree:":{"Group By Operator":{"keys:":"KEY._col0 (type: string), KEY._col1 (type: string)","mode:":"mergepartial","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"GBY_17","children":{"Select Operator":{"expressions:":"_col0 (type: string)","outputColumnNames:":["_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_18","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_72"}}}}}}}},"Stage-7":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"x","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_26","children":{"Filter Operator":{"predicate:":"(key is not null and value is not null) (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_64","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_28","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","value expressions:":"_col1 (type: string)","OperatorId:":"RS_45","OutputOperators:":"[JOIN_47]"}}}}}}}},{"TableScan":{"alias:":"y","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_29","children":{"Filter Operator":{"predicate:":"key is not null (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_65","children":{"Select Operator":{"expressions:":"key (type: string)","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_31","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_46","OutputOperators:":"[JOIN_47]"}}}}}}}}],"Reduce Operator Tree:":{"Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{"0":"_col0 (type: string)","1":"_col0 (type: string)"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"JOIN_47","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_74"}}}}}},"Stage-8":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"OperatorId:":"TS_75","children":{"Reduce Output Operator":{"key expressions:":"_col1 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col1 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","value expressions:":"_col0 (type: string)","OperatorId:":"RS_48","OutputOperators:":"[JOIN_50]"}}}},{"TableScan":{"OperatorId:":"TS_79","children":{"Reduce Output Operator":{"key expressions:":"_col1 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col1 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_49","OutputOperators:":"[JOIN_50]"}}}}],"Reduce Operator Tree:":{"Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col1 (type: string)"},"outputColumnNames:":["_col0","_col4"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"JOIN_50","children":{"Select Operator":{"expressions:":"_col0 (type: string), _col4 (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_51","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_76"}}}}}}}},"Stage-10":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"srctable","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_32","children":{"Filter Operator":{"predicate:":"value is not null (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_66","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_34","children":{"Union":{"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"UNION_38","children":{"Group By Operator":{"keys:":"_col1 (type: string), _col0 (type: string)","mode:":"hash","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"GBY_41","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string), _col1 (type: string)","sort order:":"++","Map-reduce partition columns:":"_col0 (type: string), _col1 (type: string)","Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_42","OutputOperators:":"[GBY_43]"}}}}}}}}}}}},{"TableScan":{"alias:":"srctable","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_35","children":{"Filter Operator":{"predicate:":"value is not null (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_67","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_37","children":{"Union":{"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"UNION_38","children":{"Group By Operator":{"keys:":"_col1 (type: string), _col0 (type: string)","mode:":"hash","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"GBY_41","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string), _col1 (type: string)","sort order:":"++","Map-reduce partition columns:":"_col0 (type: string), _col1 (type: string)","Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_42","OutputOperators:":"[GBY_43]"}}}}}}}}}}}}],"Reduce Operator Tree:":{"Group By Operator":{"keys:":"KEY._col0 (type: string), KEY._col1 (type: string)","mode:":"mergepartial","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"GBY_43","children":{"Select Operator":{"expressions:":"_col0 (type: string)","outputColumnNames:":["_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_44","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_78"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_80"}}}}}} diff --git a/ql/src/test/results/clientpositive/input4.q.out b/ql/src/test/results/clientpositive/input4.q.out index 69843183a5..83912f6608 100644 --- a/ql/src/test/results/clientpositive/input4.q.out +++ b/ql/src/test/results/clientpositive/input4.q.out @@ -44,7 +44,7 @@ PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN FORMATTED SELECT Input4Alias.VALUE, Input4Alias.KEY FROM INPUT4 AS Input4Alias POSTHOOK: type: QUERY -{"STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"TableScan":{"alias:":"input4alias","Statistics:":"Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"value (type: string), key (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_1","children":{"ListSink":{"OperatorId:":"LIST_SINK_3"}}}}}}}}}} +{"STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"TableScan":{"alias:":"input4alias","Statistics:":"Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE","children":{"Select Operator":{"expressions:":"value (type: string), key (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE","children":{"ListSink":{}}}}}}}}}} PREHOOK: query: SELECT Input4Alias.VALUE, Input4Alias.KEY FROM INPUT4 AS Input4Alias PREHOOK: type: QUERY PREHOOK: Input: default@input4 diff --git a/ql/src/test/results/clientpositive/join0.q.out b/ql/src/test/results/clientpositive/join0.q.out index b1e85a39da..c02319e7de 100644 --- a/ql/src/test/results/clientpositive/join0.q.out +++ b/ql/src/test/results/clientpositive/join0.q.out @@ -112,7 +112,7 @@ SELECT src1.key as k1, src1.value as v1, (SELECT * FROM src WHERE src.key < 10) src2 SORT BY k1, v1, k2, v2 POSTHOOK: type: QUERY -{"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-2":{"DEPENDENT STAGES":"Stage-1"},"Stage-0":{"DEPENDENT STAGES":"Stage-2"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"TS_0","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_13","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_2","children":{"Reduce Output Operator":{"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_6","OutputOperators:":"[JOIN_8]"}}}}}}}},{"TableScan":{"alias:":"src","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"TS_3","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_14","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_5","children":{"Reduce Output Operator":{"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_7","OutputOperators:":"[JOIN_8]"}}}}}}}}],"Reduce Operator Tree:":{"Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"JOIN_8","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_15"}}}}}},"Stage-2":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"OperatorId:":"TS_16","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)","sort order:":"++++","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"RS_10","OutputOperators:":"[SEL_11]"}}}}],"Reduce Operator Tree:":{"Select Operator":{"expressions:":"KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string)","outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_11","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_12"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_17"}}}}}} +{"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-2":{"DEPENDENT STAGES":"Stage-1"},"Stage-0":{"DEPENDENT STAGES":"Stage-2"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","children":{"Reduce Output Operator":{"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)"}}}}}}}},{"TableScan":{"alias:":"src","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","children":{"Reduce Output Operator":{"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)"}}}}}}}}],"Reduce Operator Tree:":{"Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"}}}}}}},"Stage-2":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)","sort order:":"++++","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE"}}}}],"Reduce Operator Tree:":{"Select Operator":{"expressions:":"KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string)","outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{}}}}}} Warning: Shuffle Join JOIN[8][tables = [src1, src2]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 FROM diff --git a/ql/src/test/results/clientpositive/parallel_join0.q.out b/ql/src/test/results/clientpositive/parallel_join0.q.out index b1e85a39da..c02319e7de 100644 --- a/ql/src/test/results/clientpositive/parallel_join0.q.out +++ b/ql/src/test/results/clientpositive/parallel_join0.q.out @@ -112,7 +112,7 @@ SELECT src1.key as k1, src1.value as v1, (SELECT * FROM src WHERE src.key < 10) src2 SORT BY k1, v1, k2, v2 POSTHOOK: type: QUERY -{"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-2":{"DEPENDENT STAGES":"Stage-1"},"Stage-0":{"DEPENDENT STAGES":"Stage-2"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"TS_0","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_13","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_2","children":{"Reduce Output Operator":{"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_6","OutputOperators:":"[JOIN_8]"}}}}}}}},{"TableScan":{"alias:":"src","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"TS_3","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_14","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_5","children":{"Reduce Output Operator":{"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_7","OutputOperators:":"[JOIN_8]"}}}}}}}}],"Reduce Operator Tree:":{"Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"JOIN_8","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_15"}}}}}},"Stage-2":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"OperatorId:":"TS_16","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)","sort order:":"++++","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"RS_10","OutputOperators:":"[SEL_11]"}}}}],"Reduce Operator Tree:":{"Select Operator":{"expressions:":"KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string)","outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_11","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_12"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_17"}}}}}} +{"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-2":{"DEPENDENT STAGES":"Stage-1"},"Stage-0":{"DEPENDENT STAGES":"Stage-2"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","children":{"Reduce Output Operator":{"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)"}}}}}}}},{"TableScan":{"alias:":"src","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","children":{"Reduce Output Operator":{"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)"}}}}}}}}],"Reduce Operator Tree:":{"Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"}}}}}}},"Stage-2":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)","sort order:":"++++","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE"}}}}],"Reduce Operator Tree:":{"Select Operator":{"expressions:":"KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string)","outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{}}}}}} Warning: Shuffle Join JOIN[8][tables = [src1, src2]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 FROM diff --git a/ql/src/test/results/clientpositive/plan_json.q.out b/ql/src/test/results/clientpositive/plan_json.q.out index ba6d0becf0..75d5b73d88 100644 --- a/ql/src/test/results/clientpositive/plan_json.q.out +++ b/ql/src/test/results/clientpositive/plan_json.q.out @@ -2,4 +2,4 @@ PREHOOK: query: EXPLAIN FORMATTED SELECT count(1) FROM src PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN FORMATTED SELECT count(1) FROM src POSTHOOK: type: QUERY -{"STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_8"}}}}}} +{"STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"1","Processor Tree:":{"ListSink":{}}}}}}