diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Connection.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Connection.java new file mode 100644 index 0000000..03ee1e1 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Connection.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.jsonexplain; + +public final class Connection { + public final String type; + public final Vertex from; + + public Connection(String type, Vertex from) { + super(); + this.type = type; + this.from = from; + } +} diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/DagJsonParser.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/DagJsonParser.java new file mode 100644 index 0000000..1f01685 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/DagJsonParser.java @@ -0,0 +1,167 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.jsonexplain; + +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import org.apache.hadoop.hive.common.jsonexplain.JsonParser; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class DagJsonParser implements JsonParser { + public final Map stages = new LinkedHashMap<>(); + protected final Logger LOG; + // the objects that have been printed. + public final Set printSet = new LinkedHashSet<>(); + // the vertex that should be inlined. + public final Map> inlineMap = new LinkedHashMap<>(); + + public DagJsonParser() { + super(); + LOG = LoggerFactory.getLogger(this.getClass().getName()); + } + + public void extractStagesAndPlans(JSONObject inputObject) throws Exception { + // extract stages + JSONObject dependency = inputObject.getJSONObject("STAGE DEPENDENCIES"); + if (dependency != null && dependency.length() > 0) { + // iterate for the first time to get all the names of stages. + for (String stageName : JSONObject.getNames(dependency)) { + this.stages.put(stageName, new Stage(stageName, this)); + } + // iterate for the second time to get all the dependency. + for (String stageName : JSONObject.getNames(dependency)) { + JSONObject dependentStageNames = dependency.getJSONObject(stageName); + this.stages.get(stageName).addDependency(dependentStageNames, this.stages); + } + } + // extract stage plans + JSONObject stagePlans = inputObject.getJSONObject("STAGE PLANS"); + if (stagePlans != null && stagePlans.length() > 0) { + for (String stageName : JSONObject.getNames(stagePlans)) { + JSONObject stagePlan = stagePlans.getJSONObject(stageName); + this.stages.get(stageName).extractVertex(stagePlan); + } + } + } + + /** + * @param indentFlag + * help to generate correct indent + * @return + */ + public static String prefixString(int indentFlag) { + StringBuilder sb = new StringBuilder(); + for (int index = 0; index < indentFlag; index++) { + sb.append(" "); + } + return sb.toString(); + } + + /** + * @param indentFlag + * @param tail + * help to generate correct indent with a specific tail + * @return + */ + public static String prefixString(int indentFlag, String tail) { + StringBuilder sb = new StringBuilder(); + for (int index = 0; index < indentFlag; index++) { + sb.append(" "); + } + int len = sb.length(); + return sb.replace(len - tail.length(), len, tail).toString(); + } + + @Override + public void print(JSONObject inputObject, PrintStream outputStream) throws Exception { + LOG.info("JsonParser is parsing:" + inputObject.toString()); + this.extractStagesAndPlans(inputObject); + Printer printer = new Printer(); + // print out the cbo info + if (inputObject.has("cboInfo")) { + printer.println(inputObject.getString("cboInfo")); + printer.println(); + } + // print out the vertex dependency in root stage + for (Stage candidate : this.stages.values()) { + if (candidate.tezStageDependency != null && candidate.tezStageDependency.size() > 0) { + printer.println("Vertex dependency in root stage"); + for (Entry> entry : candidate.tezStageDependency.entrySet()) { + StringBuilder sb = new StringBuilder(); + sb.append(entry.getKey().name); + sb.append(" <- "); + boolean printcomma = false; + for (Connection connection : entry.getValue()) { + if (printcomma) { + sb.append(", "); + } else { + printcomma = true; + } + sb.append(connection.from.name + " (" + connection.type + ")"); + } + printer.println(sb.toString()); + } + printer.println(); + } + } + // print out all the stages that have no childStages. + for (Stage candidate : this.stages.values()) { + if (candidate.childStages.isEmpty()) { + candidate.print(printer, 0); + } + } + outputStream.println(printer.toString()); + } + + public void addInline(Op op, Connection connection) { + List list = inlineMap.get(op); + if (list == null) { + list = new ArrayList<>(); + list.add(connection); + inlineMap.put(op, list); + } else { + list.add(connection); + } + } + + public boolean isInline(Vertex v) { + for (List list : inlineMap.values()) { + for (Connection connection : list) { + if (connection.from.equals(v)) { + return true; + } + } + } + return false; + } + + public abstract String mapEdgeType(String edgeName); + + public abstract String getFrameworkName(); +} diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/DagJsonParserUtils.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/DagJsonParserUtils.java new file mode 100644 index 0000000..a518ac1 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/DagJsonParserUtils.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.jsonexplain; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + + +public class DagJsonParserUtils { + + public static List OperatorNoStats = Arrays.asList(new String[] { "File Output Operator", + "Reduce Output Operator" }); + + public static String renameReduceOutputOperator(String operatorName, Vertex vertex) { + if (operatorName.equals("Reduce Output Operator") && vertex.edgeType != null) { + return vertex.edgeType; + } else { + return operatorName; + } + } + + public static String attrsToString(Map attrs) { + StringBuffer sb = new StringBuffer(); + boolean first = true; + for (Entry entry : attrs.entrySet()) { + if (first) { + first = false; + } else { + sb.append(","); + } + sb.append(entry.getKey() + entry.getValue()); + } + return sb.toString(); + } +} diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/JsonParserFactory.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/JsonParserFactory.java index db118bf..2a5d47a 100644 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/JsonParserFactory.java +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/JsonParserFactory.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.common.jsonexplain; +import org.apache.hadoop.hive.common.jsonexplain.spark.SparkJsonParser; import org.apache.hadoop.hive.common.jsonexplain.tez.TezJsonParser; import org.apache.hadoop.hive.conf.HiveConf; @@ -35,6 +36,9 @@ public static JsonParser getParser(HiveConf conf) { if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) { return new TezJsonParser(); } + if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")) { + return new SparkJsonParser(); + } return null; } } diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Op.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Op.java new file mode 100644 index 0000000..3e1f354 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Op.java @@ -0,0 +1,358 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.jsonexplain; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import org.apache.hadoop.hive.common.jsonexplain.Vertex.VertexType; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +public final class Op { + public final String name; + // tezJsonParser + public final DagJsonParser parser; + public final String operatorId; + public Op parent; + public final List children; + public final Map attrs; + // the jsonObject for this operator + public final JSONObject opObject; + // the vertex that this operator belongs to + public final Vertex vertex; + // the vertex that this operator output to + public final String outputVertexName; + // the Operator type + public final OpType type; + + public enum OpType { + MAPJOIN, MERGEJOIN, RS, OTHERS + }; + + public Op(String name, String id, String outputVertexName, List children, + Map attrs, JSONObject opObject, Vertex vertex, DagJsonParser tezJsonParser) + throws JSONException { + super(); + this.name = name; + this.operatorId = id; + this.type = deriveOpType(operatorId); + this.outputVertexName = outputVertexName; + this.children = children; + this.attrs = attrs; + this.opObject = opObject; + this.vertex = vertex; + this.parser = tezJsonParser; + } + + private OpType deriveOpType(String operatorId) { + if (operatorId != null) { + if (operatorId.startsWith(OpType.MAPJOIN.toString())) { + return OpType.MAPJOIN; + } else if (operatorId.startsWith(OpType.MERGEJOIN.toString())) { + return OpType.MERGEJOIN; + } else if (operatorId.startsWith(OpType.RS.toString())) { + return OpType.RS; + } else { + return OpType.OTHERS; + } + } else { + return OpType.OTHERS; + } + } + + private void inlineJoinOp() throws Exception { + // inline map join operator + if (this.type == OpType.MAPJOIN) { + JSONObject joinObj = opObject.getJSONObject(this.name); + // get the map for posToVertex + Map posToVertex = new LinkedHashMap<>(); + if (joinObj.has("input vertices:")) { + JSONObject verticeObj = joinObj.getJSONObject("input vertices:"); + for (String pos : JSONObject.getNames(verticeObj)) { + String vertexName = verticeObj.getString(pos); + // update the connection + Connection c = null; + for (Connection connection : vertex.parentConnections) { + if (connection.from.name.equals(vertexName)) { + posToVertex.put(pos, connection.from); + c = connection; + break; + } + } + if (c != null) { + parser.addInline(this, c); + } + } + // update the attrs + this.attrs.remove("input vertices:"); + } + // update the keys to use operator name + JSONObject keys = joinObj.getJSONObject("keys:"); + // find out the vertex for the big table + Set parentVertexes = new HashSet<>(); + for (Connection connection : vertex.parentConnections) { + parentVertexes.add(connection.from); + } + parentVertexes.removeAll(posToVertex.values()); + Map posToOpId = new LinkedHashMap<>(); + if (keys.length() != 0) { + for (String key : JSONObject.getNames(keys)) { + // first search from the posToVertex + if (posToVertex.containsKey(key)) { + Vertex vertex = posToVertex.get(key); + if (vertex.rootOps.size() == 1) { + posToOpId.put(key, vertex.rootOps.get(0).operatorId); + } else if ((vertex.rootOps.size() == 0 && vertex.vertexType == VertexType.UNION)) { + posToOpId.put(key, vertex.name); + } else { + Op singleRSOp = vertex.getSingleRSOp(); + if (singleRSOp != null) { + posToOpId.put(key, singleRSOp.operatorId); + } else { + throw new Exception( + "There are none or more than one root operators in a single vertex " + + vertex.name + + " when hive explain user is trying to identify the operator id."); + } + } + } + // then search from parent + else if (parent != null) { + posToOpId.put(key, parent.operatorId); + } + // then assume it is from its own vertex + else if (parentVertexes.size() == 1) { + Vertex vertex = parentVertexes.iterator().next(); + parentVertexes.clear(); + if (vertex.rootOps.size() == 1) { + posToOpId.put(key, vertex.rootOps.get(0).operatorId); + } else if ((vertex.rootOps.size() == 0 && vertex.vertexType == VertexType.UNION)) { + posToOpId.put(key, vertex.name); + } else { + Op singleRSOp = vertex.getSingleRSOp(); + if (singleRSOp != null) { + posToOpId.put(key, singleRSOp.operatorId); + } else { + throw new Exception( + "There are none or more than one root operators in a single vertex " + + vertex.name + + " when hive explain user is trying to identify the operator id."); + } + } + } + // finally throw an exception + else { + throw new Exception( + "Can not find the source operator on one of the branches of map join."); + } + } + } + this.attrs.remove("keys:"); + StringBuffer sb = new StringBuffer(); + JSONArray conditionMap = joinObj.getJSONArray("condition map:"); + for (int index = 0; index < conditionMap.length(); index++) { + JSONObject cond = conditionMap.getJSONObject(index); + String k = (String) cond.keys().next(); + JSONObject condObject = new JSONObject((String)cond.get(k)); + String type = condObject.getString("type"); + String left = condObject.getString("left"); + String right = condObject.getString("right"); + if (keys.length() != 0) { + sb.append(posToOpId.get(left) + "." + keys.get(left) + "=" + posToOpId.get(right) + "." + + keys.get(right) + "(" + type + "),"); + } else { + // probably a cross product + sb.append("(" + type + "),"); + } + } + this.attrs.remove("condition map:"); + this.attrs.put("Conds:", sb.substring(0, sb.length() - 1)); + } + // should be merge join + else { + Map posToOpId = new LinkedHashMap<>(); + if (vertex.mergeJoinDummyVertexs.size() == 0) { + if (vertex.tagToInput.size() != vertex.parentConnections.size()) { + throw new Exception("tagToInput size " + vertex.tagToInput.size() + + " is different from parentConnections size " + vertex.parentConnections.size()); + } + for (Entry entry : vertex.tagToInput.entrySet()) { + Connection c = null; + for (Connection connection : vertex.parentConnections) { + if (connection.from.name.equals(entry.getValue())) { + Vertex v = connection.from; + if (v.rootOps.size() == 1) { + posToOpId.put(entry.getKey(), v.rootOps.get(0).operatorId); + } else if ((v.rootOps.size() == 0 && v.vertexType == VertexType.UNION)) { + posToOpId.put(entry.getKey(), v.name); + } else { + Op singleRSOp = v.getSingleRSOp(); + if (singleRSOp != null) { + posToOpId.put(entry.getKey(), singleRSOp.operatorId); + } else { + throw new Exception( + "There are none or more than one root operators in a single vertex " + v.name + + " when hive explain user is trying to identify the operator id."); + } + } + c = connection; + break; + } + } + if (c == null) { + throw new Exception("Can not find " + entry.getValue() + + " while parsing keys of merge join operator"); + } + } + } else { + posToOpId.put(vertex.tag, this.parent.operatorId); + for (Vertex v : vertex.mergeJoinDummyVertexs) { + if (v.rootOps.size() != 1) { + throw new Exception("Can not find a single root operators in a single vertex " + v.name + + " when hive explain user is trying to identify the operator id."); + } + posToOpId.put(v.tag, v.rootOps.get(0).operatorId); + } + } + JSONObject joinObj = opObject.getJSONObject(this.name); + // update the keys to use operator name + JSONObject keys = joinObj.getJSONObject("keys:"); + if (keys.length() != 0) { + for (String key : JSONObject.getNames(keys)) { + if (!posToOpId.containsKey(key)) { + throw new Exception( + "Can not find the source operator on one of the branches of merge join."); + } + } + // inline merge join operator in a self-join + if (this.vertex != null) { + for (Vertex v : this.vertex.mergeJoinDummyVertexs) { + parser.addInline(this, new Connection(null, v)); + } + } + } + // update the attrs + this.attrs.remove("keys:"); + StringBuffer sb = new StringBuffer(); + JSONArray conditionMap = joinObj.getJSONArray("condition map:"); + for (int index = 0; index < conditionMap.length(); index++) { + JSONObject cond = conditionMap.getJSONObject(index); + String k = (String) cond.keys().next(); + JSONObject condObject = new JSONObject((String)cond.get(k)); + String type = condObject.getString("type"); + String left = condObject.getString("left"); + String right = condObject.getString("right"); + if (keys.length() != 0) { + sb.append(posToOpId.get(left) + "." + keys.get(left) + "=" + posToOpId.get(right) + "." + + keys.get(right) + "(" + type + "),"); + } else { + // probably a cross product + sb.append("(" + type + "),"); + } + } + this.attrs.remove("condition map:"); + this.attrs.put("Conds:", sb.substring(0, sb.length() - 1)); + } + } + + private String getNameWithOpIdStats() { + StringBuffer sb = new StringBuffer(); + sb.append(DagJsonParserUtils.renameReduceOutputOperator(name, vertex)); + if (operatorId != null) { + sb.append(" [" + operatorId + "]"); + } + if (!DagJsonParserUtils.OperatorNoStats.contains(name) && attrs.containsKey("Statistics:")) { + sb.append(" (" + attrs.get("Statistics:") + ")"); + } + attrs.remove("Statistics:"); + return sb.toString(); + } + + /** + * @param printer + * @param indentFlag + * @param branchOfJoinOp + * This parameter is used to show if it is a branch of a Join + * operator so that we can decide the corresponding indent. + * @throws Exception + */ + public void print(Printer printer, int indentFlag, boolean branchOfJoinOp) throws Exception { + // print name + if (parser.printSet.contains(this)) { + printer.println(DagJsonParser.prefixString(indentFlag) + " Please refer to the previous " + + this.getNameWithOpIdStats()); + return; + } + parser.printSet.add(this); + if (!branchOfJoinOp) { + printer.println(DagJsonParser.prefixString(indentFlag) + this.getNameWithOpIdStats()); + } else { + printer.println(DagJsonParser.prefixString(indentFlag, "<-") + this.getNameWithOpIdStats()); + } + branchOfJoinOp = false; + // if this operator is a Map Join Operator or a Merge Join Operator + if (this.type == OpType.MAPJOIN || this.type == OpType.MERGEJOIN) { + inlineJoinOp(); + branchOfJoinOp = true; + } + // if this operator is the last operator, we summarize the non-inlined + // vertex + List noninlined = new ArrayList<>(); + if (this.parent == null) { + if (this.vertex != null) { + for (Connection connection : this.vertex.parentConnections) { + if (!parser.isInline(connection.from)) { + noninlined.add(connection); + } + } + } + } + // print attr + indentFlag++; + if (!attrs.isEmpty()) { + printer.println(DagJsonParser.prefixString(indentFlag) + + DagJsonParserUtils.attrsToString(attrs)); + } + // print inline vertex + if (parser.inlineMap.containsKey(this)) { + for (int index = 0; index < parser.inlineMap.get(this).size(); index++) { + Connection connection = parser.inlineMap.get(this).get(index); + connection.from.print(printer, indentFlag, connection.type, this.vertex); + } + } + // print parent op, i.e., where data comes from + if (this.parent != null) { + this.parent.print(printer, indentFlag, branchOfJoinOp); + } + // print next vertex + else { + for (int index = 0; index < noninlined.size(); index++) { + Vertex v = noninlined.get(index).from; + v.print(printer, indentFlag, noninlined.get(index).type, this.vertex); + } + } + } +} diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Printer.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Printer.java new file mode 100644 index 0000000..6f040f6 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Printer.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.jsonexplain; + +public final class Printer { + public static final String lineSeparator = System.getProperty("line.separator");; + private final StringBuilder builder = new StringBuilder(); + + public void print(String string) { + builder.append(string); + } + + public void println(String string) { + builder.append(string); + builder.append(lineSeparator); + } + + public void println() { + builder.append(lineSeparator); + } + + public String toString() { + return builder.toString(); + } +} diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Stage.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Stage.java new file mode 100644 index 0000000..d21a565 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Stage.java @@ -0,0 +1,262 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.jsonexplain; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.jsonexplain.Vertex.VertexType; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +public final class Stage { + //external name is used to show at the console + String externalName; + //internal name is used to track the stages + public final String internalName; + //tezJsonParser + public final DagJsonParser parser; + // upstream stages, e.g., root stage + public final List parentStages = new ArrayList<>(); + // downstream stages. + public final List childStages = new ArrayList<>(); + public final Map vertexs =new LinkedHashMap<>(); + public final Map attrs = new TreeMap<>(); + Map> tezStageDependency; + // some stage may contain only a single operator, e.g., create table operator, + // fetch operator. + Op op; + + public Stage(String name, DagJsonParser tezJsonParser) { + super(); + internalName = name; + externalName = name; + parser = tezJsonParser; + } + + public void addDependency(JSONObject object, Map stages) throws JSONException { + if (object.has("DEPENDENT STAGES")) { + String names = object.getString("DEPENDENT STAGES"); + for (String name : names.split(",")) { + Stage parent = stages.get(name.trim()); + this.parentStages.add(parent); + parent.childStages.add(this); + } + } + if (object.has("CONDITIONAL CHILD TASKS")) { + String names = object.getString("CONDITIONAL CHILD TASKS"); + this.externalName = this.internalName + "(CONDITIONAL CHILD TASKS: " + names + ")"; + for (String name : names.split(",")) { + Stage child = stages.get(name.trim()); + child.externalName = child.internalName + "(CONDITIONAL)"; + child.parentStages.add(this); + this.childStages.add(child); + } + } + } + + /** + * @param object + * @throws Exception + * If the object of stage contains "Tez", we need to extract the + * vertices and edges Else we need to directly extract operators + * and/or attributes. + */ + public void extractVertex(JSONObject object) throws Exception { + if (object.has(this.parser.getFrameworkName())) { + this.tezStageDependency = new TreeMap<>(); + JSONObject tez = (JSONObject) object.get(this.parser.getFrameworkName()); + JSONObject vertices = tez.getJSONObject("Vertices:"); + if (tez.has("Edges:")) { + JSONObject edges = tez.getJSONObject("Edges:"); + // iterate for the first time to get all the vertices + for (String to : JSONObject.getNames(edges)) { + vertexs.put(to, new Vertex(to, vertices.getJSONObject(to), parser)); + } + // iterate for the second time to get all the vertex dependency + for (String to : JSONObject.getNames(edges)) { + Object o = edges.get(to); + Vertex v = vertexs.get(to); + // 1 to 1 mapping + if (o instanceof JSONObject) { + JSONObject obj = (JSONObject) o; + String parent = obj.getString("parent"); + Vertex parentVertex = vertexs.get(parent); + if (parentVertex == null) { + parentVertex = new Vertex(parent, vertices.getJSONObject(parent), parser); + vertexs.put(parent, parentVertex); + } + String type = obj.getString("type"); + // for union vertex, we reverse the dependency relationship + if (!"CONTAINS".equals(type)) { + v.addDependency(new Connection(type, parentVertex)); + parentVertex.setType(type); + parentVertex.children.add(v); + } else { + parentVertex.addDependency(new Connection(type, v)); + v.children.add(parentVertex); + } + this.tezStageDependency.put(v, Arrays.asList(new Connection(type, parentVertex))); + } else { + // 1 to many mapping + JSONArray from = (JSONArray) o; + List list = new ArrayList<>(); + for (int index = 0; index < from.length(); index++) { + JSONObject obj = from.getJSONObject(index); + String parent = obj.getString("parent"); + Vertex parentVertex = vertexs.get(parent); + if (parentVertex == null) { + parentVertex = new Vertex(parent, vertices.getJSONObject(parent), parser); + vertexs.put(parent, parentVertex); + } + String type = obj.getString("type"); + if (!"CONTAINS".equals(type)) { + v.addDependency(new Connection(type, parentVertex)); + parentVertex.setType(type); + parentVertex.children.add(v); + } else { + parentVertex.addDependency(new Connection(type, v)); + v.children.add(parentVertex); + } + list.add(new Connection(type, parentVertex)); + } + this.tezStageDependency.put(v, list); + } + } + } else { + for (String vertexName : JSONObject.getNames(vertices)) { + vertexs.put(vertexName, new Vertex(vertexName, vertices.getJSONObject(vertexName), parser)); + } + } + // The opTree in vertex is extracted + for (Vertex v : vertexs.values()) { + if (v.vertexType == VertexType.MAP || v.vertexType == VertexType.REDUCE) { + v.extractOpTree(); + v.checkMultiReduceOperator(); + } + } + } else { + String[] names = JSONObject.getNames(object); + if (names != null) { + for (String name : names) { + if (name.contains("Operator")) { + this.op = extractOp(name, object.getJSONObject(name)); + } else { + if (!object.get(name).toString().isEmpty()) { + attrs.put(name, object.get(name).toString()); + } + } + } + } + } + } + + /** + * @param opName + * @param opObj + * @return + * @throws Exception + * This method address the create table operator, fetch operator, + * etc + */ + Op extractOp(String opName, JSONObject opObj) throws Exception { + Map attrs = new TreeMap<>(); + Vertex v = null; + if (opObj.length() > 0) { + String[] names = JSONObject.getNames(opObj); + for (String name : names) { + Object o = opObj.get(name); + if (isPrintable(o) && !o.toString().isEmpty()) { + attrs.put(name, o.toString()); + } else if (o instanceof JSONObject) { + JSONObject attrObj = (JSONObject) o; + if (attrObj.length() > 0) { + if (name.equals("Processor Tree:")) { + JSONObject object = new JSONObject(new LinkedHashMap<>()); + object.put(name, attrObj); + v = new Vertex(null, object, parser); + v.extractOpTree(); + } else { + for (String attrName : JSONObject.getNames(attrObj)) { + if (!attrObj.get(attrName).toString().isEmpty()) { + attrs.put(attrName, attrObj.get(attrName).toString()); + } + } + } + } + } else { + throw new Exception("Unsupported object in " + this.internalName); + } + } + } + Op op = new Op(opName, null, null, null, attrs, null, v, parser); + if (v != null) { + parser.addInline(op, new Connection(null, v)); + } + return op; + } + + private boolean isPrintable(Object val) { + if (val instanceof Boolean || val instanceof String || val instanceof Integer + || val instanceof Long || val instanceof Byte || val instanceof Float + || val instanceof Double || val instanceof Path) { + return true; + } + if (val != null && val.getClass().isPrimitive()) { + return true; + } + return false; + } + + public void print(Printer printer, int indentFlag) throws Exception { + // print stagename + if (parser.printSet.contains(this)) { + printer.println(DagJsonParser.prefixString(indentFlag) + " Please refer to the previous " + + externalName); + return; + } + parser.printSet.add(this); + printer.println(DagJsonParser.prefixString(indentFlag) + externalName); + // print vertexes + indentFlag++; + for (Vertex candidate : this.vertexs.values()) { + if (!parser.isInline(candidate) && candidate.children.isEmpty()) { + candidate.print(printer, indentFlag, null, null); + } + } + if (!attrs.isEmpty()) { + printer.println(DagJsonParser.prefixString(indentFlag) + + DagJsonParserUtils.attrsToString(attrs)); + } + if (op != null) { + op.print(printer, indentFlag, false); + } + indentFlag++; + // print dependent stages + for (Stage stage : this.parentStages) { + stage.print(printer, indentFlag); + } + } +} diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java new file mode 100644 index 0000000..e5283c2 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java @@ -0,0 +1,320 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.jsonexplain; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import org.apache.hadoop.hive.common.jsonexplain.Op.OpType; +import org.apache.hadoop.util.hash.Hash; +import org.codehaus.jackson.JsonParseException; +import org.codehaus.jackson.map.JsonMappingException; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +public final class Vertex implements Comparable{ + public final String name; + //tezJsonParser + public final DagJsonParser parser; + // vertex's parent connections. + public final List parentConnections = new ArrayList<>(); + // vertex's children vertex. + public final List children = new ArrayList<>(); + // the jsonObject for this vertex + public final JSONObject vertexObject; + // whether this vertex is dummy (which does not really exists but is created), + // e.g., a dummy vertex for a mergejoin branch + public boolean dummy; + // the rootOps in this vertex + public final List rootOps = new ArrayList<>(); + // we create a dummy vertex for a mergejoin branch for a self join if this + // vertex is a mergejoin + public final List mergeJoinDummyVertexs = new ArrayList<>(); + // whether this vertex has multiple reduce operators + public boolean hasMultiReduceOp = false; + // execution mode + public String executionMode = ""; + // tagToInput for reduce work + public Map tagToInput = new LinkedHashMap<>(); + // tag + public String tag; + + public static enum VertexType { + MAP, REDUCE, UNION, UNKNOWN + }; + public VertexType vertexType; + + public static enum EdgeType { + BROADCAST, SHUFFLE, MULTICAST, PARTITION_ONLY_SHUFFLE, UNKNOWN + }; + public String edgeType; + + public Vertex(String name, JSONObject vertexObject, DagJsonParser dagJsonParser) { + super(); + this.name = name; + if (this.name != null) { + if (this.name.contains("Map")) { + this.vertexType = VertexType.MAP; + } else if (this.name.contains("Reduce")) { + this.vertexType = VertexType.REDUCE; + } else if (this.name.contains("Union")) { + this.vertexType = VertexType.UNION; + } else { + this.vertexType = VertexType.UNKNOWN; + } + } else { + this.vertexType = VertexType.UNKNOWN; + } + this.dummy = false; + this.vertexObject = vertexObject; + parser = dagJsonParser; + } + + public void addDependency(Connection connection) throws JSONException { + this.parentConnections.add(connection); + } + + /** + * @throws JSONException + * @throws JsonParseException + * @throws JsonMappingException + * @throws IOException + * @throws Exception + * We assume that there is a single top-level Map Operator Tree or a + * Reduce Operator Tree in a vertex + */ + public void extractOpTree() throws JSONException, JsonParseException, JsonMappingException, + IOException, Exception { + if (vertexObject.length() != 0) { + for (String key : JSONObject.getNames(vertexObject)) { + if (key.equals("Map Operator Tree:")) { + extractOp(vertexObject.getJSONArray(key).getJSONObject(0)); + } else if (key.equals("Reduce Operator Tree:") || key.equals("Processor Tree:")) { + extractOp(vertexObject.getJSONObject(key)); + } else if (key.equals("Join:")) { + // this is the case when we have a map-side SMB join + // one input of the join is treated as a dummy vertex + JSONArray array = vertexObject.getJSONArray(key); + for (int index = 0; index < array.length(); index++) { + JSONObject mpOpTree = array.getJSONObject(index); + Vertex v = new Vertex(null, mpOpTree, parser); + v.extractOpTree(); + v.dummy = true; + mergeJoinDummyVertexs.add(v); + } + } else if (key.equals("Merge File Operator")) { + JSONObject opTree = vertexObject.getJSONObject(key); + if (opTree.has("Map Operator Tree:")) { + extractOp(opTree.getJSONArray("Map Operator Tree:").getJSONObject(0)); + } else { + throw new Exception("Merge File Operator does not have a Map Operator Tree"); + } + } else if (key.equals("Execution mode:")) { + executionMode = " " + vertexObject.getString(key); + } else if (key.equals("tagToInput:")) { + JSONObject tagToInput = vertexObject.getJSONObject(key); + for (String tag : JSONObject.getNames(tagToInput)) { + this.tagToInput.put(tag, (String) tagToInput.get(tag)); + } + } else if (key.equals("tag:")) { + this.tag = vertexObject.getString(key); + } else if (key.equals("Local Work:")) { + extractOp(vertexObject.getJSONObject(key)); + } else { + throw new Exception("Unsupported operator tree in vertex " + this.name); + } + } + } + } + + /** + * @param operator + * @param parent + * @return + * @throws JSONException + * @throws JsonParseException + * @throws JsonMappingException + * @throws IOException + * @throws Exception + * assumption: each operator only has one parent but may have many + * children + */ + Op extractOp(JSONObject operator) throws JSONException, JsonParseException, JsonMappingException, + IOException, Exception { + String[] names = JSONObject.getNames(operator); + if (names.length != 1) { + throw new Exception("Expect only one operator in " + operator.toString()); + } else { + String opName = names[0]; + JSONObject attrObj = (JSONObject) operator.get(opName); + Map attrs = new TreeMap<>(); + List children = new ArrayList<>(); + String id = null; + String outputVertexName = null; + if (JSONObject.getNames(attrObj) != null) { + for (String attrName : JSONObject.getNames(attrObj)) { + if (attrName.equals("children")) { + Object childrenObj = attrObj.get(attrName); + if (childrenObj instanceof JSONObject) { + if (((JSONObject) childrenObj).length() != 0) { + children.add(extractOp((JSONObject) childrenObj)); + } + } else if (childrenObj instanceof JSONArray) { + if (((JSONArray) childrenObj).length() != 0) { + JSONArray array = ((JSONArray) childrenObj); + for (int index = 0; index < array.length(); index++) { + children.add(extractOp(array.getJSONObject(index))); + } + } + } else { + throw new Exception("Unsupported operator " + this.name + + "'s children operator is neither a jsonobject nor a jsonarray"); + } + } else { + if (attrName.equals("OperatorId:")) { + id = attrObj.get(attrName).toString(); + } else if (attrName.equals("outputname:")) { + outputVertexName = attrObj.get(attrName).toString(); + } else { + if (!attrObj.get(attrName).toString().isEmpty()) { + attrs.put(attrName, attrObj.get(attrName).toString()); + } + } + } + } + } + Op op = new Op(opName, id, outputVertexName, children, attrs, operator, this, parser); + if (!children.isEmpty()) { + for (Op child : children) { + child.parent = op; + } + } else { + this.rootOps.add(op); + } + return op; + } + } + + public void print(Printer printer, int indentFlag, String type, Vertex callingVertex) + throws JSONException, Exception { + // print vertexname + if (parser.printSet.contains(this) && !hasMultiReduceOp) { + if (type != null) { + printer.println(DagJsonParser.prefixString(indentFlag, "<-") + + " Please refer to the previous " + this.name + " [" + type + "]"); + } else { + printer.println(DagJsonParser.prefixString(indentFlag, "<-") + + " Please refer to the previous " + this.name); + } + return; + } + parser.printSet.add(this); + if (type != null) { + printer.println(DagJsonParser.prefixString(indentFlag, "<-") + this.name + " [" + type + "]" + + this.executionMode); + } else if (this.name != null) { + printer.println(DagJsonParser.prefixString(indentFlag) + this.name + this.executionMode); + } + // print operators + if (hasMultiReduceOp && !(callingVertex.vertexType == VertexType.UNION)) { + // find the right op + Op choose = null; + for (Op op : this.rootOps) { + if (op.outputVertexName.equals(callingVertex.name)) { + choose = op; + } + } + if (choose != null) { + choose.print(printer, indentFlag, false); + } else { + throw new Exception("Can not find the right reduce output operator for vertex " + this.name); + } + } else { + for (Op op : this.rootOps) { + // dummy vertex is treated as a branch of a join operator + if (this.dummy) { + op.print(printer, indentFlag, true); + } else { + op.print(printer, indentFlag, false); + } + } + } + if (vertexType == VertexType.UNION) { + // print dependent vertexs + indentFlag++; + for (int index = 0; index < this.parentConnections.size(); index++) { + Connection connection = this.parentConnections.get(index); + connection.from.print(printer, indentFlag, connection.type, this); + } + } + } + + /** + * We check if a vertex has multiple reduce operators. + */ + public void checkMultiReduceOperator() { + // check if it is a reduce vertex and its children is more than 1; + if (!this.name.contains("Reduce") || this.rootOps.size() < 2) { + return; + } + // check if all the child ops are reduce output operators + for (Op op : this.rootOps) { + if (op.type != OpType.RS) { + return; + } + } + this.hasMultiReduceOp = true; + } + + public void setType(String type) { + this.edgeType = this.parser.mapEdgeType(type); + } + + //The following code should be gone after HIVE-11075 using topological order + @Override + public int compareTo(Vertex o) { + return this.name.compareTo(o.name); + } + + public Op getSingleRSOp() { + if (rootOps.size() == 0) { + return null; + } else { + Op ret = null; + for (Op op : rootOps) { + if (op.type == OpType.RS) { + if (ret == null) { + ret = op; + } else { + // find more than one RS Op + return null; + } + } + } + return ret; + } + } +} diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/spark/SparkJsonParser.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/spark/SparkJsonParser.java new file mode 100644 index 0000000..9485aa4 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/spark/SparkJsonParser.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.jsonexplain.spark; + +import org.apache.hadoop.hive.common.jsonexplain.DagJsonParser; + + +public class SparkJsonParser extends DagJsonParser { + + @Override + public String mapEdgeType(String edgeName) { + return edgeName; + } + + @Override + public String getFrameworkName() { + return "Spark"; + } +} \ No newline at end of file diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Connection.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Connection.java deleted file mode 100644 index d341cb1..0000000 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Connection.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.common.jsonexplain.tez; - -public final class Connection { - public final String type; - public final Vertex from; - - public Connection(String type, Vertex from) { - super(); - this.type = type; - this.from = from; - } -} diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Op.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Op.java deleted file mode 100644 index 718791c..0000000 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Op.java +++ /dev/null @@ -1,356 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.common.jsonexplain.tez; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -import org.apache.hadoop.hive.common.jsonexplain.tez.Vertex.VertexType; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -public final class Op { - public final String name; - // tezJsonParser - public final TezJsonParser parser; - public final String operatorId; - public Op parent; - public final List children; - public final Map attrs; - // the jsonObject for this operator - public final JSONObject opObject; - // the vertex that this operator belongs to - public final Vertex vertex; - // the vertex that this operator output to - public final String outputVertexName; - // the Operator type - public final OpType type; - - public enum OpType { - MAPJOIN, MERGEJOIN, RS, OTHERS - }; - - public Op(String name, String id, String outputVertexName, List children, - Map attrs, JSONObject opObject, Vertex vertex, TezJsonParser tezJsonParser) - throws JSONException { - super(); - this.name = name; - this.operatorId = id; - this.type = deriveOpType(operatorId); - this.outputVertexName = outputVertexName; - this.children = children; - this.attrs = attrs; - this.opObject = opObject; - this.vertex = vertex; - this.parser = tezJsonParser; - } - - private OpType deriveOpType(String operatorId) { - if (operatorId != null) { - if (operatorId.startsWith(OpType.MAPJOIN.toString())) { - return OpType.MAPJOIN; - } else if (operatorId.startsWith(OpType.MERGEJOIN.toString())) { - return OpType.MERGEJOIN; - } else if (operatorId.startsWith(OpType.RS.toString())) { - return OpType.RS; - } else { - return OpType.OTHERS; - } - } else { - return OpType.OTHERS; - } - } - - private void inlineJoinOp() throws Exception { - // inline map join operator - if (this.type == OpType.MAPJOIN) { - JSONObject joinObj = opObject.getJSONObject(this.name); - // get the map for posToVertex - JSONObject verticeObj = joinObj.getJSONObject("input vertices:"); - Map posToVertex = new LinkedHashMap<>(); - for (String pos : JSONObject.getNames(verticeObj)) { - String vertexName = verticeObj.getString(pos); - // update the connection - Connection c = null; - for (Connection connection : vertex.parentConnections) { - if (connection.from.name.equals(vertexName)) { - posToVertex.put(pos, connection.from); - c = connection; - break; - } - } - if (c != null) { - parser.addInline(this, c); - } - } - // update the attrs - this.attrs.remove("input vertices:"); - // update the keys to use operator name - JSONObject keys = joinObj.getJSONObject("keys:"); - // find out the vertex for the big table - Set parentVertexes = new HashSet<>(); - for (Connection connection : vertex.parentConnections) { - parentVertexes.add(connection.from); - } - parentVertexes.removeAll(posToVertex.values()); - Map posToOpId = new LinkedHashMap<>(); - if (keys.length() != 0) { - for (String key : JSONObject.getNames(keys)) { - // first search from the posToVertex - if (posToVertex.containsKey(key)) { - Vertex vertex = posToVertex.get(key); - if (vertex.rootOps.size() == 1) { - posToOpId.put(key, vertex.rootOps.get(0).operatorId); - } else if ((vertex.rootOps.size() == 0 && vertex.vertexType == VertexType.UNION)) { - posToOpId.put(key, vertex.name); - } else { - Op singleRSOp = vertex.getSingleRSOp(); - if (singleRSOp != null) { - posToOpId.put(key, singleRSOp.operatorId); - } else { - throw new Exception( - "There are none or more than one root operators in a single vertex " - + vertex.name - + " when hive explain user is trying to identify the operator id."); - } - } - } - // then search from parent - else if (parent != null) { - posToOpId.put(key, parent.operatorId); - } - // then assume it is from its own vertex - else if (parentVertexes.size() == 1) { - Vertex vertex = parentVertexes.iterator().next(); - parentVertexes.clear(); - if (vertex.rootOps.size() == 1) { - posToOpId.put(key, vertex.rootOps.get(0).operatorId); - } else if ((vertex.rootOps.size() == 0 && vertex.vertexType == VertexType.UNION)) { - posToOpId.put(key, vertex.name); - } else { - Op singleRSOp = vertex.getSingleRSOp(); - if (singleRSOp != null) { - posToOpId.put(key, singleRSOp.operatorId); - } else { - throw new Exception( - "There are none or more than one root operators in a single vertex " - + vertex.name - + " when hive explain user is trying to identify the operator id."); - } - } - } - // finally throw an exception - else { - throw new Exception( - "Can not find the source operator on one of the branches of map join."); - } - } - } - this.attrs.remove("keys:"); - StringBuffer sb = new StringBuffer(); - JSONArray conditionMap = joinObj.getJSONArray("condition map:"); - for (int index = 0; index < conditionMap.length(); index++) { - JSONObject cond = conditionMap.getJSONObject(index); - String k = (String) cond.keys().next(); - JSONObject condObject = new JSONObject((String)cond.get(k)); - String type = condObject.getString("type"); - String left = condObject.getString("left"); - String right = condObject.getString("right"); - if (keys.length() != 0) { - sb.append(posToOpId.get(left) + "." + keys.get(left) + "=" + posToOpId.get(right) + "." - + keys.get(right) + "(" + type + "),"); - } else { - // probably a cross product - sb.append("(" + type + "),"); - } - } - this.attrs.remove("condition map:"); - this.attrs.put("Conds:", sb.substring(0, sb.length() - 1)); - } - // should be merge join - else { - Map posToOpId = new LinkedHashMap<>(); - if (vertex.mergeJoinDummyVertexs.size() == 0) { - if (vertex.tagToInput.size() != vertex.parentConnections.size()) { - throw new Exception("tagToInput size " + vertex.tagToInput.size() - + " is different from parentConnections size " + vertex.parentConnections.size()); - } - for (Entry entry : vertex.tagToInput.entrySet()) { - Connection c = null; - for (Connection connection : vertex.parentConnections) { - if (connection.from.name.equals(entry.getValue())) { - Vertex v = connection.from; - if (v.rootOps.size() == 1) { - posToOpId.put(entry.getKey(), v.rootOps.get(0).operatorId); - } else if ((v.rootOps.size() == 0 && v.vertexType == VertexType.UNION)) { - posToOpId.put(entry.getKey(), v.name); - } else { - Op singleRSOp = v.getSingleRSOp(); - if (singleRSOp != null) { - posToOpId.put(entry.getKey(), singleRSOp.operatorId); - } else { - throw new Exception( - "There are none or more than one root operators in a single vertex " + v.name - + " when hive explain user is trying to identify the operator id."); - } - } - c = connection; - break; - } - } - if (c == null) { - throw new Exception("Can not find " + entry.getValue() - + " while parsing keys of merge join operator"); - } - } - } else { - posToOpId.put(vertex.tag, this.parent.operatorId); - for (Vertex v : vertex.mergeJoinDummyVertexs) { - if (v.rootOps.size() != 1) { - throw new Exception("Can not find a single root operators in a single vertex " + v.name - + " when hive explain user is trying to identify the operator id."); - } - posToOpId.put(v.tag, v.rootOps.get(0).operatorId); - } - } - JSONObject joinObj = opObject.getJSONObject(this.name); - // update the keys to use operator name - JSONObject keys = joinObj.getJSONObject("keys:"); - if (keys.length() != 0) { - for (String key : JSONObject.getNames(keys)) { - if (!posToOpId.containsKey(key)) { - throw new Exception( - "Can not find the source operator on one of the branches of merge join."); - } - } - // inline merge join operator in a self-join - if (this.vertex != null) { - for (Vertex v : this.vertex.mergeJoinDummyVertexs) { - parser.addInline(this, new Connection(null, v)); - } - } - } - // update the attrs - this.attrs.remove("keys:"); - StringBuffer sb = new StringBuffer(); - JSONArray conditionMap = joinObj.getJSONArray("condition map:"); - for (int index = 0; index < conditionMap.length(); index++) { - JSONObject cond = conditionMap.getJSONObject(index); - String k = (String) cond.keys().next(); - JSONObject condObject = new JSONObject((String)cond.get(k)); - String type = condObject.getString("type"); - String left = condObject.getString("left"); - String right = condObject.getString("right"); - if (keys.length() != 0) { - sb.append(posToOpId.get(left) + "." + keys.get(left) + "=" + posToOpId.get(right) + "." - + keys.get(right) + "(" + type + "),"); - } else { - // probably a cross product - sb.append("(" + type + "),"); - } - } - this.attrs.remove("condition map:"); - this.attrs.put("Conds:", sb.substring(0, sb.length() - 1)); - } - } - - private String getNameWithOpIdStats() { - StringBuffer sb = new StringBuffer(); - sb.append(TezJsonParserUtils.renameReduceOutputOperator(name, vertex)); - if (operatorId != null) { - sb.append(" [" + operatorId + "]"); - } - if (!TezJsonParserUtils.OperatorNoStats.contains(name) && attrs.containsKey("Statistics:")) { - sb.append(" (" + attrs.get("Statistics:") + ")"); - } - attrs.remove("Statistics:"); - return sb.toString(); - } - - /** - * @param printer - * @param indentFlag - * @param branchOfJoinOp - * This parameter is used to show if it is a branch of a Join - * operator so that we can decide the corresponding indent. - * @throws Exception - */ - public void print(Printer printer, int indentFlag, boolean branchOfJoinOp) throws Exception { - // print name - if (parser.printSet.contains(this)) { - printer.println(TezJsonParser.prefixString(indentFlag) + " Please refer to the previous " - + this.getNameWithOpIdStats()); - return; - } - parser.printSet.add(this); - if (!branchOfJoinOp) { - printer.println(TezJsonParser.prefixString(indentFlag) + this.getNameWithOpIdStats()); - } else { - printer.println(TezJsonParser.prefixString(indentFlag, "<-") + this.getNameWithOpIdStats()); - } - branchOfJoinOp = false; - // if this operator is a Map Join Operator or a Merge Join Operator - if (this.type == OpType.MAPJOIN || this.type == OpType.MERGEJOIN) { - inlineJoinOp(); - branchOfJoinOp = true; - } - // if this operator is the last operator, we summarize the non-inlined - // vertex - List noninlined = new ArrayList<>(); - if (this.parent == null) { - if (this.vertex != null) { - for (Connection connection : this.vertex.parentConnections) { - if (!parser.isInline(connection.from)) { - noninlined.add(connection); - } - } - } - } - // print attr - indentFlag++; - if (!attrs.isEmpty()) { - printer.println(TezJsonParser.prefixString(indentFlag) - + TezJsonParserUtils.attrsToString(attrs)); - } - // print inline vertex - if (parser.inlineMap.containsKey(this)) { - for (int index = 0; index < parser.inlineMap.get(this).size(); index++) { - Connection connection = parser.inlineMap.get(this).get(index); - connection.from.print(printer, indentFlag, connection.type, this.vertex); - } - } - // print parent op, i.e., where data comes from - if (this.parent != null) { - this.parent.print(printer, indentFlag, branchOfJoinOp); - } - // print next vertex - else { - for (int index = 0; index < noninlined.size(); index++) { - Vertex v = noninlined.get(index).from; - v.print(printer, indentFlag, noninlined.get(index).type, this.vertex); - } - } - } -} diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Printer.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Printer.java deleted file mode 100644 index d3c91d6..0000000 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Printer.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.common.jsonexplain.tez; - -public final class Printer { - public static final String lineSeparator = System.getProperty("line.separator");; - private final StringBuilder builder = new StringBuilder(); - - public void print(String string) { - builder.append(string); - } - - public void println(String string) { - builder.append(string); - builder.append(lineSeparator); - } - - public void println() { - builder.append(lineSeparator); - } - - public String toString() { - return builder.toString(); - } -} diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Stage.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Stage.java deleted file mode 100644 index 63937f8..0000000 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Stage.java +++ /dev/null @@ -1,262 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.common.jsonexplain.tez; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.jsonexplain.tez.Vertex.VertexType; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -public final class Stage { - //external name is used to show at the console - String externalName; - //internal name is used to track the stages - public final String internalName; - //tezJsonParser - public final TezJsonParser parser; - // upstream stages, e.g., root stage - public final List parentStages = new ArrayList<>(); - // downstream stages. - public final List childStages = new ArrayList<>(); - public final Map vertexs =new LinkedHashMap<>(); - public final Map attrs = new TreeMap<>(); - Map> tezStageDependency; - // some stage may contain only a single operator, e.g., create table operator, - // fetch operator. - Op op; - - public Stage(String name, TezJsonParser tezJsonParser) { - super(); - internalName = name; - externalName = name; - parser = tezJsonParser; - } - - public void addDependency(JSONObject object, Map stages) throws JSONException { - if (object.has("DEPENDENT STAGES")) { - String names = object.getString("DEPENDENT STAGES"); - for (String name : names.split(",")) { - Stage parent = stages.get(name.trim()); - this.parentStages.add(parent); - parent.childStages.add(this); - } - } - if (object.has("CONDITIONAL CHILD TASKS")) { - String names = object.getString("CONDITIONAL CHILD TASKS"); - this.externalName = this.internalName + "(CONDITIONAL CHILD TASKS: " + names + ")"; - for (String name : names.split(",")) { - Stage child = stages.get(name.trim()); - child.externalName = child.internalName + "(CONDITIONAL)"; - child.parentStages.add(this); - this.childStages.add(child); - } - } - } - - /** - * @param object - * @throws Exception - * If the object of stage contains "Tez", we need to extract the - * vertices and edges Else we need to directly extract operators - * and/or attributes. - */ - public void extractVertex(JSONObject object) throws Exception { - if (object.has("Tez")) { - this.tezStageDependency = new TreeMap<>(); - JSONObject tez = (JSONObject) object.get("Tez"); - JSONObject vertices = tez.getJSONObject("Vertices:"); - if (tez.has("Edges:")) { - JSONObject edges = tez.getJSONObject("Edges:"); - // iterate for the first time to get all the vertices - for (String to : JSONObject.getNames(edges)) { - vertexs.put(to, new Vertex(to, vertices.getJSONObject(to), parser)); - } - // iterate for the second time to get all the vertex dependency - for (String to : JSONObject.getNames(edges)) { - Object o = edges.get(to); - Vertex v = vertexs.get(to); - // 1 to 1 mapping - if (o instanceof JSONObject) { - JSONObject obj = (JSONObject) o; - String parent = obj.getString("parent"); - Vertex parentVertex = vertexs.get(parent); - if (parentVertex == null) { - parentVertex = new Vertex(parent, vertices.getJSONObject(parent), parser); - vertexs.put(parent, parentVertex); - } - String type = obj.getString("type"); - // for union vertex, we reverse the dependency relationship - if (!"CONTAINS".equals(type)) { - v.addDependency(new Connection(type, parentVertex)); - parentVertex.setType(type); - parentVertex.children.add(v); - } else { - parentVertex.addDependency(new Connection(type, v)); - v.children.add(parentVertex); - } - this.tezStageDependency.put(v, Arrays.asList(new Connection(type, parentVertex))); - } else { - // 1 to many mapping - JSONArray from = (JSONArray) o; - List list = new ArrayList<>(); - for (int index = 0; index < from.length(); index++) { - JSONObject obj = from.getJSONObject(index); - String parent = obj.getString("parent"); - Vertex parentVertex = vertexs.get(parent); - if (parentVertex == null) { - parentVertex = new Vertex(parent, vertices.getJSONObject(parent), parser); - vertexs.put(parent, parentVertex); - } - String type = obj.getString("type"); - if (!"CONTAINS".equals(type)) { - v.addDependency(new Connection(type, parentVertex)); - parentVertex.setType(type); - parentVertex.children.add(v); - } else { - parentVertex.addDependency(new Connection(type, v)); - v.children.add(parentVertex); - } - list.add(new Connection(type, parentVertex)); - } - this.tezStageDependency.put(v, list); - } - } - } else { - for (String vertexName : JSONObject.getNames(vertices)) { - vertexs.put(vertexName, new Vertex(vertexName, vertices.getJSONObject(vertexName), parser)); - } - } - // The opTree in vertex is extracted - for (Vertex v : vertexs.values()) { - if (v.vertexType == VertexType.MAP || v.vertexType == VertexType.REDUCE) { - v.extractOpTree(); - v.checkMultiReduceOperator(); - } - } - } else { - String[] names = JSONObject.getNames(object); - if (names != null) { - for (String name : names) { - if (name.contains("Operator")) { - this.op = extractOp(name, object.getJSONObject(name)); - } else { - if (!object.get(name).toString().isEmpty()) { - attrs.put(name, object.get(name).toString()); - } - } - } - } - } - } - - /** - * @param opName - * @param opObj - * @return - * @throws Exception - * This method address the create table operator, fetch operator, - * etc - */ - Op extractOp(String opName, JSONObject opObj) throws Exception { - Map attrs = new TreeMap<>(); - Vertex v = null; - if (opObj.length() > 0) { - String[] names = JSONObject.getNames(opObj); - for (String name : names) { - Object o = opObj.get(name); - if (isPrintable(o) && !o.toString().isEmpty()) { - attrs.put(name, o.toString()); - } else if (o instanceof JSONObject) { - JSONObject attrObj = (JSONObject) o; - if (attrObj.length() > 0) { - if (name.equals("Processor Tree:")) { - JSONObject object = new JSONObject(new LinkedHashMap<>()); - object.put(name, attrObj); - v = new Vertex(null, object, parser); - v.extractOpTree(); - } else { - for (String attrName : JSONObject.getNames(attrObj)) { - if (!attrObj.get(attrName).toString().isEmpty()) { - attrs.put(attrName, attrObj.get(attrName).toString()); - } - } - } - } - } else { - throw new Exception("Unsupported object in " + this.internalName); - } - } - } - Op op = new Op(opName, null, null, null, attrs, null, v, parser); - if (v != null) { - parser.addInline(op, new Connection(null, v)); - } - return op; - } - - private boolean isPrintable(Object val) { - if (val instanceof Boolean || val instanceof String || val instanceof Integer - || val instanceof Long || val instanceof Byte || val instanceof Float - || val instanceof Double || val instanceof Path) { - return true; - } - if (val != null && val.getClass().isPrimitive()) { - return true; - } - return false; - } - - public void print(Printer printer, int indentFlag) throws Exception { - // print stagename - if (parser.printSet.contains(this)) { - printer.println(TezJsonParser.prefixString(indentFlag) + " Please refer to the previous " - + externalName); - return; - } - parser.printSet.add(this); - printer.println(TezJsonParser.prefixString(indentFlag) + externalName); - // print vertexes - indentFlag++; - for (Vertex candidate : this.vertexs.values()) { - if (!parser.isInline(candidate) && candidate.children.isEmpty()) { - candidate.print(printer, indentFlag, null, null); - } - } - if (!attrs.isEmpty()) { - printer.println(TezJsonParser.prefixString(indentFlag) - + TezJsonParserUtils.attrsToString(attrs)); - } - if (op != null) { - op.print(printer, indentFlag, false); - } - indentFlag++; - // print dependent stages - for (Stage stage : this.parentStages) { - stage.print(printer, indentFlag); - } - } -} diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java index ea86048..294dc6b 100644 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java @@ -18,146 +18,29 @@ package org.apache.hadoop.hive.common.jsonexplain.tez; -import java.io.PrintStream; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; +import org.apache.hadoop.hive.common.jsonexplain.DagJsonParser; -import org.apache.hadoop.hive.common.jsonexplain.JsonParser; -import org.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public final class TezJsonParser implements JsonParser { - public final Map stages = new LinkedHashMap<>(); - protected final Logger LOG; - // the objects that have been printed. - public final Set printSet = new LinkedHashSet<>(); - // the vertex that should be inlined. - public final Map> inlineMap = new LinkedHashMap<>(); - - public TezJsonParser() { - super(); - LOG = LoggerFactory.getLogger(this.getClass().getName()); - } - - public void extractStagesAndPlans(JSONObject inputObject) throws Exception { - // extract stages - JSONObject dependency = inputObject.getJSONObject("STAGE DEPENDENCIES"); - if (dependency != null && dependency.length() > 0) { - // iterate for the first time to get all the names of stages. - for (String stageName : JSONObject.getNames(dependency)) { - this.stages.put(stageName, new Stage(stageName, this)); - } - // iterate for the second time to get all the dependency. - for (String stageName : JSONObject.getNames(dependency)) { - JSONObject dependentStageNames = dependency.getJSONObject(stageName); - this.stages.get(stageName).addDependency(dependentStageNames, this.stages); - } - } - // extract stage plans - JSONObject stagePlans = inputObject.getJSONObject("STAGE PLANS"); - if (stagePlans != null && stagePlans.length() > 0) { - for (String stageName : JSONObject.getNames(stagePlans)) { - JSONObject stagePlan = stagePlans.getJSONObject(stageName); - this.stages.get(stageName).extractVertex(stagePlan); - } - } - } - - /** - * @param indentFlag - * help to generate correct indent - * @return - */ - public static String prefixString(int indentFlag) { - StringBuilder sb = new StringBuilder(); - for (int index = 0; index < indentFlag; index++) { - sb.append(" "); - } - return sb.toString(); - } - - /** - * @param indentFlag - * @param tail - * help to generate correct indent with a specific tail - * @return - */ - public static String prefixString(int indentFlag, String tail) { - StringBuilder sb = new StringBuilder(); - for (int index = 0; index < indentFlag; index++) { - sb.append(" "); - } - int len = sb.length(); - return sb.replace(len - tail.length(), len, tail).toString(); - } +public class TezJsonParser extends DagJsonParser { @Override - public void print(JSONObject inputObject, PrintStream outputStream) throws Exception { - LOG.info("JsonParser is parsing:" + inputObject.toString()); - this.extractStagesAndPlans(inputObject); - Printer printer = new Printer(); - // print out the cbo info - if (inputObject.has("cboInfo")) { - printer.println(inputObject.getString("cboInfo")); - printer.println(); - } - // print out the vertex dependency in root stage - for (Stage candidate : this.stages.values()) { - if (candidate.tezStageDependency != null && candidate.tezStageDependency.size() > 0) { - printer.println("Vertex dependency in root stage"); - for (Entry> entry : candidate.tezStageDependency.entrySet()) { - StringBuilder sb = new StringBuilder(); - sb.append(entry.getKey().name); - sb.append(" <- "); - boolean printcomma = false; - for (Connection connection : entry.getValue()) { - if (printcomma) { - sb.append(", "); - } else { - printcomma = true; - } - sb.append(connection.from.name + " (" + connection.type + ")"); - } - printer.println(sb.toString()); - } - printer.println(); - } + public String mapEdgeType(String edgeName) { + switch (edgeName) { + case "BROADCAST_EDGE": + return "BROADCAST"; + case "SIMPLE_EDGE": + return "SHUFFLE"; + case "CUSTOM_SIMPLE_EDGE": + return "PARTITION_ONLY_SHUFFLE"; + case "CUSTOM_EDGE": + return "MULTICAST"; + default: + return "UNKNOWN"; } - // print out all the stages that have no childStages. - for (Stage candidate : this.stages.values()) { - if (candidate.childStages.isEmpty()) { - candidate.print(printer, 0); - } - } - outputStream.println(printer.toString()); } - public void addInline(Op op, Connection connection) { - List list = inlineMap.get(op); - if (list == null) { - list = new ArrayList<>(); - list.add(connection); - inlineMap.put(op, list); - } else { - list.add(connection); - } - } - - public boolean isInline(Vertex v) { - for (List list : inlineMap.values()) { - for (Connection connection : list) { - if (connection.from.equals(v)) { - return true; - } - } - } - return false; + @Override + public String getFrameworkName() { + return "Tez"; } -} +} \ No newline at end of file diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParserUtils.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParserUtils.java deleted file mode 100644 index 363a422..0000000 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParserUtils.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.common.jsonexplain.tez; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - - -public class TezJsonParserUtils { - - public static List OperatorNoStats = Arrays.asList(new String[] { "File Output Operator", - "Reduce Output Operator" }); - - public static String renameReduceOutputOperator(String operatorName, Vertex vertex) { - if (operatorName.equals("Reduce Output Operator") && vertex.edgeType != null) { - return vertex.edgeType.name(); - } else { - return operatorName; - } - } - - public static String attrsToString(Map attrs) { - StringBuffer sb = new StringBuffer(); - boolean first = true; - for (Entry entry : attrs.entrySet()) { - if (first) { - first = false; - } else { - sb.append(","); - } - sb.append(entry.getKey() + entry.getValue()); - } - return sb.toString(); - } -} diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Vertex.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Vertex.java deleted file mode 100644 index 3d559bd..0000000 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Vertex.java +++ /dev/null @@ -1,331 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.common.jsonexplain.tez; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; - -import org.apache.hadoop.hive.common.jsonexplain.tez.Op.OpType; -import org.apache.hadoop.util.hash.Hash; -import org.codehaus.jackson.JsonParseException; -import org.codehaus.jackson.map.JsonMappingException; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -public final class Vertex implements Comparable{ - public final String name; - //tezJsonParser - public final TezJsonParser parser; - // vertex's parent connections. - public final List parentConnections = new ArrayList<>(); - // vertex's children vertex. - public final List children = new ArrayList<>(); - // the jsonObject for this vertex - public final JSONObject vertexObject; - // whether this vertex is dummy (which does not really exists but is created), - // e.g., a dummy vertex for a mergejoin branch - public boolean dummy; - // the rootOps in this vertex - public final List rootOps = new ArrayList<>(); - // we create a dummy vertex for a mergejoin branch for a self join if this - // vertex is a mergejoin - public final List mergeJoinDummyVertexs = new ArrayList<>(); - // whether this vertex has multiple reduce operators - public boolean hasMultiReduceOp = false; - // execution mode - public String executionMode = ""; - // tagToInput for reduce work - public Map tagToInput = new LinkedHashMap<>(); - // tag - public String tag; - - public static enum VertexType { - MAP, REDUCE, UNION, UNKNOWN - }; - public VertexType vertexType; - - public static enum EdgeType { - BROADCAST, SHUFFLE, MULTICAST, PARTITION_ONLY_SHUFFLE, UNKNOWN - }; - public EdgeType edgeType; - - public Vertex(String name, JSONObject vertexObject, TezJsonParser tezJsonParser) { - super(); - this.name = name; - if (this.name != null) { - if (this.name.contains("Map")) { - this.vertexType = VertexType.MAP; - } else if (this.name.contains("Reduce")) { - this.vertexType = VertexType.REDUCE; - } else if (this.name.contains("Union")) { - this.vertexType = VertexType.UNION; - } else { - this.vertexType = VertexType.UNKNOWN; - } - } else { - this.vertexType = VertexType.UNKNOWN; - } - this.dummy = false; - this.vertexObject = vertexObject; - parser = tezJsonParser; - } - - public void addDependency(Connection connection) throws JSONException { - this.parentConnections.add(connection); - } - - /** - * @throws JSONException - * @throws JsonParseException - * @throws JsonMappingException - * @throws IOException - * @throws Exception - * We assume that there is a single top-level Map Operator Tree or a - * Reduce Operator Tree in a vertex - */ - public void extractOpTree() throws JSONException, JsonParseException, JsonMappingException, - IOException, Exception { - if (vertexObject.length() != 0) { - for (String key : JSONObject.getNames(vertexObject)) { - if (key.equals("Map Operator Tree:")) { - extractOp(vertexObject.getJSONArray(key).getJSONObject(0)); - } else if (key.equals("Reduce Operator Tree:") || key.equals("Processor Tree:")) { - extractOp(vertexObject.getJSONObject(key)); - } else if (key.equals("Join:")) { - // this is the case when we have a map-side SMB join - // one input of the join is treated as a dummy vertex - JSONArray array = vertexObject.getJSONArray(key); - for (int index = 0; index < array.length(); index++) { - JSONObject mpOpTree = array.getJSONObject(index); - Vertex v = new Vertex(null, mpOpTree, parser); - v.extractOpTree(); - v.dummy = true; - mergeJoinDummyVertexs.add(v); - } - } else if (key.equals("Merge File Operator")) { - JSONObject opTree = vertexObject.getJSONObject(key); - if (opTree.has("Map Operator Tree:")) { - extractOp(opTree.getJSONArray("Map Operator Tree:").getJSONObject(0)); - } else { - throw new Exception("Merge File Operator does not have a Map Operator Tree"); - } - } else if (key.equals("Execution mode:")) { - executionMode = " " + vertexObject.getString(key); - } else if (key.equals("tagToInput:")) { - JSONObject tagToInput = vertexObject.getJSONObject(key); - for (String tag : JSONObject.getNames(tagToInput)) { - this.tagToInput.put(tag, (String) tagToInput.get(tag)); - } - } else if (key.equals("tag:")) { - this.tag = vertexObject.getString(key); - } else { - throw new Exception("Unsupported operator tree in vertex " + this.name); - } - } - } - } - - /** - * @param operator - * @param parent - * @return - * @throws JSONException - * @throws JsonParseException - * @throws JsonMappingException - * @throws IOException - * @throws Exception - * assumption: each operator only has one parent but may have many - * children - */ - Op extractOp(JSONObject operator) throws JSONException, JsonParseException, JsonMappingException, - IOException, Exception { - String[] names = JSONObject.getNames(operator); - if (names.length != 1) { - throw new Exception("Expect only one operator in " + operator.toString()); - } else { - String opName = names[0]; - JSONObject attrObj = (JSONObject) operator.get(opName); - Map attrs = new TreeMap<>(); - List children = new ArrayList<>(); - String id = null; - String outputVertexName = null; - for (String attrName : JSONObject.getNames(attrObj)) { - if (attrName.equals("children")) { - Object childrenObj = attrObj.get(attrName); - if (childrenObj instanceof JSONObject) { - if (((JSONObject) childrenObj).length() != 0) { - children.add(extractOp((JSONObject) childrenObj)); - } - } else if (childrenObj instanceof JSONArray) { - if (((JSONArray) childrenObj).length() != 0) { - JSONArray array = ((JSONArray) childrenObj); - for (int index = 0; index < array.length(); index++) { - children.add(extractOp(array.getJSONObject(index))); - } - } - } else { - throw new Exception("Unsupported operator " + this.name - + "'s children operator is neither a jsonobject nor a jsonarray"); - } - } else { - if (attrName.equals("OperatorId:")) { - id = attrObj.get(attrName).toString(); - } else if (attrName.equals("outputname:")) { - outputVertexName = attrObj.get(attrName).toString(); - } else { - if (!attrObj.get(attrName).toString().isEmpty()) { - attrs.put(attrName, attrObj.get(attrName).toString()); - } - } - } - } - Op op = new Op(opName, id, outputVertexName, children, attrs, operator, this, parser); - if (!children.isEmpty()) { - for (Op child : children) { - child.parent = op; - } - } else { - this.rootOps.add(op); - } - return op; - } - } - - public void print(Printer printer, int indentFlag, String type, Vertex callingVertex) - throws JSONException, Exception { - // print vertexname - if (parser.printSet.contains(this) && !hasMultiReduceOp) { - if (type != null) { - printer.println(TezJsonParser.prefixString(indentFlag, "<-") - + " Please refer to the previous " + this.name + " [" + type + "]"); - } else { - printer.println(TezJsonParser.prefixString(indentFlag, "<-") - + " Please refer to the previous " + this.name); - } - return; - } - parser.printSet.add(this); - if (type != null) { - printer.println(TezJsonParser.prefixString(indentFlag, "<-") + this.name + " [" + type + "]" - + this.executionMode); - } else if (this.name != null) { - printer.println(TezJsonParser.prefixString(indentFlag) + this.name + this.executionMode); - } - // print operators - if (hasMultiReduceOp && !(callingVertex.vertexType == VertexType.UNION)) { - // find the right op - Op choose = null; - for (Op op : this.rootOps) { - if (op.outputVertexName.equals(callingVertex.name)) { - choose = op; - } - } - if (choose != null) { - choose.print(printer, indentFlag, false); - } else { - throw new Exception("Can not find the right reduce output operator for vertex " + this.name); - } - } else { - for (Op op : this.rootOps) { - // dummy vertex is treated as a branch of a join operator - if (this.dummy) { - op.print(printer, indentFlag, true); - } else { - op.print(printer, indentFlag, false); - } - } - } - if (vertexType == VertexType.UNION) { - // print dependent vertexs - indentFlag++; - for (int index = 0; index < this.parentConnections.size(); index++) { - Connection connection = this.parentConnections.get(index); - connection.from.print(printer, indentFlag, connection.type, this); - } - } - } - - /** - * We check if a vertex has multiple reduce operators. - */ - public void checkMultiReduceOperator() { - // check if it is a reduce vertex and its children is more than 1; - if (!this.name.contains("Reduce") || this.rootOps.size() < 2) { - return; - } - // check if all the child ops are reduce output operators - for (Op op : this.rootOps) { - if (op.type != OpType.RS) { - return; - } - } - this.hasMultiReduceOp = true; - } - - public void setType(String type) { - switch (type) { - case "BROADCAST_EDGE": - this.edgeType = EdgeType.BROADCAST; - break; - case "SIMPLE_EDGE": - this.edgeType = EdgeType.SHUFFLE; - break; - case "CUSTOM_SIMPLE_EDGE": - this.edgeType = EdgeType.PARTITION_ONLY_SHUFFLE; - break; - case "CUSTOM_EDGE": - this.edgeType = EdgeType.MULTICAST; - break; - default: - this.edgeType = EdgeType.UNKNOWN; - } - } - - //The following code should be gone after HIVE-11075 using topological order - @Override - public int compareTo(Vertex o) { - return this.name.compareTo(o.name); - } - - public Op getSingleRSOp() { - if (rootOps.size() == 0) { - return null; - } else { - Op ret = null; - for (Op op : rootOps) { - if (op.type == OpType.RS) { - if (ret == null) { - ret = op; - } else { - // find more than one RS Op - return null; - } - } - } - return ret; - } - } -} diff --git a/data/conf/spark/standalone/hive-site.xml b/data/conf/spark/standalone/hive-site.xml index 989e65e..dea5ac3 100644 --- a/data/conf/spark/standalone/hive-site.xml +++ b/data/conf/spark/standalone/hive-site.xml @@ -240,4 +240,9 @@ Internal marker for test. Used for masking env-dependent values + + hive.explain.user + false + + diff --git a/data/conf/spark/yarn-client/hive-site.xml b/data/conf/spark/yarn-client/hive-site.xml index 9cda40d..f5112f6 100644 --- a/data/conf/spark/yarn-client/hive-site.xml +++ b/data/conf/spark/yarn-client/hive-site.xml @@ -285,4 +285,9 @@ 30000ms + + hive.explain.user + false + + diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index fb85b9e..35cc08d 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -890,6 +890,7 @@ spark.query.files=add_part_multiple.q, \ escape_distributeby1.q, \ escape_orderby1.q, \ escape_sortby1.q, \ + explainuser_1.q, \ filter_join_breaktask.q, \ filter_join_breaktask2.q, \ groupby1.q, \ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java index a3a19f4..92225ac 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java @@ -210,7 +210,7 @@ public void initialize(HiveConf hiveConf) { if(HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTCORRELATION) && !HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEGROUPBYSKEW) && !HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_OPTIMIZE_SKEWJOIN_COMPILETIME) && - !isTezExecEngine) { + !isTezExecEngine && !isSparkExecEngine) { transformations.add(new CorrelationOptimizer()); } if (HiveConf.getFloatVar(hiveConf, HiveConf.ConfVars.HIVELIMITPUSHDOWNMEMORYUSAGE) > 0) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java index f62cf9a..d5fe668 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java @@ -187,8 +187,9 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { && !config.isDependency() && !config.isLogical() && !config.isAuthorize() - && (HiveConf.getBoolVar(ctx.getConf(), HiveConf.ConfVars.HIVE_EXPLAIN_USER) && HiveConf - .getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez"))); + && (HiveConf.getBoolVar(ctx.getConf(), HiveConf.ConfVars.HIVE_EXPLAIN_USER) && (HiveConf + .getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez") || HiveConf + .getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")))); ExplainWork work = new ExplainWork(ctx.getResFile(), pCtx, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/SparkWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/SparkWork.java index 066e32d..9d46cac 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/SparkWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/SparkWork.java @@ -34,6 +34,7 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.ql.plan.Explain.Vectorization; +import org.apache.hadoop.hive.ql.plan.Explain.Level; import com.google.common.base.Preconditions; @@ -43,7 +44,8 @@ * roots and and ReduceWork at all other nodes. */ @SuppressWarnings("serial") -@Explain(displayName = "Spark", vectorization = Vectorization.SUMMARY_PATH) +@Explain(displayName = "Spark", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }, + vectorization = Vectorization.SUMMARY_PATH) public class SparkWork extends AbstractOperatorDesc { private static int counter; private final String name; @@ -76,7 +78,8 @@ public String getName() { /** * @return a map of "vertex name" to BaseWork */ - @Explain(displayName = "Vertices", vectorization = Vectorization.SUMMARY_PATH) + @Explain(displayName = "Vertices", explainLevels = { Explain.Level.USER, Explain.Level.DEFAULT, Explain.Level.EXTENDED }, + vectorization = Vectorization.SUMMARY_PATH) public Map getWorkMap() { Map result = new LinkedHashMap(); for (BaseWork w: getAllWork()) { @@ -378,7 +381,8 @@ public String toString() { } } - @Explain(displayName = "Edges") + @Explain(displayName = "Edges", explainLevels = { Explain.Level.USER, Explain.Level.DEFAULT, Explain.Level.EXTENDED }, + vectorization = Vectorization.SUMMARY_PATH) public Map> getDependencyMap() { Map allDependencies = new HashMap(); Map> result = diff --git a/ql/src/test/results/clientpositive/spark/explainuser_1.q.out b/ql/src/test/results/clientpositive/spark/explainuser_1.q.out new file mode 100644 index 0000000..5f760bd --- /dev/null +++ b/ql/src/test/results/clientpositive/spark/explainuser_1.q.out @@ -0,0 +1,5985 @@ +PREHOOK: query: explain create table src_orc_merge_test_part(key int, value string) partitioned by (ds string, ts string) stored as orc +PREHOOK: type: CREATETABLE +POSTHOOK: query: explain create table src_orc_merge_test_part(key int, value string) partitioned by (ds string, ts string) stored as orc +POSTHOOK: type: CREATETABLE +Stage-0 + Create Table Operator: + name:default.src_orc_merge_test_part + +PREHOOK: query: create table src_orc_merge_test_part(key int, value string) partitioned by (ds string, ts string) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_orc_merge_test_part +POSTHOOK: query: create table src_orc_merge_test_part(key int, value string) partitioned by (ds string, ts string) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_orc_merge_test_part +PREHOOK: query: alter table src_orc_merge_test_part add partition (ds='2012-01-03', ts='2012-01-03+14:46:31') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@src_orc_merge_test_part +POSTHOOK: query: alter table src_orc_merge_test_part add partition (ds='2012-01-03', ts='2012-01-03+14:46:31') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@src_orc_merge_test_part +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +PREHOOK: query: desc extended src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: query: desc extended src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_orc_merge_test_part +key int +value string +ds string +ts string + +# Partition Information +# col_name data_type comment + +ds string +ts string + +#### A masked pattern was here #### +PREHOOK: query: explain insert overwrite table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src +PREHOOK: type: QUERY +POSTHOOK: query: explain insert overwrite table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Stage-2 + Stats-Aggr Operator + Stage-0 + Move Operator + table:{"name:":"default.src_orc_merge_test_part"} + Stage-1 + Map 1 + File Output Operator [FS_3] + table:{"name:":"default.src_orc_merge_test_part"} + Select Operator [SEL_1] (rows=500 width=10) + Output:["_col0","_col1"] + TableScan [TS_0] (rows=500 width=10) + default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + +PREHOOK: query: insert overwrite table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: query: insert overwrite table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2012-01-03,ts=2012-01-03+14:46:31).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2012-01-03,ts=2012-01-03+14:46:31).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain insert into table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src limit 100 +PREHOOK: type: QUERY +POSTHOOK: query: explain insert into table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src limit 100 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) + +Stage-2 + Stats-Aggr Operator + Stage-0 + Move Operator + table:{"name:":"default.src_orc_merge_test_part"} + Stage-1 + Reducer 2 + File Output Operator [FS_7] + table:{"name:":"default.src_orc_merge_test_part"} + Select Operator [SEL_6] (rows=100 width=10) + Output:["_col0","_col1"] + Limit [LIM_5] (rows=100 width=10) + Number of rows:100 + Select Operator [SEL_4] (rows=100 width=10) + Output:["_col0","_col1"] + <-Map 1 [GROUP] + GROUP [RS_3] + Limit [LIM_2] (rows=100 width=10) + Number of rows:100 + Select Operator [SEL_1] (rows=500 width=10) + Output:["_col0","_col1"] + TableScan [TS_0] (rows=500 width=10) + default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + +PREHOOK: query: explain select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Stage-0 + Fetch Operator + limit:1 + +PREHOOK: query: explain select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +PREHOOK: type: QUERY +POSTHOOK: query: explain select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_8] + Group By Operator [GBY_6] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] + <-Map 1 [GROUP] + GROUP [RS_5] + Group By Operator [GBY_4] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] + Select Operator [SEL_2] (rows=500 width=94) + Output:["_col0","_col1"] + TableScan [TS_0] (rows=500 width=94) + default@src_orc_merge_test_part,src_orc_merge_test_part,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + +PREHOOK: query: alter table src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31') concatenate +PREHOOK: type: ALTER_PARTITION_MERGE +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: query: alter table src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31') concatenate +POSTHOOK: type: ALTER_PARTITION_MERGE +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +PREHOOK: query: explain select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_8] + Group By Operator [GBY_6] (rows=1 width=8) + Output:["_col0"],aggregations:["count(VALUE._col0)"] + <-Map 1 [GROUP] + GROUP [RS_5] + Group By Operator [GBY_4] (rows=1 width=8) + Output:["_col0"],aggregations:["count(1)"] + Select Operator [SEL_2] (rows=500 width=94) + TableScan [TS_0] (rows=500 width=94) + default@src_orc_merge_test_part,src_orc_merge_test_part,Tbl:COMPLETE,Col:NONE + +PREHOOK: query: explain select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +PREHOOK: type: QUERY +POSTHOOK: query: explain select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_8] + Group By Operator [GBY_6] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] + <-Map 1 [GROUP] + GROUP [RS_5] + Group By Operator [GBY_4] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] + Select Operator [SEL_2] (rows=500 width=94) + Output:["_col0","_col1"] + TableScan [TS_0] (rows=500 width=94) + default@src_orc_merge_test_part,src_orc_merge_test_part,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + +PREHOOK: query: drop table src_orc_merge_test_part +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Output: default@src_orc_merge_test_part +POSTHOOK: query: drop table src_orc_merge_test_part +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Output: default@src_orc_merge_test_part +Warning: Map Join MAPJOIN[20][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +PREHOOK: query: explain select sum(hash(a.k1,a.v1,a.k2, a.v2)) +from ( +select src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (select * FROM src WHERE src.key < 10) src1 + JOIN + (select * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2 +) a +PREHOOK: type: QUERY +POSTHOOK: query: explain select sum(hash(a.k1,a.v1,a.k2, a.v2)) +from ( +select src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (select * FROM src WHERE src.key < 10) src1 + JOIN + (select * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2 +) a +POSTHOOK: type: QUERY +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_17] + Group By Operator [GBY_15] (rows=1 width=8) + Output:["_col0"],aggregations:["sum(VALUE._col0)"] + <-Reducer 2 [GROUP] + GROUP [RS_14] + Group By Operator [GBY_13] (rows=1 width=8) + Output:["_col0"],aggregations:["sum(hash(_col0,_col1,_col2,_col3))"] + Select Operator [SEL_11] (rows=27556 width=22) + Output:["_col0","_col1","_col2","_col3"] + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_10] + Map Join Operator [MAPJOIN_20] (rows=27556 width=22) + Conds:(Inner),Output:["_col0","_col1","_col2","_col3"] + <-Select Operator [SEL_2] (rows=166 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_18] (rows=166 width=10) + predicate:(key < 10) + TableScan [TS_0] (rows=500 width=10) + default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Map Reduce Local Work + Stage-2 + Map 4 + keys: [HASHTABLESINK_22] + Select Operator [SEL_5] (rows=166 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_19] (rows=166 width=10) + predicate:(key < 10) + TableScan [TS_3] (rows=500 width=10) + default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Map Reduce Local Work + +PREHOOK: query: explain select key, (c_int+1)+2 as x, sum(c_int) from cbo_t1 group by c_float, cbo_t1.c_int, key +PREHOOK: type: QUERY +POSTHOOK: query: explain select key, (c_int+1)+2 as x, sum(c_int) from cbo_t1 group by c_float, cbo_t1.c_int, key +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_7] + Select Operator [SEL_5] (rows=10 width=97) + Output:["_col0","_col1","_col2"] + Group By Operator [GBY_4] (rows=10 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Map 1 [GROUP] + GROUP [RS_3] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_2] (rows=10 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float + Select Operator [SEL_1] (rows=20 width=88) + Output:["key","c_int","c_float"] + TableScan [TS_0] (rows=20 width=88) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + +PREHOOK: query: explain select x, y, count(*) from (select key, (c_int+c_float+1+2) as x, sum(c_int) as y from cbo_t1 group by c_float, cbo_t1.c_int, key) R group by y, x +PREHOOK: type: QUERY +POSTHOOK: query: explain select x, y, count(*) from (select key, (c_int+c_float+1+2) as x, sum(c_int) as y from cbo_t1 group by c_float, cbo_t1.c_int, key) R group by y, x +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Reducer 2 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_12] + Select Operator [SEL_11] (rows=5 width=20) + Output:["_col0","_col1","_col2"] + Group By Operator [GBY_10] (rows=5 width=20) + Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 + <-Reducer 2 [GROUP] + GROUP [RS_9] + PartitionCols:_col0, _col1 + Group By Operator [GBY_8] (rows=5 width=20) + Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col0, _col1 + Select Operator [SEL_5] (rows=10 width=101) + Output:["_col0","_col1"] + Group By Operator [GBY_4] (rows=10 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Map 1 [GROUP] + GROUP [RS_3] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_2] (rows=10 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float + Select Operator [SEL_1] (rows=20 width=88) + Output:["key","c_int","c_float"] + TableScan [TS_0] (rows=20 width=88) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + +PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key order by a) cbo_t1 join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key order by q/10 desc, r asc) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c order by cbo_t3.c_int+c desc, c +PREHOOK: type: QUERY +POSTHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key order by a) cbo_t1 join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key order by q/10 desc, r asc) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c order by cbo_t3.c_int+c desc, c +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 10 <- Reducer 9 (SORT) +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Reducer 2 (SORT) +Reducer 4 <- Reducer 10 (PARTITION-LEVEL SORT), Reducer 3 (PARTITION-LEVEL SORT) +Reducer 5 <- Map 11 (PARTITION-LEVEL SORT), Reducer 4 (PARTITION-LEVEL SORT) +Reducer 6 <- Reducer 5 (GROUP) +Reducer 7 <- Reducer 6 (SORT) +Reducer 9 <- Map 8 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 7 + File Output Operator [FS_42] + Select Operator [SEL_41] (rows=1 width=20) + Output:["_col0","_col1","_col2"] + <-Reducer 6 [SORT] + SORT [RS_40] + Select Operator [SEL_38] (rows=1 width=20) + Output:["_col0","_col1","_col2"] + Group By Operator [GBY_37] (rows=1 width=20) + Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 + <-Reducer 5 [GROUP] + GROUP [RS_36] + PartitionCols:_col0, _col1 + Group By Operator [GBY_35] (rows=1 width=20) + Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col2, _col6 + Select Operator [SEL_34] (rows=3 width=16) + Output:["_col2","_col6"] + Filter Operator [FIL_33] (rows=3 width=16) + predicate:((_col1 > 0) or (_col6 >= 0)) + Join Operator [JOIN_32] (rows=3 width=16) + Output:["_col1","_col2","_col6"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Map 11 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_31] + PartitionCols:_col0 + Select Operator [SEL_29] (rows=18 width=84) + Output:["_col0","_col1"] + Filter Operator [FIL_45] (rows=18 width=84) + predicate:key is not null + TableScan [TS_27] (rows=20 width=84) + default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + <-Reducer 4 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_30] + PartitionCols:_col0 + Select Operator [SEL_26] (rows=1 width=101) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_25] (rows=1 width=101) + predicate:((_col1 + _col4) >= 0) + Join Operator [JOIN_24] (rows=1 width=101) + Output:["_col0","_col1","_col2","_col4"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Reducer 10 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_23] + PartitionCols:_col0 + Select Operator [SEL_20] (rows=1 width=89) + Output:["_col0","_col1"] + <-Reducer 9 [SORT] + SORT [RS_19] + Select Operator [SEL_17] (rows=1 width=105) + Output:["_col0","_col1","_col2","_col3"] + Group By Operator [GBY_16] (rows=1 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Map 8 [GROUP] + GROUP [RS_15] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_14] (rows=2 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float + Filter Operator [FIL_44] (rows=5 width=93) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and key is not null) + TableScan [TS_11] (rows=20 width=88) + default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + <-Reducer 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_22] + PartitionCols:_col0 + Select Operator [SEL_9] (rows=1 width=97) + Output:["_col0","_col1","_col2"] + <-Reducer 2 [SORT] + SORT [RS_8] + Select Operator [SEL_6] (rows=1 width=97) + Output:["_col0","_col1","_col2"] + Group By Operator [GBY_5] (rows=1 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Map 1 [GROUP] + GROUP [RS_4] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_3] (rows=2 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float + Filter Operator [FIL_43] (rows=5 width=93) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and key is not null) + TableScan [TS_0] (rows=20 width=88) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + +PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b % c asc, b desc) cbo_t1 left outer join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) cbo_t2 on cbo_t1.a=p left outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c having cbo_t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0 order by cbo_t3.c_int % c asc, cbo_t3.c_int desc +PREHOOK: type: QUERY +POSTHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b % c asc, b desc) cbo_t1 left outer join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) cbo_t2 on cbo_t1.a=p left outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c having cbo_t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0 order by cbo_t3.c_int % c asc, cbo_t3.c_int desc +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Reducer 2 (SORT) +Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT), Reducer 9 (PARTITION-LEVEL SORT) +Reducer 5 <- Map 10 (PARTITION-LEVEL SORT), Reducer 4 (PARTITION-LEVEL SORT) +Reducer 6 <- Reducer 5 (GROUP) +Reducer 7 <- Reducer 6 (SORT) +Reducer 9 <- Map 8 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 7 + File Output Operator [FS_39] + Select Operator [SEL_38] (rows=1 width=20) + Output:["_col0","_col1","_col2"] + <-Reducer 6 [SORT] + SORT [RS_37] + Group By Operator [GBY_34] (rows=1 width=20) + Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 + <-Reducer 5 [GROUP] + GROUP [RS_33] + PartitionCols:_col0, _col1 + Group By Operator [GBY_32] (rows=1 width=20) + Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col6, _col2 + Select Operator [SEL_31] (rows=1 width=16) + Output:["_col2","_col6"] + Filter Operator [FIL_30] (rows=1 width=16) + predicate:(((_col1 > 0) or (_col6 >= 0)) and ((_col6 >= 1) or (_col2 >= 1)) and ((UDFToLong(_col6) + _col2) >= 0)) + Join Operator [JOIN_29] (rows=3 width=16) + Output:["_col1","_col2","_col6"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Map 10 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_28] + PartitionCols:_col0 + Select Operator [SEL_26] (rows=18 width=84) + Output:["_col0","_col1"] + Filter Operator [FIL_42] (rows=18 width=84) + predicate:((c_int > 0) and key is not null) + TableScan [TS_24] (rows=20 width=84) + default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + <-Reducer 4 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_27] + PartitionCols:_col0 + Select Operator [SEL_23] (rows=1 width=101) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_22] (rows=1 width=101) + predicate:((_col1 + _col4) >= 0) + Join Operator [JOIN_21] (rows=1 width=101) + Output:["_col0","_col1","_col2","_col4"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Reducer 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_19] + PartitionCols:_col0 + Select Operator [SEL_9] (rows=1 width=97) + Output:["_col0","_col1","_col2"] + <-Reducer 2 [SORT] + SORT [RS_8] + Select Operator [SEL_6] (rows=1 width=105) + Output:["_col0","_col1","_col2","_col3"] + Group By Operator [GBY_5] (rows=1 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Map 1 [GROUP] + GROUP [RS_4] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_3] (rows=1 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float + Filter Operator [FIL_40] (rows=2 width=93) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) + TableScan [TS_0] (rows=20 width=88) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + <-Reducer 9 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_20] + PartitionCols:_col0 + Select Operator [SEL_17] (rows=1 width=89) + Output:["_col0","_col1"] + Group By Operator [GBY_16] (rows=1 width=93) + Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Map 8 [GROUP] + GROUP [RS_15] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_14] (rows=1 width=93) + Output:["_col0","_col1","_col2"],keys:key, c_int, c_float + Filter Operator [FIL_41] (rows=2 width=93) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) + TableScan [TS_11] (rows=20 width=88) + default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + +PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b+c, a desc) cbo_t1 right outer join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) cbo_t2 on cbo_t1.a=p right outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 2) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c +PREHOOK: type: QUERY +POSTHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b+c, a desc) cbo_t1 right outer join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) cbo_t2 on cbo_t1.a=p right outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 2) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Reducer 2 (SORT) +Reducer 4 <- Map 8 (PARTITION-LEVEL SORT), Reducer 3 (PARTITION-LEVEL SORT), Reducer 7 (PARTITION-LEVEL SORT) +Reducer 5 <- Reducer 4 (GROUP) +Reducer 7 <- Map 6 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 5 + File Output Operator [FS_31] + Select Operator [SEL_30] (rows=1 width=20) + Output:["_col0","_col1","_col2"] + Group By Operator [GBY_29] (rows=1 width=20) + Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 + <-Reducer 4 [GROUP] + GROUP [RS_28] + PartitionCols:_col0, _col1 + Group By Operator [GBY_27] (rows=1 width=20) + Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col2, _col6 + Select Operator [SEL_26] (rows=1 width=20) + Output:["_col2","_col6"] + Filter Operator [FIL_25] (rows=1 width=20) + predicate:(((_col1 + _col4) >= 2) and ((_col1 > 0) or (_col6 >= 0))) + Join Operator [JOIN_24] (rows=4 width=20) + Output:["_col1","_col2","_col4","_col6"],condition map:[{"":"{\"type\":\"Right Outer\",\"left\":0,\"right\":1}"},{"":"{\"type\":\"Right Outer\",\"left\":0,\"right\":2}"}],keys:{"0":"_col0","1":"_col0","2":"_col0"} + <-Map 8 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_23] + PartitionCols:_col0 + Select Operator [SEL_20] (rows=20 width=84) + Output:["_col0","_col1"] + TableScan [TS_19] (rows=20 width=84) + default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + <-Reducer 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_21] + PartitionCols:_col0 + Select Operator [SEL_9] (rows=1 width=97) + Output:["_col0","_col1","_col2"] + <-Reducer 2 [SORT] + SORT [RS_8] + Select Operator [SEL_6] (rows=1 width=105) + Output:["_col0","_col1","_col2","_col3"] + Group By Operator [GBY_5] (rows=1 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Map 1 [GROUP] + GROUP [RS_4] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_3] (rows=1 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float + Filter Operator [FIL_32] (rows=2 width=93) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0)) + TableScan [TS_0] (rows=20 width=88) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + <-Reducer 7 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_22] + PartitionCols:_col0 + Select Operator [SEL_17] (rows=1 width=89) + Output:["_col0","_col1"] + Group By Operator [GBY_16] (rows=1 width=93) + Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Map 6 [GROUP] + GROUP [RS_15] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_14] (rows=1 width=93) + Output:["_col0","_col1","_col2"],keys:key, c_int, c_float + Filter Operator [FIL_33] (rows=2 width=93) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0)) + TableScan [TS_11] (rows=20 width=88) + default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + +PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by c+a desc) cbo_t1 full outer join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by p+q desc, r asc) cbo_t2 on cbo_t1.a=p full outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c having cbo_t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0 order by cbo_t3.c_int +PREHOOK: type: QUERY +POSTHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by c+a desc) cbo_t1 full outer join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by p+q desc, r asc) cbo_t2 on cbo_t1.a=p full outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c having cbo_t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0 order by cbo_t3.c_int +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Reducer 2 (SORT) +Reducer 4 <- Map 10 (PARTITION-LEVEL SORT), Reducer 3 (PARTITION-LEVEL SORT), Reducer 9 (PARTITION-LEVEL SORT) +Reducer 5 <- Reducer 4 (GROUP) +Reducer 6 <- Reducer 5 (SORT) +Reducer 8 <- Map 7 (GROUP) +Reducer 9 <- Reducer 8 (SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 6 + File Output Operator [FS_37] + Select Operator [SEL_36] (rows=1 width=20) + Output:["_col0","_col1","_col2"] + <-Reducer 5 [SORT] + SORT [RS_35] + Group By Operator [GBY_33] (rows=1 width=20) + Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 + <-Reducer 4 [GROUP] + GROUP [RS_32] + PartitionCols:_col0, _col1 + Group By Operator [GBY_31] (rows=1 width=20) + Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col6, _col2 + Select Operator [SEL_30] (rows=1 width=20) + Output:["_col2","_col6"] + Filter Operator [FIL_29] (rows=1 width=20) + predicate:(((_col1 + _col4) >= 0) and ((_col1 > 0) or (_col6 >= 0)) and ((_col6 >= 1) or (_col2 >= 1)) and ((UDFToLong(_col6) + _col2) >= 0)) + Join Operator [JOIN_28] (rows=4 width=20) + Output:["_col1","_col2","_col4","_col6"],condition map:[{"":"{\"type\":\"Outer\",\"left\":0,\"right\":1}"},{"":"{\"type\":\"Right Outer\",\"left\":0,\"right\":2}"}],keys:{"0":"_col0","1":"_col0","2":"_col0"} + <-Map 10 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_27] + PartitionCols:_col0 + Select Operator [SEL_24] (rows=20 width=84) + Output:["_col0","_col1"] + Filter Operator [FIL_40] (rows=20 width=84) + predicate:(c_int > 0) + TableScan [TS_22] (rows=20 width=84) + default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + <-Reducer 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_25] + PartitionCols:_col0 + Select Operator [SEL_9] (rows=1 width=97) + Output:["_col0","_col1","_col2"] + <-Reducer 2 [SORT] + SORT [RS_8] + Select Operator [SEL_6] (rows=1 width=105) + Output:["_col0","_col1","_col2","_col3"] + Group By Operator [GBY_5] (rows=1 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Map 1 [GROUP] + GROUP [RS_4] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_3] (rows=1 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float + Filter Operator [FIL_38] (rows=2 width=93) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0)) + TableScan [TS_0] (rows=20 width=88) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + <-Reducer 9 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_26] + PartitionCols:_col0 + Select Operator [SEL_20] (rows=1 width=89) + Output:["_col0","_col1"] + <-Reducer 8 [SORT] + SORT [RS_19] + Select Operator [SEL_17] (rows=1 width=105) + Output:["_col0","_col1","_col2","_col3"] + Group By Operator [GBY_16] (rows=1 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Map 7 [GROUP] + GROUP [RS_15] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_14] (rows=1 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float + Filter Operator [FIL_39] (rows=2 width=93) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0)) + TableScan [TS_11] (rows=20 width=88) + default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + +PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) cbo_t1 join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c +PREHOOK: type: QUERY +POSTHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) cbo_t1 join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT), Reducer 7 (PARTITION-LEVEL SORT) +Reducer 4 <- Map 8 (PARTITION-LEVEL SORT), Reducer 3 (PARTITION-LEVEL SORT) +Reducer 5 <- Reducer 4 (GROUP) +Reducer 7 <- Map 6 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 5 + File Output Operator [FS_33] + Select Operator [SEL_32] (rows=1 width=20) + Output:["_col0","_col1","_col2"] + Group By Operator [GBY_31] (rows=1 width=20) + Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 + <-Reducer 4 [GROUP] + GROUP [RS_30] + PartitionCols:_col0, _col1 + Group By Operator [GBY_29] (rows=1 width=20) + Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col2, _col6 + Select Operator [SEL_28] (rows=3 width=16) + Output:["_col2","_col6"] + Filter Operator [FIL_27] (rows=3 width=16) + predicate:((_col1 > 0) or (_col6 >= 0)) + Join Operator [JOIN_26] (rows=3 width=16) + Output:["_col1","_col2","_col6"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Map 8 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_25] + PartitionCols:_col0 + Select Operator [SEL_23] (rows=18 width=84) + Output:["_col0","_col1"] + Filter Operator [FIL_36] (rows=18 width=84) + predicate:key is not null + TableScan [TS_21] (rows=20 width=84) + default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + <-Reducer 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_24] + PartitionCols:_col0 + Select Operator [SEL_20] (rows=1 width=101) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_19] (rows=1 width=101) + predicate:((_col1 + _col4) >= 0) + Join Operator [JOIN_18] (rows=1 width=101) + Output:["_col0","_col1","_col2","_col4"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_16] + PartitionCols:_col0 + Select Operator [SEL_6] (rows=1 width=97) + Output:["_col0","_col1","_col2"] + Group By Operator [GBY_5] (rows=1 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Map 1 [GROUP] + GROUP [RS_4] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_3] (rows=1 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float + Filter Operator [FIL_34] (rows=2 width=93) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) + TableScan [TS_0] (rows=20 width=88) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + <-Reducer 7 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_17] + PartitionCols:_col0 + Select Operator [SEL_14] (rows=1 width=89) + Output:["_col0","_col1"] + Group By Operator [GBY_13] (rows=1 width=93) + Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Map 6 [GROUP] + GROUP [RS_12] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_11] (rows=1 width=93) + Output:["_col0","_col1","_col2"],keys:key, c_int, c_float + Filter Operator [FIL_35] (rows=2 width=93) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) + TableScan [TS_8] (rows=20 width=88) + default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + +PREHOOK: query: explain select unionsrc.key FROM (select 'tst1' as key, count(1) as value from src) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain select unionsrc.key FROM (select 'tst1' as key, count(1) as value from src) unionsrc +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Stage-0 + Fetch Operator + limit:1 + +PREHOOK: query: explain select unionsrc.key FROM (select 'max' as key, max(c_int) as value from cbo_t3 s1 + UNION ALL + select 'min' as key, min(c_int) as value from cbo_t3 s2 + UNION ALL + select 'avg' as key, avg(c_int) as value from cbo_t3 s3) unionsrc order by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select unionsrc.key FROM (select 'max' as key, max(c_int) as value from cbo_t3 s1 + UNION ALL + select 'min' as key, min(c_int) as value from cbo_t3 s2 + UNION ALL + select 'avg' as key, avg(c_int) as value from cbo_t3 s3) unionsrc order by unionsrc.key +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Reducer 2 (SORT), Reducer 5 (SORT), Reducer 7 (SORT) +Reducer 5 <- Map 4 (GROUP) +Reducer 7 <- Map 6 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_26] + Select Operator [SEL_25] (rows=3 width=87) + Output:["_col0"] + <-Reducer 2 [SORT] + SORT [RS_24] + Select Operator [SEL_5] (rows=1 width=87) + Output:["_col0"] + Group By Operator [GBY_4] (rows=1 width=8) + Output:["_col0"],aggregations:["count(VALUE._col0)"] + <-Map 1 [GROUP] + GROUP [RS_3] + Group By Operator [GBY_2] (rows=1 width=8) + Output:["_col0"],aggregations:["count(key)"] + Select Operator [SEL_1] (rows=20 width=80) + Output:["key"] + TableScan [TS_0] (rows=20 width=80) + default@cbo_t3,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + <-Reducer 5 [SORT] + SORT [RS_24] + Select Operator [SEL_12] (rows=1 width=87) + Output:["_col0"] + Group By Operator [GBY_11] (rows=1 width=8) + Output:["_col0"],aggregations:["count(VALUE._col0)"] + <-Map 4 [GROUP] + GROUP [RS_10] + Group By Operator [GBY_9] (rows=1 width=8) + Output:["_col0"],aggregations:["count(key)"] + Select Operator [SEL_8] (rows=20 width=80) + Output:["key"] + TableScan [TS_7] (rows=20 width=80) + default@cbo_t3,s2,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + <-Reducer 7 [SORT] + SORT [RS_24] + Select Operator [SEL_21] (rows=1 width=87) + Output:["_col0"] + Group By Operator [GBY_20] (rows=1 width=8) + Output:["_col0"],aggregations:["count(VALUE._col0)"] + <-Map 6 [GROUP] + GROUP [RS_19] + Group By Operator [GBY_18] (rows=1 width=8) + Output:["_col0"],aggregations:["count(key)"] + Select Operator [SEL_17] (rows=20 width=80) + Output:["key"] + TableScan [TS_16] (rows=20 width=80) + default@cbo_t3,s3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + +PREHOOK: query: explain select unionsrc.key, count(1) FROM (select 'max' as key, max(c_int) as value from cbo_t3 s1 + UNION ALL + select 'min' as key, min(c_int) as value from cbo_t3 s2 + UNION ALL + select 'avg' as key, avg(c_int) as value from cbo_t3 s3) unionsrc group by unionsrc.key order by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select unionsrc.key, count(1) FROM (select 'max' as key, max(c_int) as value from cbo_t3 s1 + UNION ALL + select 'min' as key, min(c_int) as value from cbo_t3 s2 + UNION ALL + select 'avg' as key, avg(c_int) as value from cbo_t3 s3) unionsrc group by unionsrc.key order by unionsrc.key +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Reducer 2 (GROUP), Reducer 6 (GROUP), Reducer 8 (GROUP) +Reducer 4 <- Reducer 3 (SORT) +Reducer 6 <- Map 5 (GROUP) +Reducer 8 <- Map 7 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 4 + File Output Operator [FS_31] + Select Operator [SEL_30] (rows=1 width=95) + Output:["_col0","_col1"] + <-Reducer 3 [SORT] + SORT [RS_29] + Group By Operator [GBY_27] (rows=1 width=95) + Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 + <-Reducer 2 [GROUP] + GROUP [RS_26] + PartitionCols:_col0 + Group By Operator [GBY_25] (rows=1 width=95) + Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 + Select Operator [SEL_5] (rows=1 width=87) + Output:["_col0"] + Group By Operator [GBY_4] (rows=1 width=8) + Output:["_col0"],aggregations:["count(VALUE._col0)"] + <-Map 1 [GROUP] + GROUP [RS_3] + Group By Operator [GBY_2] (rows=1 width=8) + Output:["_col0"],aggregations:["count(key)"] + Select Operator [SEL_1] (rows=20 width=80) + Output:["key"] + TableScan [TS_0] (rows=20 width=80) + default@cbo_t3,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + <-Reducer 6 [GROUP] + GROUP [RS_26] + PartitionCols:_col0 + Group By Operator [GBY_25] (rows=1 width=95) + Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 + Select Operator [SEL_12] (rows=1 width=87) + Output:["_col0"] + Group By Operator [GBY_11] (rows=1 width=8) + Output:["_col0"],aggregations:["count(VALUE._col0)"] + <-Map 5 [GROUP] + GROUP [RS_10] + Group By Operator [GBY_9] (rows=1 width=8) + Output:["_col0"],aggregations:["count(key)"] + Select Operator [SEL_8] (rows=20 width=80) + Output:["key"] + TableScan [TS_7] (rows=20 width=80) + default@cbo_t3,s2,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + <-Reducer 8 [GROUP] + GROUP [RS_26] + PartitionCols:_col0 + Group By Operator [GBY_25] (rows=1 width=95) + Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 + Select Operator [SEL_21] (rows=1 width=87) + Output:["_col0"] + Group By Operator [GBY_20] (rows=1 width=8) + Output:["_col0"],aggregations:["count(VALUE._col0)"] + <-Map 7 [GROUP] + GROUP [RS_19] + Group By Operator [GBY_18] (rows=1 width=8) + Output:["_col0"],aggregations:["count(key)"] + Select Operator [SEL_17] (rows=20 width=80) + Output:["key"] + TableScan [TS_16] (rows=20 width=80) + default@cbo_t3,s3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + +PREHOOK: query: explain select cbo_t1.key from cbo_t1 join cbo_t3 where cbo_t1.key=cbo_t3.key and cbo_t1.key >= 1 +PREHOOK: type: QUERY +POSTHOOK: query: explain select cbo_t1.key from cbo_t1 join cbo_t3 where cbo_t1.key=cbo_t3.key and cbo_t1.key >= 1 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_10] + Join Operator [JOIN_8] (rows=18 width=85) + Output:["_col0"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_6] + PartitionCols:_col0 + Select Operator [SEL_2] (rows=6 width=85) + Output:["_col0"] + Filter Operator [FIL_11] (rows=6 width=85) + predicate:(UDFToDouble(key) >= 1.0) + TableScan [TS_0] (rows=20 width=80) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_7] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=6 width=85) + Output:["_col0"] + Filter Operator [FIL_12] (rows=6 width=85) + predicate:(UDFToDouble(key) >= 1.0) + TableScan [TS_3] (rows=20 width=80) + default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + +PREHOOK: query: explain select cbo_t1.c_int, cbo_t2.c_int from cbo_t1 left outer join cbo_t2 on cbo_t1.key=cbo_t2.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select cbo_t1.c_int, cbo_t2.c_int from cbo_t1 left outer join cbo_t2 on cbo_t1.key=cbo_t2.key +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_8] + Select Operator [SEL_7] (rows=100 width=8) + Output:["_col0","_col1"] + Join Operator [JOIN_6] (rows=100 width=8) + Output:["_col1","_col3"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_4] + PartitionCols:_col0 + Select Operator [SEL_1] (rows=20 width=84) + Output:["_col0","_col1"] + TableScan [TS_0] (rows=20 width=84) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_5] + PartitionCols:_col0 + Select Operator [SEL_3] (rows=20 width=84) + Output:["_col0","_col1"] + TableScan [TS_2] (rows=20 width=84) + default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + +PREHOOK: query: explain select cbo_t1.c_int, cbo_t2.c_int from cbo_t1 full outer join cbo_t2 on cbo_t1.key=cbo_t2.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select cbo_t1.c_int, cbo_t2.c_int from cbo_t1 full outer join cbo_t2 on cbo_t1.key=cbo_t2.key +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_8] + Select Operator [SEL_7] (rows=100 width=8) + Output:["_col0","_col1"] + Join Operator [JOIN_6] (rows=100 width=8) + Output:["_col1","_col3"],condition map:[{"":"{\"type\":\"Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_4] + PartitionCols:_col0 + Select Operator [SEL_1] (rows=20 width=84) + Output:["_col0","_col1"] + TableScan [TS_0] (rows=20 width=84) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_5] + PartitionCols:_col0 + Select Operator [SEL_3] (rows=20 width=84) + Output:["_col0","_col1"] + TableScan [TS_2] (rows=20 width=84) + default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + +PREHOOK: query: explain select b, cbo_t1.c, cbo_t2.p, q, cbo_t3.c_int from (select key as a, c_int as b, cbo_t1.c_float as c from cbo_t1) cbo_t1 join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key +PREHOOK: type: QUERY +POSTHOOK: query: explain select b, cbo_t1.c, cbo_t2.p, q, cbo_t3.c_int from (select key as a, c_int as b, cbo_t1.c_float as c from cbo_t1) cbo_t1 join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT), Map 4 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_14] + Select Operator [SEL_13] (rows=291 width=101) + Output:["_col0","_col1","_col2","_col3","_col4"] + Join Operator [JOIN_12] (rows=291 width=101) + Output:["_col1","_col2","_col4","_col5","_col6"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"},{"":"{\"type\":\"Inner\",\"left\":0,\"right\":2}"}],keys:{"0":"_col0","1":"_col0","2":"_col0"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_9] + PartitionCols:_col0 + Select Operator [SEL_2] (rows=18 width=87) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_15] (rows=18 width=87) + predicate:key is not null + TableScan [TS_0] (rows=20 width=88) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_10] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=18 width=84) + Output:["_col0","_col1"] + Filter Operator [FIL_16] (rows=18 width=84) + predicate:key is not null + TableScan [TS_3] (rows=20 width=84) + default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + <-Map 4 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_11] + PartitionCols:_col0 + Select Operator [SEL_8] (rows=18 width=84) + Output:["_col0","_col1"] + Filter Operator [FIL_17] (rows=18 width=84) + predicate:key is not null + TableScan [TS_6] (rows=20 width=84) + default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + +PREHOOK: query: explain select key, cbo_t1.c_int, cbo_t2.p, q from cbo_t1 join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2) cbo_t2 on cbo_t1.key=p join (select key as a, c_int as b, cbo_t3.c_float as c from cbo_t3)cbo_t3 on cbo_t1.key=a +PREHOOK: type: QUERY +POSTHOOK: query: explain select key, cbo_t1.c_int, cbo_t2.p, q from cbo_t1 join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2) cbo_t2 on cbo_t1.key=p join (select key as a, c_int as b, cbo_t3.c_float as c from cbo_t3)cbo_t3 on cbo_t1.key=a +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT), Map 4 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_14] + Select Operator [SEL_13] (rows=291 width=178) + Output:["_col0","_col1","_col2","_col3"] + Join Operator [JOIN_12] (rows=291 width=178) + Output:["_col0","_col1","_col3","_col4"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"},{"":"{\"type\":\"Inner\",\"left\":0,\"right\":2}"}],keys:{"0":"_col0","1":"_col0","2":"_col0"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_9] + PartitionCols:_col0 + Select Operator [SEL_2] (rows=18 width=84) + Output:["_col0","_col1"] + Filter Operator [FIL_15] (rows=18 width=84) + predicate:key is not null + TableScan [TS_0] (rows=20 width=84) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_10] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=18 width=80) + Output:["_col0"] + Filter Operator [FIL_16] (rows=18 width=80) + predicate:key is not null + TableScan [TS_3] (rows=20 width=80) + default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + <-Map 4 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_11] + PartitionCols:_col0 + Select Operator [SEL_8] (rows=18 width=84) + Output:["_col0","_col1"] + Filter Operator [FIL_17] (rows=18 width=84) + predicate:key is not null + TableScan [TS_6] (rows=20 width=84) + default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + +PREHOOK: query: explain select * from (select q, b, cbo_t2.p, cbo_t1.c, cbo_t3.c_int from (select key as a, c_int as b, cbo_t1.c_float as c from cbo_t1 where (cbo_t1.c_int + 1 == 2) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0)) cbo_t1 full outer join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2 where (cbo_t2.c_int + 1 == 2) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0)) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q == 2) and (b > 0 or c_int >= 0)) R where (q + 1 = 2) and (R.b > 0 or c_int >= 0) +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from (select q, b, cbo_t2.p, cbo_t1.c, cbo_t3.c_int from (select key as a, c_int as b, cbo_t1.c_float as c from cbo_t1 where (cbo_t1.c_int + 1 == 2) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0)) cbo_t1 full outer join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2 where (cbo_t2.c_int + 1 == 2) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0)) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q == 2) and (b > 0 or c_int >= 0)) R where (q + 1 = 2) and (R.b > 0 or c_int >= 0) +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 4 (PARTITION-LEVEL SORT) +Reducer 3 <- Map 5 (PARTITION-LEVEL SORT), Reducer 2 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_19] + Select Operator [SEL_18] (rows=36 width=101) + Output:["_col0","_col1","_col2","_col3","_col4"] + Filter Operator [FIL_17] (rows=36 width=101) + predicate:((_col1 > 0) or (_col6 >= 0)) + Join Operator [JOIN_16] (rows=36 width=101) + Output:["_col1","_col2","_col3","_col4","_col6"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Map 5 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_15] + PartitionCols:_col0 + Select Operator [SEL_13] (rows=18 width=84) + Output:["_col0","_col1"] + Filter Operator [FIL_22] (rows=18 width=84) + predicate:key is not null + TableScan [TS_11] (rows=20 width=84) + default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_14] + PartitionCols:_col0 + Filter Operator [FIL_9] (rows=10 width=182) + predicate:(((_col1 + _col4) = 2) and ((_col4 + 1) = 2)) + Join Operator [JOIN_8] (rows=40 width=182) + Output:["_col0","_col1","_col2","_col3","_col4"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_6] + PartitionCols:_col0 + Select Operator [SEL_2] (rows=9 width=93) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_20] (rows=9 width=93) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + TableScan [TS_0] (rows=20 width=88) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + <-Map 4 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_7] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=9 width=89) + Output:["_col0","_col1"] + Filter Operator [FIL_21] (rows=9 width=93) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + TableScan [TS_3] (rows=20 width=88) + default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + +PREHOOK: query: explain select * from (select q, b, cbo_t2.p, cbo_t1.c, cbo_t3.c_int from (select key as a, c_int as b, cbo_t1.c_float as c from cbo_t1 where (cbo_t1.c_int + 1 == 2) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0)) cbo_t1 right outer join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2 where (cbo_t2.c_int + 1 == 2) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0)) cbo_t2 on cbo_t1.a=p right outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q == 2) and (b > 0 or c_int >= 0)) R where (q + 1 = 2) and (R.b > 0 or c_int >= 0) +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from (select q, b, cbo_t2.p, cbo_t1.c, cbo_t3.c_int from (select key as a, c_int as b, cbo_t1.c_float as c from cbo_t1 where (cbo_t1.c_int + 1 == 2) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0)) cbo_t1 right outer join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2 where (cbo_t2.c_int + 1 == 2) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0)) cbo_t2 on cbo_t1.a=p right outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q == 2) and (b > 0 or c_int >= 0)) R where (q + 1 = 2) and (R.b > 0 or c_int >= 0) +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT), Map 4 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_14] + Select Operator [SEL_13] (rows=50 width=101) + Output:["_col0","_col1","_col2","_col3","_col4"] + Filter Operator [FIL_12] (rows=50 width=101) + predicate:(((_col1 + _col4) = 2) and ((_col1 > 0) or (_col6 >= 0)) and ((_col4 + 1) = 2)) + Join Operator [JOIN_11] (rows=200 width=101) + Output:["_col1","_col2","_col3","_col4","_col6"],condition map:[{"":"{\"type\":\"Right Outer\",\"left\":0,\"right\":1}"},{"":"{\"type\":\"Right Outer\",\"left\":0,\"right\":2}"}],keys:{"0":"_col0","1":"_col0","2":"_col0"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_8] + PartitionCols:_col0 + Select Operator [SEL_2] (rows=10 width=93) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_15] (rows=10 width=93) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0))) + TableScan [TS_0] (rows=20 width=88) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_9] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=10 width=89) + Output:["_col0","_col1"] + Filter Operator [FIL_16] (rows=10 width=93) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0))) + TableScan [TS_3] (rows=20 width=88) + default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + <-Map 4 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_10] + PartitionCols:_col0 + Select Operator [SEL_7] (rows=20 width=84) + Output:["_col0","_col1"] + TableScan [TS_6] (rows=20 width=84) + default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + +PREHOOK: query: explain select key, (c_int+1)+2 as x, sum(c_int) from cbo_t1 group by c_float, cbo_t1.c_int, key order by x limit 1 +PREHOOK: type: QUERY +POSTHOOK: query: explain select key, (c_int+1)+2 as x, sum(c_int) from cbo_t1 group by c_float, cbo_t1.c_int, key order by x limit 1 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Reducer 2 (SORT) + +Stage-0 + Fetch Operator + limit:1 + Stage-1 + Reducer 3 + File Output Operator [FS_10] + Limit [LIM_9] (rows=1 width=97) + Number of rows:1 + Select Operator [SEL_8] (rows=10 width=97) + Output:["_col0","_col1","_col2"] + <-Reducer 2 [SORT] + SORT [RS_7] + Select Operator [SEL_5] (rows=10 width=97) + Output:["_col0","_col1","_col2"] + Group By Operator [GBY_4] (rows=10 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Map 1 [GROUP] + GROUP [RS_3] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_2] (rows=10 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float + Select Operator [SEL_1] (rows=20 width=88) + Output:["key","c_int","c_float"] + TableScan [TS_0] (rows=20 width=88) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + +PREHOOK: query: explain select x, y, count(*) from (select key, (c_int+c_float+1+2) as x, sum(c_int) as y from cbo_t1 group by c_float, cbo_t1.c_int, key) R group by y, x order by x,y limit 1 +PREHOOK: type: QUERY +POSTHOOK: query: explain select x, y, count(*) from (select key, (c_int+c_float+1+2) as x, sum(c_int) as y from cbo_t1 group by c_float, cbo_t1.c_int, key) R group by y, x order by x,y limit 1 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Reducer 2 (GROUP) +Reducer 4 <- Reducer 3 (SORT) + +Stage-0 + Fetch Operator + limit:1 + Stage-1 + Reducer 4 + File Output Operator [FS_15] + Limit [LIM_14] (rows=1 width=20) + Number of rows:1 + Select Operator [SEL_13] (rows=5 width=20) + Output:["_col0","_col1","_col2"] + <-Reducer 3 [SORT] + SORT [RS_12] + Group By Operator [GBY_10] (rows=5 width=20) + Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 + <-Reducer 2 [GROUP] + GROUP [RS_9] + PartitionCols:_col0, _col1 + Group By Operator [GBY_8] (rows=5 width=20) + Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col1, _col0 + Select Operator [SEL_5] (rows=10 width=101) + Output:["_col0","_col1"] + Group By Operator [GBY_4] (rows=10 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Map 1 [GROUP] + GROUP [RS_3] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_2] (rows=10 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float + Select Operator [SEL_1] (rows=20 width=88) + Output:["key","c_int","c_float"] + TableScan [TS_0] (rows=20 width=88) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + +PREHOOK: query: explain select key from(select key from (select key from cbo_t1 limit 5)cbo_t2 limit 5)cbo_t3 limit 5 +PREHOOK: type: QUERY +POSTHOOK: query: explain select key from(select key from (select key from cbo_t1 limit 5)cbo_t2 limit 5)cbo_t3 limit 5 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Reducer 2 (GROUP) + +Stage-0 + Fetch Operator + limit:5 + Stage-1 + Reducer 3 + File Output Operator [FS_13] + Limit [LIM_12] (rows=5 width=85) + Number of rows:5 + Limit [LIM_10] (rows=5 width=85) + Number of rows:5 + Select Operator [SEL_9] (rows=5 width=85) + Output:["_col0"] + <-Reducer 2 [GROUP] + GROUP [RS_8] + Limit [LIM_7] (rows=5 width=85) + Number of rows:5 + Limit [LIM_5] (rows=5 width=85) + Number of rows:5 + Select Operator [SEL_4] (rows=5 width=85) + Output:["_col0"] + <-Map 1 [GROUP] + GROUP [RS_3] + Limit [LIM_2] (rows=5 width=85) + Number of rows:5 + Select Operator [SEL_1] (rows=20 width=80) + Output:["_col0"] + TableScan [TS_0] (rows=20 width=80) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + +PREHOOK: query: explain select key, c_int from(select key, c_int from (select key, c_int from cbo_t1 order by c_int limit 5)cbo_t1 order by c_int limit 5)cbo_t2 order by c_int limit 5 +PREHOOK: type: QUERY +POSTHOOK: query: explain select key, c_int from(select key, c_int from (select key, c_int from cbo_t1 order by c_int limit 5)cbo_t1 order by c_int limit 5)cbo_t2 order by c_int limit 5 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (SORT) +Reducer 3 <- Reducer 2 (SORT) +Reducer 4 <- Reducer 3 (SORT) + +Stage-0 + Fetch Operator + limit:5 + Stage-1 + Reducer 4 + File Output Operator [FS_13] + Limit [LIM_12] (rows=5 width=89) + Number of rows:5 + Select Operator [SEL_11] (rows=5 width=89) + Output:["_col0","_col1"] + <-Reducer 3 [SORT] + SORT [RS_10] + Limit [LIM_8] (rows=5 width=89) + Number of rows:5 + Select Operator [SEL_7] (rows=5 width=89) + Output:["_col0","_col1"] + <-Reducer 2 [SORT] + SORT [RS_6] + Limit [LIM_4] (rows=5 width=89) + Number of rows:5 + Select Operator [SEL_3] (rows=20 width=84) + Output:["_col0","_col1"] + <-Map 1 [SORT] + SORT [RS_2] + Select Operator [SEL_1] (rows=20 width=84) + Output:["_col0","_col1"] + TableScan [TS_0] (rows=20 width=84) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + +PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key order by a limit 5) cbo_t1 join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key order by q/10 desc, r asc limit 5) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c order by cbo_t3.c_int+c desc, c limit 5 +PREHOOK: type: QUERY +POSTHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key order by a limit 5) cbo_t1 join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key order by q/10 desc, r asc limit 5) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c order by cbo_t3.c_int+c desc, c limit 5 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 10 <- Reducer 9 (SORT) +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Reducer 2 (SORT) +Reducer 4 <- Reducer 10 (PARTITION-LEVEL SORT), Reducer 3 (PARTITION-LEVEL SORT) +Reducer 5 <- Map 11 (PARTITION-LEVEL SORT), Reducer 4 (PARTITION-LEVEL SORT) +Reducer 6 <- Reducer 5 (GROUP) +Reducer 7 <- Reducer 6 (SORT) +Reducer 9 <- Map 8 (GROUP) + +Stage-0 + Fetch Operator + limit:5 + Stage-1 + Reducer 7 + File Output Operator [FS_49] + Limit [LIM_48] (rows=1 width=20) + Number of rows:5 + Select Operator [SEL_47] (rows=1 width=20) + Output:["_col0","_col1","_col2"] + <-Reducer 6 [SORT] + SORT [RS_46] + Select Operator [SEL_44] (rows=1 width=20) + Output:["_col0","_col1","_col2"] + Group By Operator [GBY_43] (rows=1 width=20) + Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 + <-Reducer 5 [GROUP] + GROUP [RS_42] + PartitionCols:_col0, _col1 + Group By Operator [GBY_41] (rows=1 width=20) + Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col2, _col6 + Select Operator [SEL_40] (rows=3 width=16) + Output:["_col2","_col6"] + Filter Operator [FIL_39] (rows=3 width=16) + predicate:((_col1 > 0) or (_col6 >= 0)) + Join Operator [JOIN_38] (rows=3 width=16) + Output:["_col1","_col2","_col6"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Map 11 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_37] + PartitionCols:_col0 + Select Operator [SEL_35] (rows=18 width=84) + Output:["_col0","_col1"] + Filter Operator [FIL_52] (rows=18 width=84) + predicate:key is not null + TableScan [TS_33] (rows=20 width=84) + default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + <-Reducer 4 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_36] + PartitionCols:_col0 + Select Operator [SEL_32] (rows=1 width=101) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_31] (rows=1 width=101) + predicate:((_col1 + _col4) >= 0) + Join Operator [JOIN_30] (rows=2 width=101) + Output:["_col0","_col1","_col2","_col4"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Reducer 10 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_29] + PartitionCols:_col0 + Filter Operator [FIL_26] (rows=2 width=105) + predicate:_col0 is not null + Limit [LIM_24] (rows=3 width=105) + Number of rows:5 + Select Operator [SEL_23] (rows=3 width=105) + Output:["_col0","_col1"] + <-Reducer 9 [SORT] + SORT [RS_22] + Select Operator [SEL_20] (rows=3 width=105) + Output:["_col0","_col1","_col2","_col3"] + Group By Operator [GBY_19] (rows=3 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Map 8 [GROUP] + GROUP [RS_18] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_17] (rows=3 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float + Filter Operator [FIL_51] (rows=6 width=93) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0))) + TableScan [TS_14] (rows=20 width=88) + default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + <-Reducer 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_28] + PartitionCols:_col0 + Filter Operator [FIL_12] (rows=2 width=97) + predicate:_col0 is not null + Limit [LIM_10] (rows=3 width=97) + Number of rows:5 + Select Operator [SEL_9] (rows=3 width=97) + Output:["_col0","_col1","_col2"] + <-Reducer 2 [SORT] + SORT [RS_8] + Select Operator [SEL_6] (rows=3 width=97) + Output:["_col0","_col1","_col2"] + Group By Operator [GBY_5] (rows=3 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Map 1 [GROUP] + GROUP [RS_4] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_3] (rows=3 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float + Filter Operator [FIL_50] (rows=6 width=93) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0))) + TableScan [TS_0] (rows=20 width=88) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + +PREHOOK: query: explain select cbo_t1.c_int from cbo_t1 left semi join cbo_t2 on cbo_t1.key=cbo_t2.key where (cbo_t1.c_int + 1 == 2) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) +PREHOOK: type: QUERY +POSTHOOK: query: explain select cbo_t1.c_int from cbo_t1 left semi join cbo_t2 on cbo_t1.key=cbo_t2.key where (cbo_t1.c_int + 1 == 2) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_12] + Select Operator [SEL_11] (rows=9 width=4) + Output:["_col0"] + Join Operator [JOIN_10] (rows=9 width=4) + Output:["_col1"],condition map:[{"":"{\"type\":\"Left Semi\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_8] + PartitionCols:_col0 + Select Operator [SEL_2] (rows=9 width=93) + Output:["_col0","_col1"] + Filter Operator [FIL_13] (rows=9 width=93) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + TableScan [TS_0] (rows=20 width=88) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_9] + PartitionCols:_col0 + Group By Operator [GBY_7] (rows=5 width=85) + Output:["_col0"],keys:_col0 + Select Operator [SEL_5] (rows=18 width=80) + Output:["_col0"] + Filter Operator [FIL_14] (rows=18 width=80) + predicate:key is not null + TableScan [TS_3] (rows=20 width=80) + default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + +PREHOOK: query: explain select * from (select c, b, a from (select key as a, c_int as b, cbo_t1.c_float as c from cbo_t1 where (cbo_t1.c_int + 1 == 2) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0)) cbo_t1 left semi join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2 where (cbo_t2.c_int + 1 == 2) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0)) cbo_t2 on cbo_t1.a=p left semi join cbo_t3 on cbo_t1.a=key where (b + 1 == 2) and (b > 0 or c >= 0)) R where (b + 1 = 2) and (R.b > 0 or c >= 0) +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from (select c, b, a from (select key as a, c_int as b, cbo_t1.c_float as c from cbo_t1 where (cbo_t1.c_int + 1 == 2) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0)) cbo_t1 left semi join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2 where (cbo_t2.c_int + 1 == 2) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0)) cbo_t2 on cbo_t1.a=p left semi join cbo_t3 on cbo_t1.a=key where (b + 1 == 2) and (b > 0 or c >= 0)) R where (b + 1 = 2) and (R.b > 0 or c >= 0) +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT), Map 4 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_18] + Select Operator [SEL_17] (rows=16 width=93) + Output:["_col0","_col1","_col2"] + Join Operator [JOIN_16] (rows=16 width=93) + Output:["_col0","_col1","_col2"],condition map:[{"":"{\"type\":\"Left Semi\",\"left\":0,\"right\":1}"},{"":"{\"type\":\"Left Semi\",\"left\":0,\"right\":2}"}],keys:{"0":"_col0","1":"_col0","2":"_col0"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_13] + PartitionCols:_col0 + Select Operator [SEL_2] (rows=9 width=93) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_19] (rows=9 width=93) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + TableScan [TS_0] (rows=20 width=88) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_14] + PartitionCols:_col0 + Group By Operator [GBY_10] (rows=3 width=85) + Output:["_col0"],keys:_col0 + Select Operator [SEL_5] (rows=9 width=85) + Output:["_col0"] + Filter Operator [FIL_20] (rows=9 width=93) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + TableScan [TS_3] (rows=20 width=88) + default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + <-Map 4 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_15] + PartitionCols:_col0 + Group By Operator [GBY_12] (rows=6 width=85) + Output:["_col0"],keys:_col0 + Select Operator [SEL_8] (rows=18 width=80) + Output:["_col0"] + Filter Operator [FIL_21] (rows=18 width=80) + predicate:key is not null + TableScan [TS_6] (rows=20 width=80) + default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + +PREHOOK: query: explain select a, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by a+b desc, c asc) cbo_t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by q+r/10 desc, p) cbo_t2 on cbo_t1.a=p left semi join cbo_t3 on cbo_t1.a=key where (b + 1 >= 0) and (b > 0 or a >= 0) group by a, c having a > 0 and (a >=1 or c >= 1) and (a + c) >= 0 order by c, a +PREHOOK: type: QUERY +POSTHOOK: query: explain select a, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by a+b desc, c asc) cbo_t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by q+r/10 desc, p) cbo_t2 on cbo_t1.a=p left semi join cbo_t3 on cbo_t1.a=key where (b + 1 >= 0) and (b > 0 or a >= 0) group by a, c having a > 0 and (a >=1 or c >= 1) and (a + c) >= 0 order by c, a +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Reducer 2 (SORT) +Reducer 4 <- Map 10 (PARTITION-LEVEL SORT), Reducer 3 (PARTITION-LEVEL SORT), Reducer 9 (PARTITION-LEVEL SORT) +Reducer 5 <- Reducer 4 (GROUP) +Reducer 6 <- Reducer 5 (SORT) +Reducer 8 <- Map 7 (GROUP) +Reducer 9 <- Reducer 8 (SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 6 + File Output Operator [FS_41] + Select Operator [SEL_40] (rows=1 width=101) + Output:["_col0","_col1","_col2"] + <-Reducer 5 [SORT] + SORT [RS_39] + Select Operator [SEL_38] (rows=1 width=101) + Output:["_col0","_col1","_col2"] + Group By Operator [GBY_37] (rows=1 width=101) + Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 + <-Reducer 4 [GROUP] + GROUP [RS_36] + PartitionCols:_col0, _col1 + Group By Operator [GBY_35] (rows=1 width=101) + Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col1, _col0 + Join Operator [JOIN_33] (rows=1 width=93) + Output:["_col0","_col1"],condition map:[{"":"{\"type\":\"Left Semi\",\"left\":0,\"right\":1}"},{"":"{\"type\":\"Left Semi\",\"left\":0,\"right\":2}"}],keys:{"0":"_col0","1":"_col0","2":"_col0"} + <-Map 10 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_32] + PartitionCols:_col0 + Group By Operator [GBY_29] (rows=3 width=85) + Output:["_col0"],keys:_col0 + Select Operator [SEL_25] (rows=6 width=85) + Output:["_col0"] + Filter Operator [FIL_44] (rows=6 width=85) + predicate:(UDFToDouble(key) > 0.0) + TableScan [TS_23] (rows=20 width=80) + default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + <-Reducer 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_30] + PartitionCols:_col0 + Select Operator [SEL_10] (rows=1 width=93) + Output:["_col0","_col1"] + <-Reducer 2 [SORT] + SORT [RS_9] + Select Operator [SEL_8] (rows=1 width=101) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_7] (rows=1 width=101) + predicate:(((UDFToDouble(_col2) >= 1.0) or (_col3 >= 1)) and ((UDFToDouble(_col2) + UDFToDouble(_col3)) >= 0.0)) + Select Operator [SEL_6] (rows=1 width=101) + Output:["_col1","_col2","_col3"] + Group By Operator [GBY_5] (rows=1 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Map 1 [GROUP] + GROUP [RS_4] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_3] (rows=1 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float + Filter Operator [FIL_42] (rows=1 width=93) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and (((c_int + 1) + 1) >= 0) and (((c_int + 1) > 0) or (UDFToDouble(key) >= 0.0)) and (UDFToDouble(key) > 0.0)) + TableScan [TS_0] (rows=20 width=88) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + <-Reducer 9 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_31] + PartitionCols:_col0 + Group By Operator [GBY_27] (rows=1 width=85) + Output:["_col0"],keys:_col0 + Select Operator [SEL_21] (rows=1 width=85) + Output:["_col0"] + <-Reducer 8 [SORT] + SORT [RS_20] + Select Operator [SEL_18] (rows=1 width=93) + Output:["_col0","_col1"] + Group By Operator [GBY_17] (rows=1 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Map 7 [GROUP] + GROUP [RS_16] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_15] (rows=1 width=101) + Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float + Filter Operator [FIL_43] (rows=1 width=93) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and (UDFToDouble(key) > 0.0)) + TableScan [TS_12] (rows=20 width=88) + default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + +PREHOOK: query: explain select cbo_t1.key as x, c_int as c_int, (((c_int+c_float)*10)+5) as y from cbo_t1 +PREHOOK: type: QUERY +POSTHOOK: query: explain select cbo_t1.key as x, c_int as c_int, (((c_int+c_float)*10)+5) as y from cbo_t1 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Stage-0 + Fetch Operator + limit:-1 + Select Operator [SEL_1] + Output:["_col0","_col1","_col2"] + TableScan [TS_0] + Output:["key","c_int","c_float"] + +PREHOOK: query: explain select null from cbo_t1 +PREHOOK: type: QUERY +POSTHOOK: query: explain select null from cbo_t1 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Stage-0 + Fetch Operator + limit:-1 + Select Operator [SEL_1] + Output:["_col0"] + TableScan [TS_0] + +PREHOOK: query: explain select key from cbo_t1 where c_int = -6 or c_int = +6 +PREHOOK: type: QUERY +POSTHOOK: query: explain select key from cbo_t1 where c_int = -6 or c_int = +6 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Stage-0 + Fetch Operator + limit:-1 + Select Operator [SEL_2] + Output:["_col0"] + Filter Operator [FIL_4] + predicate:((c_int = -6) or (c_int = 6)) + TableScan [TS_0] + Output:["key","c_int"] + +PREHOOK: query: explain select count(cbo_t1.dt) from cbo_t1 join cbo_t2 on cbo_t1.dt = cbo_t2.dt where cbo_t1.dt = '2014' +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(cbo_t1.dt) from cbo_t1 join cbo_t2 on cbo_t1.dt = cbo_t2.dt where cbo_t1.dt = '2014' +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 4 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_15] + Group By Operator [GBY_13] (rows=1 width=8) + Output:["_col0"],aggregations:["count(VALUE._col0)"] + <-Reducer 2 [GROUP] + GROUP [RS_12] + Group By Operator [GBY_11] (rows=1 width=8) + Output:["_col0"],aggregations:["count('2014')"] + Join Operator [JOIN_8] (rows=400 width=8) + condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_6] + Select Operator [SEL_2] (rows=20 width=88) + TableScan [TS_0] (rows=20 width=21) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE + <-Map 4 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_7] + Select Operator [SEL_5] (rows=20 width=88) + TableScan [TS_3] (rows=20 width=21) + default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE + +PREHOOK: query: explain select * +from src_cbo b +where not exists + (select distinct a.key + from src_cbo a + where b.value = a.value and a.value > 'val_2' + ) +PREHOOK: type: QUERY +POSTHOOK: query: explain select * +from src_cbo b +where not exists + (select distinct a.key + from src_cbo a + where b.value = a.value and a.value > 'val_2' + ) +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Reducer 4 (PARTITION-LEVEL SORT) +Reducer 4 <- Map 3 (GROUP PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_19] + Select Operator [SEL_18] (rows=1 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_17] (rows=1 width=182) + predicate:_col3 is null + Join Operator [JOIN_16] (rows=500 width=182) + Output:["_col0","_col1","_col3"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"_col1","1":"_col0"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_14] + PartitionCols:_col1 + Select Operator [SEL_1] (rows=500 width=178) + Output:["_col0","_col1"] + TableScan [TS_0] (rows=500 width=178) + default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Reducer 4 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_15] + PartitionCols:_col0 + Select Operator [SEL_13] (rows=36 width=95) + Output:["_col0","_col1"] + Group By Operator [GBY_12] (rows=36 width=91) + Output:["_col0"],keys:_col1 + Select Operator [SEL_8] (rows=41 width=178) + Output:["_col1"] + Group By Operator [GBY_7] (rows=41 width=178) + Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 + <-Map 3 [GROUP PARTITION-LEVEL SORT] + GROUP PARTITION-LEVEL SORT [RS_6] + PartitionCols:_col0 + Group By Operator [GBY_5] (rows=41 width=178) + Output:["_col0","_col1"],keys:value, key + Filter Operator [FIL_20] (rows=83 width=178) + predicate:((value = value) and (value > 'val_2')) + TableScan [TS_2] (rows=500 width=178) + default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + +PREHOOK: query: explain select * +from src_cbo b +group by key, value +having not exists + (select a.key + from src_cbo a + where b.value = a.value and a.key = b.key and a.value > 'val_12' + ) +PREHOOK: type: QUERY +POSTHOOK: query: explain select * +from src_cbo b +group by key, value +having not exists + (select a.key + from src_cbo a + where b.value = a.value and a.key = b.key and a.value > 'val_12' + ) +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT), Reducer 5 (PARTITION-LEVEL SORT) +Reducer 5 <- Map 4 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_18] + Select Operator [SEL_17] (rows=1 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_16] (rows=1 width=182) + predicate:_col4 is null + Join Operator [JOIN_15] (rows=250 width=182) + Output:["_col0","_col1","_col4"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0, _col1","1":"_col0, _col1"} + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_13] + PartitionCols:_col0, _col1 + Group By Operator [GBY_4] (rows=250 width=178) + Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 + <-Map 1 [GROUP] + GROUP [RS_3] + PartitionCols:_col0, _col1 + Group By Operator [GBY_2] (rows=250 width=178) + Output:["_col0","_col1"],keys:key, value + Select Operator [SEL_1] (rows=500 width=178) + Output:["key","value"] + TableScan [TS_0] (rows=500 width=178) + default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Reducer 5 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_14] + PartitionCols:_col0, _col1 + Select Operator [SEL_12] (rows=20 width=182) + Output:["_col0","_col1","_col2"] + Group By Operator [GBY_11] (rows=20 width=178) + Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 + <-Map 4 [GROUP] + GROUP [RS_10] + PartitionCols:_col0, _col1 + Group By Operator [GBY_9] (rows=20 width=178) + Output:["_col0","_col1"],keys:key, value + Filter Operator [FIL_19] (rows=41 width=178) + predicate:((value = value) and (key = key) and (value > 'val_12')) + TableScan [TS_6] (rows=500 width=178) + default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + +PREHOOK: query: create view cv1 as +select * +from src_cbo b +where exists + (select a.key + from src_cbo a + where b.value = a.value and a.key = b.key and a.value > 'val_9') +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src_cbo +PREHOOK: Output: database:default +PREHOOK: Output: default@cv1 +POSTHOOK: query: create view cv1 as +select * +from src_cbo b +where exists + (select a.key + from src_cbo a + where b.value = a.value and a.key = b.key and a.value > 'val_9') +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src_cbo +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cv1 +POSTHOOK: Lineage: cv1.key SIMPLE [(src_cbo)b.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cv1.value SIMPLE [(src_cbo)b.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: explain select * from cv1 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from cv1 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_11] + Join Operator [JOIN_9] (rows=1 width=178) + Output:["_col0","_col1"],condition map:[{"":"{\"type\":\"Left Semi\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0, _col1","1":"_col0, _col1"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_7] + PartitionCols:_col0, _col1 + Select Operator [SEL_1] (rows=500 width=178) + Output:["_col0","_col1"] + TableScan [TS_0] (rows=500 width=178) + default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"],properties:{"insideView":"TRUE"} + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_8] + PartitionCols:_col0, _col1 + Group By Operator [GBY_6] (rows=20 width=178) + Output:["_col0","_col1"],keys:_col0, _col1 + Select Operator [SEL_4] (rows=41 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_12] (rows=41 width=178) + predicate:((value = value) and (key = key) and (value > 'val_9')) + TableScan [TS_2] (rows=500 width=178) + default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + +PREHOOK: query: explain select * +from (select * + from src_cbo b + where exists + (select a.key + from src_cbo a + where b.value = a.value and a.key = b.key and a.value > 'val_9') + ) a +PREHOOK: type: QUERY +POSTHOOK: query: explain select * +from (select * + from src_cbo b + where exists + (select a.key + from src_cbo a + where b.value = a.value and a.key = b.key and a.value > 'val_9') + ) a +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_11] + Join Operator [JOIN_9] (rows=1 width=178) + Output:["_col0","_col1"],condition map:[{"":"{\"type\":\"Left Semi\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0, _col1","1":"_col0, _col1"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_7] + PartitionCols:_col0, _col1 + Select Operator [SEL_1] (rows=500 width=178) + Output:["_col0","_col1"] + TableScan [TS_0] (rows=500 width=178) + default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_8] + PartitionCols:_col0, _col1 + Group By Operator [GBY_6] (rows=20 width=178) + Output:["_col0","_col1"],keys:_col0, _col1 + Select Operator [SEL_4] (rows=41 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_12] (rows=41 width=178) + predicate:((value = value) and (key = key) and (value > 'val_9')) + TableScan [TS_2] (rows=500 width=178) + default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + +PREHOOK: query: explain select * +from src_cbo +where src_cbo.key in (select key from src_cbo s1 where s1.key > '9') +PREHOOK: type: QUERY +POSTHOOK: query: explain select * +from src_cbo +where src_cbo.key in (select key from src_cbo s1 where s1.key > '9') +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_12] + Join Operator [JOIN_10] (rows=166 width=178) + Output:["_col0","_col1"],condition map:[{"":"{\"type\":\"Left Semi\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_8] + PartitionCols:_col0 + Select Operator [SEL_2] (rows=166 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_13] (rows=166 width=178) + predicate:(key > '9') + TableScan [TS_0] (rows=500 width=178) + default@src_cbo,src_cbo,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_9] + PartitionCols:_col0 + Group By Operator [GBY_7] (rows=69 width=87) + Output:["_col0"],keys:_col0 + Select Operator [SEL_5] (rows=166 width=87) + Output:["_col0"] + Filter Operator [FIL_14] (rows=166 width=87) + predicate:(key > '9') + TableScan [TS_3] (rows=500 width=87) + default@src_cbo,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + +PREHOOK: query: explain select p.p_partkey, li.l_suppkey +from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey +where li.l_linenumber = 1 and + li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR' and l_linenumber = li.l_linenumber) +PREHOOK: type: QUERY +POSTHOOK: query: explain select p.p_partkey, li.l_suppkey +from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey +where li.l_linenumber = 1 and + li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR' and l_linenumber = li.l_linenumber) +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Map 5 (PARTITION-LEVEL SORT), Reducer 2 (PARTITION-LEVEL SORT) +Reducer 4 <- Map 6 (PARTITION-LEVEL SORT), Reducer 3 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 4 + File Output Operator [FS_22] + Select Operator [SEL_21] (rows=5 width=8) + Output:["_col0","_col1"] + Join Operator [JOIN_20] (rows=5 width=8) + Output:["_col0","_col3"],condition map:[{"":"{\"type\":\"Left Semi\",\"left\":0,\"right\":1}"}],keys:{"0":"_col1, _col4","1":"_col0, _col1"} + <-Map 6 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_19] + PartitionCols:_col0, _col1 + Group By Operator [GBY_17] (rows=2 width=8) + Output:["_col0","_col1"],keys:_col0, _col1 + Select Operator [SEL_12] (rows=7 width=8) + Output:["_col0","_col1"] + Filter Operator [FIL_25] (rows=7 width=96) + predicate:((l_shipmode = 'AIR') and (l_linenumber = l_linenumber)) + TableScan [TS_10] (rows=100 width=96) + default@lineitem,lineitem,Tbl:COMPLETE,Col:COMPLETE,Output:["l_orderkey","l_linenumber","l_shipmode"] + <-Reducer 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_18] + PartitionCols:_col1, _col4 + Join Operator [JOIN_15] (rows=5 width=16) + Output:["_col0","_col1","_col3","_col4"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col1"} + <-Map 5 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_14] + PartitionCols:_col1 + Select Operator [SEL_9] (rows=17 width=16) + Output:["_col0","_col1","_col2","_col3"] + Filter Operator [FIL_24] (rows=17 width=16) + predicate:((l_linenumber = 1) and l_partkey is not null) + TableScan [TS_7] (rows=100 width=16) + default@lineitem,li,Tbl:COMPLETE,Col:COMPLETE,Output:["l_orderkey","l_partkey","l_suppkey","l_linenumber"] + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_13] + PartitionCols:_col0 + Group By Operator [GBY_5] (rows=50 width=4) + Output:["_col0"],keys:KEY._col0 + <-Map 1 [GROUP] + GROUP [RS_4] + PartitionCols:_col0 + Group By Operator [GBY_3] (rows=50 width=4) + Output:["_col0"],keys:l_partkey + Filter Operator [FIL_23] (rows=100 width=4) + predicate:l_partkey is not null + TableScan [TS_0] (rows=100 width=4) + default@lineitem,lineitem,Tbl:COMPLETE,Col:COMPLETE,Output:["l_partkey"] + +PREHOOK: query: explain select key, value, count(*) +from src_cbo b +where b.key in (select key from src_cbo where src_cbo.key > '8') +group by key, value +having count(*) in (select count(*) from src_cbo s1 where s1.key > '9' group by s1.key ) +PREHOOK: type: QUERY +POSTHOOK: query: explain select key, value, count(*) +from src_cbo b +where b.key in (select key from src_cbo where src_cbo.key > '8') +group by key, value +having count(*) in (select count(*) from src_cbo s1 where s1.key > '9' group by s1.key ) +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Reducer 6 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (GROUP) +Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT), Reducer 8 (PARTITION-LEVEL SORT) +Reducer 6 <- Map 5 (GROUP) +Reducer 8 <- Map 7 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 4 + File Output Operator [FS_33] + Join Operator [JOIN_31] (rows=34 width=186) + Output:["_col0","_col1","_col2"],condition map:[{"":"{\"type\":\"Left Semi\",\"left\":0,\"right\":1}"}],keys:{"0":"_col2","1":"_col0"} + <-Reducer 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_29] + PartitionCols:_col2 + Filter Operator [FIL_17] (rows=83 width=186) + predicate:_col2 is not null + Group By Operator [GBY_16] (rows=83 width=186) + Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 + <-Reducer 2 [GROUP] + GROUP [RS_15] + PartitionCols:_col0, _col1 + Group By Operator [GBY_14] (rows=83 width=186) + Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col0, _col1 + Join Operator [JOIN_12] (rows=166 width=178) + Output:["_col0","_col1"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_10] + PartitionCols:_col0 + Select Operator [SEL_2] (rows=166 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_34] (rows=166 width=178) + predicate:(key > '8') + TableScan [TS_0] (rows=500 width=178) + default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Reducer 6 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_11] + PartitionCols:_col0 + Group By Operator [GBY_8] (rows=69 width=87) + Output:["_col0"],keys:KEY._col0 + <-Map 5 [GROUP] + GROUP [RS_7] + PartitionCols:_col0 + Group By Operator [GBY_6] (rows=69 width=87) + Output:["_col0"],keys:key + Filter Operator [FIL_35] (rows=166 width=87) + predicate:(key > '8') + TableScan [TS_3] (rows=500 width=87) + default@src_cbo,src_cbo,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + <-Reducer 8 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_30] + PartitionCols:_col0 + Group By Operator [GBY_28] (rows=34 width=8) + Output:["_col0"],keys:_col0 + Select Operator [SEL_26] (rows=69 width=8) + Output:["_col0"] + Filter Operator [FIL_25] (rows=69 width=8) + predicate:_col1 is not null + Select Operator [SEL_37] (rows=69 width=8) + Output:["_col1"] + Group By Operator [GBY_24] (rows=69 width=95) + Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 + <-Map 7 [GROUP] + GROUP [RS_23] + PartitionCols:_col0 + Group By Operator [GBY_22] (rows=69 width=95) + Output:["_col0","_col1"],aggregations:["count()"],keys:key + Filter Operator [FIL_36] (rows=166 width=87) + predicate:(key > '9') + TableScan [TS_19] (rows=500 width=87) + default@src_cbo,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + +PREHOOK: query: explain select p_mfgr, p_name, avg(p_size) +from part +group by p_mfgr, p_name +having p_name in + (select first_value(p_name) over(partition by p_mfgr order by p_size) from part) +PREHOOK: type: QUERY +POSTHOOK: query: explain select p_mfgr, p_name, avg(p_size) +from part +group by p_mfgr, p_name +having p_name in + (select first_value(p_name) over(partition by p_mfgr order by p_size) from part) +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT), Reducer 5 (PARTITION-LEVEL SORT) +Reducer 5 <- Map 4 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_21] + Join Operator [JOIN_19] (rows=6 width=227) + Output:["_col0","_col1","_col2"],condition map:[{"":"{\"type\":\"Left Semi\",\"left\":0,\"right\":1}"}],keys:{"0":"_col1","1":"_col0"} + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_17] + PartitionCols:_col1 + Select Operator [SEL_6] (rows=13 width=227) + Output:["_col0","_col1","_col2"] + Group By Operator [GBY_5] (rows=13 width=227) + Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0, KEY._col1 + <-Map 1 [GROUP] + GROUP [RS_4] + PartitionCols:_col0, _col1 + Group By Operator [GBY_3] (rows=13 width=295) + Output:["_col0","_col1","_col2"],aggregations:["avg(p_size)"],keys:p_name, p_mfgr + Filter Operator [FIL_22] (rows=26 width=223) + predicate:p_name is not null + TableScan [TS_0] (rows=26 width=223) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] + <-Reducer 5 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_18] + PartitionCols:_col0 + Group By Operator [GBY_16] (rows=13 width=184) + Output:["_col0"],keys:_col0 + Select Operator [SEL_11] (rows=26 width=184) + Output:["_col0"] + Filter Operator [FIL_23] (rows=26 width=491) + predicate:first_value_window_0 is not null + PTF Operator [PTF_10] (rows=26 width=491) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col5 ASC NULLS FIRST","partition by:":"_col2"}] + Select Operator [SEL_9] (rows=26 width=491) + Output:["_col1","_col2","_col5"] + <-Map 4 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_8] + PartitionCols:p_mfgr + TableScan [TS_7] (rows=26 width=223) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] + +PREHOOK: query: explain select * +from src_cbo +where src_cbo.key not in + ( select key from src_cbo s1 + where s1.key > '2' + ) order by key +PREHOOK: type: QUERY +POSTHOOK: query: explain select * +from src_cbo +where src_cbo.key not in + ( select key from src_cbo s1 + where s1.key > '2' + ) order by key +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Reducer 6 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT), Reducer 8 (PARTITION-LEVEL SORT) +Reducer 4 <- Reducer 3 (SORT) +Reducer 6 <- Map 5 (GROUP) +Reducer 8 <- Map 7 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 4 + File Output Operator [FS_26] + Select Operator [SEL_25] (rows=500 width=178) + Output:["_col0","_col1"] + <-Reducer 3 [SORT] + SORT [RS_24] + Select Operator [SEL_23] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_22] (rows=500 width=198) + predicate:((_col2 = 0) or (_col5 is null and _col0 is not null and (_col3 >= _col2))) + Join Operator [JOIN_21] (rows=500 width=198) + Output:["_col0","_col1","_col2","_col3","_col5"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_19] + PartitionCols:_col0 + Join Operator [JOIN_18] (rows=500 width=194) + Output:["_col0","_col1","_col2","_col3"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_16] + Select Operator [SEL_1] (rows=500 width=178) + Output:["_col0","_col1"] + TableScan [TS_0] (rows=500 width=178) + default@src_cbo,src_cbo,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Reducer 6 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_17] + Group By Operator [GBY_7] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"] + <-Map 5 [GROUP] + GROUP [RS_6] + Group By Operator [GBY_5] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["count()","count(key)"] + Filter Operator [FIL_27] (rows=166 width=87) + predicate:(key > '2') + TableScan [TS_2] (rows=500 width=87) + default@src_cbo,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + <-Reducer 8 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_20] + PartitionCols:_col0 + Select Operator [SEL_15] (rows=69 width=91) + Output:["_col0","_col1"] + Group By Operator [GBY_14] (rows=69 width=87) + Output:["_col0"],keys:KEY._col0 + <-Map 7 [GROUP] + GROUP [RS_13] + PartitionCols:_col0 + Group By Operator [GBY_12] (rows=69 width=87) + Output:["_col0"],keys:key + Filter Operator [FIL_28] (rows=166 width=87) + predicate:(key > '2') + TableScan [TS_9] (rows=500 width=87) + default@src_cbo,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + +PREHOOK: query: explain select p_mfgr, b.p_name, p_size +from part b +where b.p_name not in + (select p_name + from (select p_mfgr, p_name, p_size as r from part) a + where r < 10 and b.p_mfgr = a.p_mfgr + ) +PREHOOK: type: QUERY +POSTHOOK: query: explain select p_mfgr, b.p_name, p_size +from part b +where b.p_name not in + (select p_name + from (select p_mfgr, p_name, p_size as r from part) a + where r < 10 and b.p_mfgr = a.p_mfgr + ) +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Reducer 5 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT), Reducer 7 (PARTITION-LEVEL SORT) +Reducer 5 <- Map 4 (GROUP) +Reducer 7 <- Map 6 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_26] + Select Operator [SEL_25] (rows=13 width=223) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_24] (rows=13 width=243) + predicate:(not CASE WHEN ((_col4 = 0)) THEN (false) WHEN (_col4 is null) THEN (false) WHEN (_col8 is not null) THEN (true) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (true) ELSE (false) END) + Join Operator [JOIN_23] (rows=26 width=243) + Output:["_col0","_col1","_col2","_col4","_col5","_col8"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0, _col1","1":"_col0, _col1"} + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_21] + PartitionCols:_col0, _col1 + Join Operator [JOIN_20] (rows=26 width=239) + Output:["_col0","_col1","_col2","_col4","_col5"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"_col1","1":"_col0"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_18] + PartitionCols:_col1 + Select Operator [SEL_1] (rows=26 width=223) + Output:["_col0","_col1","_col2"] + TableScan [TS_0] (rows=26 width=223) + default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] + <-Reducer 5 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_19] + PartitionCols:_col0 + Group By Operator [GBY_7] (rows=1 width=114) + Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 + <-Map 4 [GROUP] + GROUP [RS_6] + PartitionCols:_col0 + Group By Operator [GBY_5] (rows=1 width=114) + Output:["_col0","_col1","_col2"],aggregations:["count()","count(p_name)"],keys:p_mfgr + Select Operator [SEL_4] (rows=4 width=223) + Output:["p_name","p_mfgr"] + Filter Operator [FIL_27] (rows=4 width=223) + predicate:((p_size < 10) and (p_mfgr = p_mfgr)) + TableScan [TS_2] (rows=26 width=223) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] + <-Reducer 7 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_22] + PartitionCols:_col0, _col1 + Select Operator [SEL_17] (rows=1 width=223) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_16] (rows=1 width=219) + predicate:(_col0 = _col0) + Group By Operator [GBY_14] (rows=2 width=219) + Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 + <-Map 6 [GROUP] + GROUP [RS_13] + PartitionCols:_col0, _col1 + Group By Operator [GBY_12] (rows=2 width=219) + Output:["_col0","_col1"],keys:p_name, p_mfgr + Select Operator [SEL_11] (rows=4 width=223) + Output:["p_name","p_mfgr"] + Filter Operator [FIL_28] (rows=4 width=223) + predicate:((p_size < 10) and (p_mfgr = p_mfgr)) + TableScan [TS_9] (rows=26 width=223) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] + +PREHOOK: query: explain select p_name, p_size +from +part where part.p_size not in + (select avg(p_size) + from (select p_size from part) a + where p_size < 10 + ) order by p_name +PREHOOK: type: QUERY +POSTHOOK: query: explain select p_name, p_size +from +part where part.p_size not in + (select avg(p_size) + from (select p_size from part) a + where p_size < 10 + ) order by p_name +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Reducer 6 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT), Reducer 9 (PARTITION-LEVEL SORT) +Reducer 4 <- Reducer 3 (SORT) +Reducer 6 <- Map 5 (GROUP) +Reducer 8 <- Map 5 (GROUP) +Reducer 9 <- Reducer 8 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 4 + File Output Operator [FS_36] + Select Operator [SEL_35] (rows=26 width=125) + Output:["_col0","_col1"] + <-Reducer 3 [SORT] + SORT [RS_34] + Select Operator [SEL_33] (rows=26 width=125) + Output:["_col0","_col1"] + Filter Operator [FIL_32] (rows=26 width=145) + predicate:((_col2 = 0) or (_col5 is null and _col1 is not null and (_col3 >= _col2))) + Join Operator [JOIN_31] (rows=26 width=145) + Output:["_col0","_col1","_col2","_col3","_col5"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"UDFToDouble(_col1)","1":"_col0"} + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_29] + PartitionCols:UDFToDouble(_col1) + Join Operator [JOIN_28] (rows=26 width=141) + Output:["_col0","_col1","_col2","_col3"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_26] + Select Operator [SEL_1] (rows=26 width=125) + Output:["_col0","_col1"] + TableScan [TS_0] (rows=26 width=125) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_size"] + <-Reducer 6 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_27] + Group By Operator [GBY_12] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["count()","count(_col0)"] + Group By Operator [GBY_7] (rows=1 width=8) + Output:["_col0"],aggregations:["avg(VALUE._col0)"] + <-Map 5 [GROUP] + GROUP [RS_6] + Group By Operator [GBY_5] (rows=1 width=76) + Output:["_col0"],aggregations:["avg(p_size)"] + Filter Operator [FIL_37] (rows=8 width=4) + predicate:(p_size < 10) + TableScan [TS_2] (rows=26 width=4) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_size"] + <-Reducer 9 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_30] + PartitionCols:_col0 + Select Operator [SEL_25] (rows=1 width=12) + Output:["_col0","_col1"] + Group By Operator [GBY_24] (rows=1 width=8) + Output:["_col0"],keys:KEY._col0 + <-Reducer 8 [GROUP] + GROUP [RS_23] + PartitionCols:_col0 + Group By Operator [GBY_22] (rows=1 width=8) + Output:["_col0"],keys:_col0 + Group By Operator [GBY_19] (rows=1 width=8) + Output:["_col0"],aggregations:["avg(VALUE._col0)"] + <- Please refer to the previous Map 5 [GROUP] + +PREHOOK: query: explain select b.p_mfgr, min(p_retailprice) +from part b +group by b.p_mfgr +having b.p_mfgr not in + (select p_mfgr + from (select p_mfgr, min(p_retailprice) l, max(p_retailprice) r, avg(p_retailprice) a from part group by p_mfgr) a + where min(p_retailprice) = l and r - l > 600 + ) + order by b.p_mfgr +PREHOOK: type: QUERY +POSTHOOK: query: explain select b.p_mfgr, min(p_retailprice) +from part b +group by b.p_mfgr +having b.p_mfgr not in + (select p_mfgr + from (select p_mfgr, min(p_retailprice) l, max(p_retailprice) r, avg(p_retailprice) a from part group by p_mfgr) a + where min(p_retailprice) = l and r - l > 600 + ) + order by b.p_mfgr +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 10 <- Map 6 (GROUP) +Reducer 11 <- Reducer 10 (GROUP) +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT), Reducer 8 (PARTITION-LEVEL SORT) +Reducer 4 <- Reducer 11 (PARTITION-LEVEL SORT), Reducer 3 (PARTITION-LEVEL SORT) +Reducer 5 <- Reducer 4 (SORT) +Reducer 7 <- Map 6 (GROUP) +Reducer 8 <- Reducer 7 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 5 + File Output Operator [FS_42] + Select Operator [SEL_41] (rows=3 width=106) + Output:["_col0","_col1"] + <-Reducer 4 [SORT] + SORT [RS_40] + Select Operator [SEL_39] (rows=3 width=106) + Output:["_col0","_col1"] + Filter Operator [FIL_38] (rows=3 width=126) + predicate:(not CASE WHEN ((_col3 = 0)) THEN (false) WHEN (_col3 is null) THEN (false) WHEN (_col7 is not null) THEN (true) WHEN (_col0 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (true) ELSE (false) END) + Join Operator [JOIN_37] (rows=5 width=126) + Output:["_col0","_col1","_col3","_col4","_col7"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0, _col1","1":"_col0, _col1"} + <-Reducer 11 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_36] + PartitionCols:_col0, _col1 + Select Operator [SEL_31] (rows=1 width=110) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_30] (rows=1 width=106) + predicate:(_col0 = _col0) + Group By Operator [GBY_28] (rows=1 width=106) + Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 + <-Reducer 10 [GROUP] + GROUP [RS_27] + PartitionCols:_col0, _col1 + Group By Operator [GBY_26] (rows=1 width=106) + Output:["_col0","_col1"],keys:_col0, _col1 + Select Operator [SEL_24] (rows=1 width=114) + Output:["_col0","_col1"] + Filter Operator [FIL_23] (rows=1 width=114) + predicate:((_col1 = _col1) and ((_col2 - _col1) > 600.0)) + Group By Operator [GBY_22] (rows=5 width=114) + Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0 + <-Map 6 [GROUP] + GROUP [RS_9] + PartitionCols:_col0 + Group By Operator [GBY_8] (rows=5 width=114) + Output:["_col0","_col1","_col2"],aggregations:["min(p_retailprice)","max(p_retailprice)"],keys:p_mfgr + Select Operator [SEL_7] (rows=26 width=106) + Output:["p_mfgr","p_retailprice"] + TableScan [TS_6] (rows=26 width=106) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] + <-Reducer 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_35] + PartitionCols:_col0, _col1 + Join Operator [JOIN_34] (rows=5 width=122) + Output:["_col0","_col1","_col3","_col4"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"_col1","1":"_col0"} + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_32] + PartitionCols:_col1 + Group By Operator [GBY_4] (rows=5 width=106) + Output:["_col0","_col1"],aggregations:["min(VALUE._col0)"],keys:KEY._col0 + <-Map 1 [GROUP] + GROUP [RS_3] + PartitionCols:_col0 + Group By Operator [GBY_2] (rows=5 width=106) + Output:["_col0","_col1"],aggregations:["min(p_retailprice)"],keys:p_mfgr + Select Operator [SEL_1] (rows=26 width=106) + Output:["p_mfgr","p_retailprice"] + TableScan [TS_0] (rows=26 width=106) + default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] + <-Reducer 8 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_33] + PartitionCols:_col0 + Group By Operator [GBY_16] (rows=1 width=24) + Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 + <-Reducer 7 [GROUP] + GROUP [RS_15] + PartitionCols:_col0 + Group By Operator [GBY_14] (rows=1 width=24) + Output:["_col0","_col1","_col2"],aggregations:["count()","count(_col0)"],keys:_col1 + Select Operator [SEL_12] (rows=1 width=114) + Output:["_col0","_col1"] + Filter Operator [FIL_11] (rows=1 width=114) + predicate:((_col1 = _col1) and ((_col2 - _col1) > 600.0)) + Group By Operator [GBY_10] (rows=5 width=114) + Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0 + <- Please refer to the previous Map 6 [GROUP] + +PREHOOK: query: explain select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over(), rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(), lag(c_float, 2, c_float) over() from cbo_t1 +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over(), rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(), lag(c_float, 2, c_float) over() from cbo_t1 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_6] + Select Operator [SEL_4] (rows=20 width=52) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] + PTF Operator [PTF_3] (rows=20 width=459) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"0 ASC NULLS FIRST","partition by:":"0"}] + Select Operator [SEL_2] (rows=20 width=459) + Output:["_col2","_col3"] + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_1] + PartitionCols:0 + TableScan [TS_0] (rows=20 width=7) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["c_float","c_int"] + +PREHOOK: query: explain select * from (select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over(), rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(), lag(c_float, 2, c_float) over() from cbo_t1) cbo_t1 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from (select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over(), rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(), lag(c_float, 2, c_float) over() from cbo_t1) cbo_t1 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_6] + Select Operator [SEL_4] (rows=20 width=52) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] + PTF Operator [PTF_3] (rows=20 width=459) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"0 ASC NULLS FIRST","partition by:":"0"}] + Select Operator [SEL_2] (rows=20 width=459) + Output:["_col2","_col3"] + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_1] + PartitionCols:0 + TableScan [TS_0] (rows=20 width=7) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["c_float","c_int"] + +PREHOOK: query: explain select i, a, h, b, c, d, e, f, g, a as x, a +1 as y from (select max(c_int) over (partition by key order by value range UNBOUNDED PRECEDING) a, min(c_int) over (partition by key order by value range current row) b, count(c_int) over(partition by key order by value range 1 PRECEDING) c, avg(value) over (partition by key order by value range between unbounded preceding and unbounded following) d, sum(value) over (partition by key order by value range between unbounded preceding and current row) e, avg(c_float) over (partition by key order by value range between 1 preceding and unbounded following) f, sum(c_float) over (partition by key order by value range between 1 preceding and current row) g, max(c_float) over (partition by key order by value range between 1 preceding and unbounded following) h, min(c_float) over (partition by key order by value range between 1 preceding and 1 following) i from cbo_t1) cbo_t1 +PREHOOK: type: QUERY +POSTHOOK: query: explain select i, a, h, b, c, d, e, f, g, a as x, a +1 as y from (select max(c_int) over (partition by key order by value range UNBOUNDED PRECEDING) a, min(c_int) over (partition by key order by value range current row) b, count(c_int) over(partition by key order by value range 1 PRECEDING) c, avg(value) over (partition by key order by value range between unbounded preceding and unbounded following) d, sum(value) over (partition by key order by value range between unbounded preceding and current row) e, avg(c_float) over (partition by key order by value range between 1 preceding and unbounded following) f, sum(c_float) over (partition by key order by value range between 1 preceding and current row) g, max(c_float) over (partition by key order by value range between 1 preceding and unbounded following) h, min(c_float) over (partition by key order by value range between 1 preceding and 1 following) i from cbo_t1) cbo_t1 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_6] + Select Operator [SEL_4] (rows=20 width=64) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10"] + PTF Operator [PTF_3] (rows=20 width=621) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col0"}] + Select Operator [SEL_2] (rows=20 width=621) + Output:["_col0","_col1","_col2","_col3"] + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_1] + PartitionCols:key + TableScan [TS_0] (rows=20 width=169) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["c_float","c_int","key","value"] + +PREHOOK: query: explain select *, rank() over(partition by key order by value) as rr from src1 +PREHOOK: type: QUERY +POSTHOOK: query: explain select *, rank() over(partition by key order by value) as rr from src1 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_6] + Select Operator [SEL_4] (rows=25 width=179) + Output:["_col0","_col1","_col2"] + PTF Operator [PTF_3] (rows=25 width=443) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col0"}] + Select Operator [SEL_2] (rows=25 width=443) + Output:["_col0","_col1"] + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_1] + PartitionCols:key + TableScan [TS_0] (rows=25 width=175) + default@src1,src1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + +PREHOOK: query: explain +select SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)) +FROM (select x.key AS key, count(1) AS cnt + FROM src1 x JOIN src y ON (x.key = y.key) + GROUP BY x.key) tmp +PREHOOK: type: QUERY +POSTHOOK: query: explain +select SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)) +FROM (select x.key AS key, count(1) AS cnt + FROM src1 x JOIN src y ON (x.key = y.key) + GROUP BY x.key) tmp +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 5 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (GROUP) +Reducer 4 <- Reducer 3 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 4 + File Output Operator [FS_20] + Group By Operator [GBY_18] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] + <-Reducer 3 [GROUP] + GROUP [RS_17] + Group By Operator [GBY_16] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] + Select Operator [SEL_14] (rows=14 width=94) + Output:["_col0","_col1"] + Group By Operator [GBY_13] (rows=14 width=94) + Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 + <-Reducer 2 [GROUP] + GROUP [RS_12] + PartitionCols:_col0 + Group By Operator [GBY_11] (rows=14 width=94) + Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 + Join Operator [JOIN_8] (rows=60 width=86) + Output:["_col0"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_6] + PartitionCols:_col0 + Select Operator [SEL_2] (rows=25 width=86) + Output:["_col0"] + Filter Operator [FIL_21] (rows=25 width=86) + predicate:key is not null + TableScan [TS_0] (rows=25 width=86) + default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + <-Map 5 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_7] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=500 width=87) + Output:["_col0"] + Filter Operator [FIL_22] (rows=500 width=87) + predicate:key is not null + TableScan [TS_3] (rows=500 width=87) + default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + +PREHOOK: query: explain +select SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)) +FROM (select x.key AS key, count(1) AS cnt + FROM src1 x JOIN src y ON (x.key = y.key) + GROUP BY x.key) tmp +PREHOOK: type: QUERY +POSTHOOK: query: explain +select SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)) +FROM (select x.key AS key, count(1) AS cnt + FROM src1 x JOIN src y ON (x.key = y.key) + GROUP BY x.key) tmp +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 5 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (GROUP) +Reducer 4 <- Reducer 3 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 4 + File Output Operator [FS_20] + Group By Operator [GBY_18] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] + <-Reducer 3 [GROUP] + GROUP [RS_17] + Group By Operator [GBY_16] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] + Select Operator [SEL_14] (rows=14 width=94) + Output:["_col0","_col1"] + Group By Operator [GBY_13] (rows=14 width=94) + Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 + <-Reducer 2 [GROUP] + GROUP [RS_12] + PartitionCols:_col0 + Group By Operator [GBY_11] (rows=14 width=94) + Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 + Join Operator [JOIN_8] (rows=60 width=86) + Output:["_col0"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_6] + PartitionCols:_col0 + Select Operator [SEL_2] (rows=25 width=86) + Output:["_col0"] + Filter Operator [FIL_21] (rows=25 width=86) + predicate:key is not null + TableScan [TS_0] (rows=25 width=86) + default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + <-Map 5 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_7] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=500 width=87) + Output:["_col0"] + Filter Operator [FIL_22] (rows=500 width=87) + predicate:key is not null + TableScan [TS_3] (rows=500 width=87) + default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + +PREHOOK: query: explain +select SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)) +FROM (select x.key AS key, count(1) AS cnt + FROM src1 x JOIN src y ON (x.key = y.key) + GROUP BY x.key) tmp +PREHOOK: type: QUERY +POSTHOOK: query: explain +select SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)) +FROM (select x.key AS key, count(1) AS cnt + FROM src1 x JOIN src y ON (x.key = y.key) + GROUP BY x.key) tmp +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 3 <- Map 2 (GROUP) +Reducer 4 <- Reducer 3 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 4 + File Output Operator [FS_20] + Group By Operator [GBY_18] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] + <-Reducer 3 [GROUP] + GROUP [RS_17] + Group By Operator [GBY_16] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] + Select Operator [SEL_14] (rows=14 width=94) + Output:["_col0","_col1"] + Group By Operator [GBY_13] (rows=14 width=94) + Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 + <-Map 2 [GROUP] + GROUP [RS_12] + PartitionCols:_col0 + Group By Operator [GBY_11] (rows=14 width=94) + Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 + Map Join Operator [MAPJOIN_23] (rows=60 width=86) + Conds:SEL_5._col0=SEL_5._col0(Inner),Output:["_col0"] + <-Select Operator [SEL_5] (rows=500 width=87) + Output:["_col0"] + Filter Operator [FIL_22] (rows=500 width=87) + predicate:key is not null + TableScan [TS_3] (rows=500 width=87) + default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Map Reduce Local Work + Stage-2 + Map 1 + keys: [HASHTABLESINK_25] + 0_col0,1_col0 + Select Operator [SEL_2] (rows=25 width=86) + Output:["_col0"] + Filter Operator [FIL_21] (rows=25 width=86) + predicate:key is not null + TableScan [TS_0] (rows=25 width=86) + default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Map Reduce Local Work + +PREHOOK: query: explain +select SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)) +FROM (select x.key AS key, count(1) AS cnt + FROM src1 x LEFT SEMI JOIN src y ON (x.key = y.key) + GROUP BY x.key) tmp +PREHOOK: type: QUERY +POSTHOOK: query: explain +select SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)) +FROM (select x.key AS key, count(1) AS cnt + FROM src1 x LEFT SEMI JOIN src y ON (x.key = y.key) + GROUP BY x.key) tmp +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 5 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (GROUP) +Reducer 4 <- Reducer 3 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 4 + File Output Operator [FS_22] + Group By Operator [GBY_20] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] + <-Reducer 3 [GROUP] + GROUP [RS_19] + Group By Operator [GBY_18] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] + Select Operator [SEL_16] (rows=12 width=94) + Output:["_col0","_col1"] + Group By Operator [GBY_15] (rows=12 width=94) + Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 + <-Reducer 2 [GROUP] + GROUP [RS_14] + PartitionCols:_col0 + Group By Operator [GBY_13] (rows=12 width=94) + Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 + Join Operator [JOIN_10] (rows=25 width=86) + Output:["_col0"],condition map:[{"":"{\"type\":\"Left Semi\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_8] + PartitionCols:_col0 + Select Operator [SEL_2] (rows=25 width=86) + Output:["_col0"] + Filter Operator [FIL_23] (rows=25 width=86) + predicate:key is not null + TableScan [TS_0] (rows=25 width=86) + default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + <-Map 5 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_9] + PartitionCols:_col0 + Group By Operator [GBY_7] (rows=205 width=87) + Output:["_col0"],keys:_col0 + Select Operator [SEL_5] (rows=500 width=87) + Output:["_col0"] + Filter Operator [FIL_24] (rows=500 width=87) + predicate:key is not null + TableScan [TS_3] (rows=500 width=87) + default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + +PREHOOK: query: explain create table abcd (a int, b int, c int, d int) +PREHOOK: type: CREATETABLE +POSTHOOK: query: explain create table abcd (a int, b int, c int, d int) +POSTHOOK: type: CREATETABLE +Stage-0 + Create Table Operator: + name:default.abcd + +PREHOOK: query: create table abcd (a int, b int, c int, d int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@abcd +POSTHOOK: query: create table abcd (a int, b int, c int, d int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@abcd +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in4.txt' INTO TABLE abcd +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@abcd +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in4.txt' INTO TABLE abcd +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@abcd +PREHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a +PREHOOK: type: QUERY +POSTHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_6] + Group By Operator [GBY_4] (rows=2 width=19) + Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT KEY._col1:0._col0)","count(DISTINCT KEY._col1:1._col0)","sum(VALUE._col2)"],keys:KEY._col0 + <-Map 1 [GROUP PARTITION-LEVEL SORT] + GROUP PARTITION-LEVEL SORT [RS_3] + PartitionCols:_col0 + Group By Operator [GBY_2] (rows=4 width=19) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(DISTINCT b)","count(DISTINCT c)","sum(d)"],keys:a, b, c + Select Operator [SEL_1] (rows=4 width=19) + Output:["a","b","c","d"] + TableScan [TS_0] (rows=4 width=19) + default@abcd,abcd,Tbl:COMPLETE,Col:NONE,Output:["a","b","c","d"] + +PREHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a +PREHOOK: type: QUERY +POSTHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_5] + Group By Operator [GBY_3] (rows=2 width=19) + Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT KEY._col1:0._col0)","count(DISTINCT KEY._col1:1._col0)","sum(VALUE._col0)"],keys:KEY._col0 + <-Map 1 [GROUP PARTITION-LEVEL SORT] + GROUP PARTITION-LEVEL SORT [RS_2] + PartitionCols:a + Select Operator [SEL_1] (rows=4 width=19) + Output:["a","b","c","d"] + TableScan [TS_0] (rows=4 width=19) + default@abcd,abcd,Tbl:COMPLETE,Col:NONE,Output:["a","b","c","d"] + +PREHOOK: query: explain create table src_rc_merge_test(key int, value string) stored as rcfile +PREHOOK: type: CREATETABLE +POSTHOOK: query: explain create table src_rc_merge_test(key int, value string) stored as rcfile +POSTHOOK: type: CREATETABLE +Stage-0 + Create Table Operator: + name:default.src_rc_merge_test + +PREHOOK: query: create table src_rc_merge_test(key int, value string) stored as rcfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_rc_merge_test +POSTHOOK: query: create table src_rc_merge_test(key int, value string) stored as rcfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_rc_merge_test +PREHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' into table src_rc_merge_test +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@src_rc_merge_test +POSTHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' into table src_rc_merge_test +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@src_rc_merge_test +PREHOOK: query: explain create table tgt_rc_merge_test(key int, value string) stored as rcfile +PREHOOK: type: CREATETABLE +POSTHOOK: query: explain create table tgt_rc_merge_test(key int, value string) stored as rcfile +POSTHOOK: type: CREATETABLE +Stage-0 + Create Table Operator: + name:default.tgt_rc_merge_test + +PREHOOK: query: create table tgt_rc_merge_test(key int, value string) stored as rcfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tgt_rc_merge_test +POSTHOOK: query: create table tgt_rc_merge_test(key int, value string) stored as rcfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tgt_rc_merge_test +PREHOOK: query: insert into table tgt_rc_merge_test select * from src_rc_merge_test +PREHOOK: type: QUERY +PREHOOK: Input: default@src_rc_merge_test +PREHOOK: Output: default@tgt_rc_merge_test +POSTHOOK: query: insert into table tgt_rc_merge_test select * from src_rc_merge_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_rc_merge_test +POSTHOOK: Output: default@tgt_rc_merge_test +POSTHOOK: Lineage: tgt_rc_merge_test.key SIMPLE [(src_rc_merge_test)src_rc_merge_test.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: tgt_rc_merge_test.value SIMPLE [(src_rc_merge_test)src_rc_merge_test.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: show table extended like `tgt_rc_merge_test` +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like `tgt_rc_merge_test` +POSTHOOK: type: SHOW_TABLESTATUS +tableName:tgt_rc_merge_test +#### A masked pattern was here #### +inputformat:org.apache.hadoop.hive.ql.io.RCFileInputFormat +outputformat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat +columns:struct columns { i32 key, string value} +partitioned:false +partitionColumns: +totalNumberFiles:1 +totalFileSize:171 +maxFileSize:171 +minFileSize:171 +#### A masked pattern was here #### + +PREHOOK: query: explain select count(1) from tgt_rc_merge_test +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(1) from tgt_rc_merge_test +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_6] + Group By Operator [GBY_4] (rows=1 width=8) + Output:["_col0"],aggregations:["count(1)"] + <-Map 1 [GROUP] + GROUP [RS_3] + Select Operator [SEL_1] (rows=5 width=6) + TableScan [TS_0] (rows=5 width=6) + default@tgt_rc_merge_test,tgt_rc_merge_test,Tbl:COMPLETE,Col:COMPLETE + +PREHOOK: query: explain select sum(hash(key)), sum(hash(value)) from tgt_rc_merge_test +PREHOOK: type: QUERY +POSTHOOK: query: explain select sum(hash(key)), sum(hash(value)) from tgt_rc_merge_test +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_6] + Group By Operator [GBY_4] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] + <-Map 1 [GROUP] + GROUP [RS_3] + Select Operator [SEL_1] (rows=5 width=6) + Output:["_col0","_col1"] + TableScan [TS_0] (rows=5 width=6) + default@tgt_rc_merge_test,tgt_rc_merge_test,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + +PREHOOK: query: alter table tgt_rc_merge_test concatenate +PREHOOK: type: ALTER_TABLE_MERGE +PREHOOK: Input: default@tgt_rc_merge_test +PREHOOK: Output: default@tgt_rc_merge_test +POSTHOOK: query: alter table tgt_rc_merge_test concatenate +POSTHOOK: type: ALTER_TABLE_MERGE +POSTHOOK: Input: default@tgt_rc_merge_test +POSTHOOK: Output: default@tgt_rc_merge_test +PREHOOK: query: show table extended like `tgt_rc_merge_test` +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like `tgt_rc_merge_test` +POSTHOOK: type: SHOW_TABLESTATUS +tableName:tgt_rc_merge_test +#### A masked pattern was here #### +inputformat:org.apache.hadoop.hive.ql.io.RCFileInputFormat +outputformat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat +columns:struct columns { i32 key, string value} +partitioned:false +partitionColumns: +totalNumberFiles:1 +totalFileSize:171 +maxFileSize:171 +minFileSize:171 +#### A masked pattern was here #### + +PREHOOK: query: explain select count(1) from tgt_rc_merge_test +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(1) from tgt_rc_merge_test +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_6] + Group By Operator [GBY_4] (rows=1 width=8) + Output:["_col0"],aggregations:["count(1)"] + <-Map 1 [GROUP] + GROUP [RS_3] + Select Operator [SEL_1] (rows=5 width=6) + TableScan [TS_0] (rows=5 width=6) + default@tgt_rc_merge_test,tgt_rc_merge_test,Tbl:COMPLETE,Col:COMPLETE + +PREHOOK: query: explain select sum(hash(key)), sum(hash(value)) from tgt_rc_merge_test +PREHOOK: type: QUERY +POSTHOOK: query: explain select sum(hash(key)), sum(hash(value)) from tgt_rc_merge_test +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_6] + Group By Operator [GBY_4] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] + <-Map 1 [GROUP] + GROUP [RS_3] + Select Operator [SEL_1] (rows=5 width=6) + Output:["_col0","_col1"] + TableScan [TS_0] (rows=5 width=6) + default@tgt_rc_merge_test,tgt_rc_merge_test,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + +PREHOOK: query: drop table src_rc_merge_test +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@src_rc_merge_test +PREHOOK: Output: default@src_rc_merge_test +POSTHOOK: query: drop table src_rc_merge_test +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@src_rc_merge_test +POSTHOOK: Output: default@src_rc_merge_test +PREHOOK: query: drop table tgt_rc_merge_test +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@tgt_rc_merge_test +PREHOOK: Output: default@tgt_rc_merge_test +POSTHOOK: query: drop table tgt_rc_merge_test +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@tgt_rc_merge_test +POSTHOOK: Output: default@tgt_rc_merge_test +PREHOOK: query: explain select src.key from src cross join src src2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select src.key from src cross join src src2 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_8] + Join Operator [JOIN_6] (rows=250000 width=87) + Output:["_col0"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_4] + Select Operator [SEL_1] (rows=500 width=87) + Output:["_col0"] + TableScan [TS_0] (rows=500 width=87) + default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_5] + Select Operator [SEL_3] (rows=500 width=4) + TableScan [TS_2] (rows=500 width=10) + default@src,src2,Tbl:COMPLETE,Col:COMPLETE + +PREHOOK: query: explain create table nzhang_Tmp(a int, b string) +PREHOOK: type: CREATETABLE +POSTHOOK: query: explain create table nzhang_Tmp(a int, b string) +POSTHOOK: type: CREATETABLE +Stage-0 + Create Table Operator: + name:default.nzhang_Tmp + +PREHOOK: query: create table nzhang_Tmp(a int, b string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nzhang_Tmp +POSTHOOK: query: create table nzhang_Tmp(a int, b string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nzhang_Tmp +PREHOOK: query: explain create table nzhang_CTAS1 as select key k, value from src sort by k, value limit 10 +PREHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: query: explain create table nzhang_CTAS1 as select key k, value from src sort by k, value limit 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT) + +Stage-2 + Stats-Aggr Operator + Stage-3 + Create Table Operator: + name:default.nzhang_CTAS1 + Stage-0 + Move Operator + Stage-1 + Reducer 3 + File Output Operator [FS_8] + table:{"name:":"default.nzhang_CTAS1"} + Limit [LIM_7] (rows=10 width=178) + Number of rows:10 + Select Operator [SEL_6] (rows=10 width=178) + Output:["_col0","_col1"] + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_5] + Limit [LIM_4] (rows=10 width=178) + Number of rows:10 + Select Operator [SEL_3] (rows=500 width=178) + Output:["_col0","_col1"] + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_2] + Select Operator [SEL_1] (rows=500 width=178) + Output:["_col0","_col1"] + TableScan [TS_0] (rows=500 width=178) + default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + +PREHOOK: query: create table nzhang_CTAS1 as select key k, value from src sort by k, value limit 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@nzhang_CTAS1 +POSTHOOK: query: create table nzhang_CTAS1 as select key k, value from src sort by k, value limit 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nzhang_CTAS1 +POSTHOOK: Lineage: nzhang_ctas1.k SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_ctas1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain create table nzhang_ctas3 row format serde "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" stored as RCFile as select key/2 half_key, concat(value, "_con") conb from src sort by half_key, conb limit 10 +PREHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: query: explain create table nzhang_ctas3 row format serde "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" stored as RCFile as select key/2 half_key, concat(value, "_con") conb from src sort by half_key, conb limit 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT) + +Stage-2 + Stats-Aggr Operator + Stage-3 + Create Table Operator: + name:default.nzhang_ctas3 + Stage-0 + Move Operator + Stage-1 + Reducer 3 + File Output Operator [FS_8] + table:{"name:":"default.nzhang_ctas3"} + Limit [LIM_7] (rows=10 width=192) + Number of rows:10 + Select Operator [SEL_6] (rows=10 width=192) + Output:["_col0","_col1"] + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_5] + Limit [LIM_4] (rows=10 width=192) + Number of rows:10 + Select Operator [SEL_3] (rows=500 width=192) + Output:["_col0","_col1"] + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_2] + Select Operator [SEL_1] (rows=500 width=192) + Output:["_col0","_col1"] + TableScan [TS_0] (rows=500 width=178) + default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + +PREHOOK: query: create table nzhang_ctas3 row format serde "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" stored as RCFile as select key/2 half_key, concat(value, "_con") conb from src sort by half_key, conb limit 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@nzhang_ctas3 +POSTHOOK: query: create table nzhang_ctas3 row format serde "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" stored as RCFile as select key/2 half_key, concat(value, "_con") conb from src sort by half_key, conb limit 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nzhang_ctas3 +POSTHOOK: Lineage: nzhang_ctas3.conb EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_ctas3.half_key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: explain create table if not exists nzhang_ctas3 as select key, value from src sort by key, value limit 2 +PREHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: query: explain create table if not exists nzhang_ctas3 as select key, value from src sort by key, value limit 2 +POSTHOOK: type: CREATETABLE_AS_SELECT + +PREHOOK: query: create table if not exists nzhang_ctas3 as select key, value from src sort by key, value limit 2 +PREHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: query: create table if not exists nzhang_ctas3 as select key, value from src sort by key, value limit 2 +POSTHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: query: explain create temporary table acid_dtt(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +POSTHOOK: query: explain create temporary table acid_dtt(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +Stage-0 + Create Table Operator: + name:default.acid_dtt + +PREHOOK: query: create temporary table acid_dtt(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_dtt +POSTHOOK: query: create temporary table acid_dtt(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_dtt +PREHOOK: query: explain +select src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (select * FROM src WHERE src.key < 10) src1 + JOIN + (select * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (select * FROM src WHERE src.key < 10) src1 + JOIN + (select * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2 +POSTHOOK: type: QUERY +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 4 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_12] + Select Operator [SEL_11] (rows=27556 width=356) + Output:["_col0","_col1","_col2","_col3"] + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_10] + Join Operator [JOIN_8] (rows=27556 width=356) + Output:["_col0","_col1","_col2","_col3"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_6] + Select Operator [SEL_2] (rows=166 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_13] (rows=166 width=178) + predicate:(key < 10) + TableScan [TS_0] (rows=500 width=178) + default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Map 4 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_7] + Select Operator [SEL_5] (rows=166 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_14] (rows=166 width=178) + predicate:(key < 10) + TableScan [TS_3] (rows=500 width=178) + default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + +PREHOOK: query: CREATE TABLE myinput1(key int, value int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@myinput1 +POSTHOOK: query: CREATE TABLE myinput1(key int, value int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@myinput1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in8.txt' INTO TABLE myinput1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@myinput1 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in8.txt' INTO TABLE myinput1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@myinput1 +PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value +POSTHOOK: type: QUERY +Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_6] + Select Operator [SEL_5] (rows=3 width=9) + Output:["_col0","_col1","_col2","_col3"] + Join Operator [JOIN_4] (rows=3 width=9) + Output:["_col0","_col1","_col5","_col6"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{"0":"key","1":"value"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_2] + PartitionCols:key + TableScan [TS_0] (rows=3 width=8) + default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_3] + PartitionCols:value + TableScan [TS_1] (rows=3 width=8) + default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + +PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key +POSTHOOK: type: QUERY +Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT), Map 4 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_11] + Select Operator [SEL_10] (rows=6 width=9) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + Join Operator [JOIN_9] (rows=6 width=9) + Output:["_col0","_col1","_col5","_col6","_col10","_col11"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"},{"":"{\"type\":\"Inner\",\"left\":0,\"right\":2}"}],keys:{"0":"key","1":"value","2":"key"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_4] + PartitionCols:key + Filter Operator [FIL_12] (rows=3 width=8) + predicate:key is not null + TableScan [TS_0] (rows=3 width=8) + default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_6] + PartitionCols:value + Filter Operator [FIL_13] (rows=3 width=8) + predicate:value is not null + TableScan [TS_1] (rows=3 width=8) + default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Map 4 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_8] + PartitionCols:key + Filter Operator [FIL_14] (rows=3 width=8) + predicate:key is not null + TableScan [TS_2] (rows=3 width=8) + default@myinput1,c,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + +PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key +POSTHOOK: type: QUERY +Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT), Map 4 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_8] + Select Operator [SEL_7] (rows=6 width=9) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + Join Operator [JOIN_6] (rows=6 width=9) + Output:["_col0","_col1","_col5","_col6","_col10","_col11"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"},{"":"{\"type\":\"Inner\",\"left\":0,\"right\":2}"}],keys:{"0":"key","1":"value","2":"key"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_3] + PartitionCols:key + TableScan [TS_0] (rows=3 width=8) + default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_4] + PartitionCols:value + TableScan [TS_1] (rows=3 width=8) + default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Map 4 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_5] + PartitionCols:key + TableScan [TS_2] (rows=3 width=8) + default@myinput1,c,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + +PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value +POSTHOOK: type: QUERY +Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT), Map 4 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_11] + Select Operator [SEL_10] (rows=6 width=9) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + Join Operator [JOIN_9] (rows=6 width=9) + Output:["_col0","_col1","_col5","_col6","_col10","_col11"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"},{"":"{\"type\":\"Inner\",\"left\":0,\"right\":2}"}],keys:{"0":"key, value","1":"value, key","2":"key, value"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_4] + PartitionCols:key, value + Filter Operator [FIL_12] (rows=3 width=8) + predicate:value is not null + TableScan [TS_0] (rows=3 width=8) + default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_6] + PartitionCols:value, key + Filter Operator [FIL_13] (rows=3 width=8) + predicate:key is not null + TableScan [TS_1] (rows=3 width=8) + default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Map 4 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_8] + PartitionCols:key, value + Filter Operator [FIL_14] (rows=3 width=8) + predicate:value is not null + TableScan [TS_2] (rows=3 width=8) + default@myinput1,c,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + +PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value +POSTHOOK: type: QUERY +Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT), Map 4 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_8] + Select Operator [SEL_7] (rows=6 width=9) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + Join Operator [JOIN_6] (rows=6 width=9) + Output:["_col0","_col1","_col5","_col6","_col10","_col11"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"},{"":"{\"type\":\"Inner\",\"left\":0,\"right\":2}"}],keys:{"0":"key, value","1":"value, key","2":"key, value"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_3] + PartitionCols:key, value + TableScan [TS_0] (rows=3 width=8) + default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_4] + PartitionCols:value, key + TableScan [TS_1] (rows=3 width=8) + default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Map 4 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_5] + PartitionCols:key, value + TableScan [TS_2] (rows=3 width=8) + default@myinput1,c,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + +PREHOOK: query: explain select * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key<=>b.value +PREHOOK: type: QUERY +POSTHOOK: query: explain select * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key<=>b.value +POSTHOOK: type: QUERY +Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_6] + Select Operator [SEL_5] (rows=3 width=9) + Output:["_col0","_col1","_col2","_col3"] + Join Operator [JOIN_4] (rows=3 width=9) + Output:["_col0","_col1","_col5","_col6"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"key","1":"value"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_2] + PartitionCols:key + TableScan [TS_0] (rows=3 width=8) + default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_3] + PartitionCols:value + TableScan [TS_1] (rows=3 width=8) + default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + +PREHOOK: query: explain select * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key<=>b.value +PREHOOK: type: QUERY +POSTHOOK: query: explain select * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key<=>b.value +POSTHOOK: type: QUERY +Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_6] + Select Operator [SEL_5] (rows=3 width=9) + Output:["_col0","_col1","_col2","_col3"] + Join Operator [JOIN_4] (rows=3 width=9) + Output:["_col0","_col1","_col5","_col6"],condition map:[{"":"{\"type\":\"Right Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"key","1":"value"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_2] + PartitionCols:key + TableScan [TS_0] (rows=3 width=8) + default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_3] + PartitionCols:value + TableScan [TS_1] (rows=3 width=8) + default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + +PREHOOK: query: explain select * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key<=>b.value +PREHOOK: type: QUERY +POSTHOOK: query: explain select * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key<=>b.value +POSTHOOK: type: QUERY +Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_6] + Select Operator [SEL_5] (rows=3 width=9) + Output:["_col0","_col1","_col2","_col3"] + Join Operator [JOIN_4] (rows=3 width=9) + Output:["_col0","_col1","_col5","_col6"],condition map:[{"":"{\"type\":\"Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"key","1":"value"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_2] + PartitionCols:key + TableScan [TS_0] (rows=3 width=8) + default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_3] + PartitionCols:value + TableScan [TS_1] (rows=3 width=8) + default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + +PREHOOK: query: explain select /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value +POSTHOOK: type: QUERY +Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Map 1 + File Output Operator [FS_6] + Select Operator [SEL_8] (rows=3 width=9) + Output:["_col0","_col1","_col2","_col3"] + Map Join Operator [MAPJOIN_7] (rows=3 width=9) + Conds:TS_0.key=TS_0.value(Inner),Output:["_col0","_col1","_col5","_col6"] + <-TableScan [TS_0] (rows=3 width=8) + default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Map Reduce Local Work + Stage-2 + Map 2 + keys: [HASHTABLESINK_10] + 0key,1value + TableScan [TS_1] (rows=3 width=8) + default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Map Reduce Local Work + +PREHOOK: query: CREATE TABLE smb_input(key int, value int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@smb_input +POSTHOOK: query: CREATE TABLE smb_input(key int, value int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@smb_input +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in4.txt' into table smb_input +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@smb_input +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in4.txt' into table smb_input +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@smb_input +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in5.txt' into table smb_input +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@smb_input +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in5.txt' into table smb_input +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@smb_input +PREHOOK: query: CREATE TABLE smb_input1(key int, value int) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@smb_input1 +POSTHOOK: query: CREATE TABLE smb_input1(key int, value int) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@smb_input1 +PREHOOK: query: CREATE TABLE smb_input2(key int, value int) CLUSTERED BY (value) SORTED BY (value) INTO 2 BUCKETS +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@smb_input2 +POSTHOOK: query: CREATE TABLE smb_input2(key int, value int) CLUSTERED BY (value) SORTED BY (value) INTO 2 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@smb_input2 +PREHOOK: query: from smb_input +insert overwrite table smb_input1 select * +insert overwrite table smb_input2 select * +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_input +PREHOOK: Output: default@smb_input1 +PREHOOK: Output: default@smb_input2 +POSTHOOK: query: from smb_input +insert overwrite table smb_input1 select * +insert overwrite table smb_input2 select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_input +POSTHOOK: Output: default@smb_input1 +POSTHOOK: Output: default@smb_input2 +POSTHOOK: Lineage: smb_input1.key SIMPLE [(smb_input)smb_input.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: smb_input1.value SIMPLE [(smb_input)smb_input.FieldSchema(name:value, type:int, comment:null), ] +POSTHOOK: Lineage: smb_input2.key SIMPLE [(smb_input)smb_input.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: smb_input2.value SIMPLE [(smb_input)smb_input.FieldSchema(name:value, type:int, comment:null), ] +PREHOOK: query: analyze table smb_input1 compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_input1 +PREHOOK: Output: default@smb_input1 +POSTHOOK: query: analyze table smb_input1 compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_input1 +POSTHOOK: Output: default@smb_input1 +PREHOOK: query: explain select /*+ MAPJOIN(a) */ * FROM smb_input1 a JOIN smb_input1 b ON a.key <=> b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+ MAPJOIN(a) */ * FROM smb_input1 a JOIN smb_input1 b ON a.key <=> b.key +POSTHOOK: type: QUERY +Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Map 1 + File Output Operator [FS_6] + Select Operator [SEL_8] (rows=28 width=7) + Output:["_col0","_col1","_col2","_col3"] + Sorted Merge Bucket Map Join Operator [MAPJOIN_7] (rows=28 width=7) + BucketMapJoin:true,Conds:TS_1.key=TS_1.key(Inner),Output:["_col0","_col1","_col5","_col6"] + <-TableScan [TS_1] (rows=26 width=7) + default@smb_input1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + +PREHOOK: query: explain select /*+ MAPJOIN(a) */ * FROM smb_input1 a JOIN smb_input1 b ON a.key <=> b.key AND a.value <=> b.value +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+ MAPJOIN(a) */ * FROM smb_input1 a JOIN smb_input1 b ON a.key <=> b.key AND a.value <=> b.value +POSTHOOK: type: QUERY +Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Map 2 + File Output Operator [FS_6] + Select Operator [SEL_8] (rows=28 width=7) + Output:["_col0","_col1","_col2","_col3"] + Map Join Operator [MAPJOIN_7] (rows=28 width=7) + BucketMapJoin:true,Conds:TS_1.key, value=TS_1.key, value(Inner),Output:["_col0","_col1","_col5","_col6"] + <-TableScan [TS_1] (rows=26 width=7) + default@smb_input1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Map Reduce Local Work + Stage-2 + Map 1 + keys: [HASHTABLESINK_10] + 0key, value,1key, value + TableScan [TS_0] (rows=26 width=7) + default@smb_input1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Map Reduce Local Work + +PREHOOK: query: explain select /*+ MAPJOIN(a) */ * FROM smb_input1 a RIGHT OUTER JOIN smb_input1 b ON a.key <=> b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+ MAPJOIN(a) */ * FROM smb_input1 a RIGHT OUTER JOIN smb_input1 b ON a.key <=> b.key +POSTHOOK: type: QUERY +Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Map 1 + File Output Operator [FS_6] + Select Operator [SEL_8] (rows=28 width=7) + Output:["_col0","_col1","_col2","_col3"] + Sorted Merge Bucket Map Join Operator [MAPJOIN_7] (rows=28 width=7) + BucketMapJoin:true,Conds:TS_1.key=TS_1.key(Right Outer),Output:["_col0","_col1","_col5","_col6"] + <-TableScan [TS_1] (rows=26 width=7) + default@smb_input1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + +PREHOOK: query: explain select /*+ MAPJOIN(b) */ * FROM smb_input1 a JOIN smb_input1 b ON a.key <=> b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+ MAPJOIN(b) */ * FROM smb_input1 a JOIN smb_input1 b ON a.key <=> b.key +POSTHOOK: type: QUERY +Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Map 1 + File Output Operator [FS_6] + Select Operator [SEL_8] (rows=28 width=7) + Output:["_col0","_col1","_col2","_col3"] + Sorted Merge Bucket Map Join Operator [MAPJOIN_7] (rows=28 width=7) + BucketMapJoin:true,Conds:TS_0.key=TS_0.key(Inner),Output:["_col0","_col1","_col5","_col6"] + <-TableScan [TS_0] (rows=26 width=7) + default@smb_input1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + +PREHOOK: query: explain select /*+ MAPJOIN(b) */ * FROM smb_input1 a LEFT OUTER JOIN smb_input1 b ON a.key <=> b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+ MAPJOIN(b) */ * FROM smb_input1 a LEFT OUTER JOIN smb_input1 b ON a.key <=> b.key +POSTHOOK: type: QUERY +Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Map 1 + File Output Operator [FS_6] + Select Operator [SEL_8] (rows=28 width=7) + Output:["_col0","_col1","_col2","_col3"] + Sorted Merge Bucket Map Join Operator [MAPJOIN_7] (rows=28 width=7) + BucketMapJoin:true,Conds:TS_0.key=TS_0.key(Left Outer),Output:["_col0","_col1","_col5","_col6"] + <-TableScan [TS_0] (rows=26 width=7) + default@smb_input1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + +PREHOOK: query: drop table sales +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table sales +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table things +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table things +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE sales (name STRING, id INT) +ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@sales +POSTHOOK: query: CREATE TABLE sales (name STRING, id INT) +ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@sales +PREHOOK: query: CREATE TABLE things (id INT, name STRING) partitioned by (ds string) +ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@things +POSTHOOK: query: CREATE TABLE things (id INT, name STRING) partitioned by (ds string) +ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@things +PREHOOK: query: load data local inpath '../../data/files/sales.txt' INTO TABLE sales +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@sales +POSTHOOK: query: load data local inpath '../../data/files/sales.txt' INTO TABLE sales +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@sales +PREHOOK: query: load data local inpath '../../data/files/things.txt' INTO TABLE things partition(ds='2011-10-23') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@things +POSTHOOK: query: load data local inpath '../../data/files/things.txt' INTO TABLE things partition(ds='2011-10-23') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@things +POSTHOOK: Output: default@things@ds=2011-10-23 +PREHOOK: query: load data local inpath '../../data/files/things2.txt' INTO TABLE things partition(ds='2011-10-24') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@things +POSTHOOK: query: load data local inpath '../../data/files/things2.txt' INTO TABLE things partition(ds='2011-10-24') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@things +POSTHOOK: Output: default@things@ds=2011-10-24 +PREHOOK: query: explain select name,id FROM sales LEFT SEMI JOIN things ON (sales.id = things.id) +PREHOOK: type: QUERY +POSTHOOK: query: explain select name,id FROM sales LEFT SEMI JOIN things ON (sales.id = things.id) +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_12] + Join Operator [JOIN_10] (rows=2 width=15) + Output:["_col0","_col1"],condition map:[{"":"{\"type\":\"Left Semi\",\"left\":0,\"right\":1}"}],keys:{"0":"_col1","1":"_col0"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_8] + PartitionCols:_col1 + Select Operator [SEL_2] (rows=1 width=13) + Output:["_col0","_col1"] + Filter Operator [FIL_13] (rows=1 width=13) + predicate:id is not null + TableScan [TS_0] (rows=1 width=13) + default@sales,sales,Tbl:COMPLETE,Col:NONE,Output:["name","id"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_9] + PartitionCols:_col0 + Group By Operator [GBY_7] (rows=2 width=14) + Output:["_col0"],keys:_col0 + Select Operator [SEL_5] (rows=2 width=14) + Output:["_col0"] + Filter Operator [FIL_14] (rows=2 width=14) + predicate:id is not null + TableScan [TS_3] (rows=2 width=14) + default@things,things,Tbl:COMPLETE,Col:NONE,Output:["id"] + +PREHOOK: query: drop table sales +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@sales +PREHOOK: Output: default@sales +POSTHOOK: query: drop table sales +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@sales +POSTHOOK: Output: default@sales +PREHOOK: query: drop table things +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@things +PREHOOK: Output: default@things +POSTHOOK: query: drop table things +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@things +POSTHOOK: Output: default@things +PREHOOK: query: explain select srcpart.key from srcpart join src on (srcpart.value=src.value) join src1 on (srcpart.key=src1.key) where srcpart.value > 'val_450' +PREHOOK: type: QUERY +POSTHOOK: query: explain select srcpart.key from srcpart join src on (srcpart.value=src.value) join src1 on (srcpart.key=src1.key) where srcpart.value > 'val_450' +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Map 1 + File Output Operator [FS_16] + Map Join Operator [MAPJOIN_20] (rows=805 width=10) + Conds:MAPJOIN_21._col1=MAPJOIN_21._col0(Inner),Output:["_col0"] + <-Map Join Operator [MAPJOIN_21] (rows=732 width=10) + Conds:SEL_2._col0=SEL_2._col0(Inner),Output:["_col0","_col1"] + <-Select Operator [SEL_2] (rows=666 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_17] (rows=666 width=10) + predicate:((value > 'val_450') and key is not null) + TableScan [TS_0] (rows=2000 width=10) + default@srcpart,srcpart,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Map Reduce Local Work + Stage-2 + Map 2 + keys: [HASHTABLESINK_23] + 0_col0,1_col0 + Select Operator [SEL_5] (rows=25 width=7) + Output:["_col0"] + Filter Operator [FIL_18] (rows=25 width=7) + predicate:key is not null + TableScan [TS_3] (rows=25 width=7) + default@src1,src1,Tbl:COMPLETE,Col:NONE,Output:["key"] + Map Reduce Local Work + Map 3 + keys: [HASHTABLESINK_26] + 0_col1,1_col0 + Select Operator [SEL_8] (rows=166 width=10) + Output:["_col0"] + Filter Operator [FIL_19] (rows=166 width=10) + predicate:(value > 'val_450') + TableScan [TS_6] (rows=500 width=10) + default@src,src,Tbl:COMPLETE,Col:NONE,Output:["value"] + Map Reduce Local Work + +PREHOOK: query: explain select srcpart.key from srcpart join src on (srcpart.value=src.value) join src1 on (srcpart.key=src1.key) where srcpart.value > 'val_450' +PREHOOK: type: QUERY +POSTHOOK: query: explain select srcpart.key from srcpart join src on (srcpart.value=src.value) join src1 on (srcpart.key=src1.key) where srcpart.value > 'val_450' +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Map 1 + File Output Operator [FS_16] + Map Join Operator [MAPJOIN_20] (rows=805 width=10) + Conds:MAPJOIN_21._col1=MAPJOIN_21._col0(Inner),Output:["_col0"] + <-Map Join Operator [MAPJOIN_21] (rows=732 width=10) + Conds:SEL_2._col0=SEL_2._col0(Inner),Output:["_col0","_col1"] + <-Select Operator [SEL_2] (rows=666 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_17] (rows=666 width=10) + predicate:((value > 'val_450') and key is not null) + TableScan [TS_0] (rows=2000 width=10) + default@srcpart,srcpart,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Map Reduce Local Work + Stage-2 + Map 2 + keys: [HASHTABLESINK_23] + 0_col0,1_col0 + Select Operator [SEL_5] (rows=25 width=7) + Output:["_col0"] + Filter Operator [FIL_18] (rows=25 width=7) + predicate:key is not null + TableScan [TS_3] (rows=25 width=7) + default@src1,src1,Tbl:COMPLETE,Col:NONE,Output:["key"] + Map Reduce Local Work + Map 3 + keys: [HASHTABLESINK_26] + 0_col1,1_col0 + Select Operator [SEL_8] (rows=166 width=10) + Output:["_col0"] + Filter Operator [FIL_19] (rows=166 width=10) + predicate:(value > 'val_450') + TableScan [TS_6] (rows=500 width=10) + default@src,src,Tbl:COMPLETE,Col:NONE,Output:["value"] + Map Reduce Local Work + +PREHOOK: query: explain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noop(on part + partition by p_mfgr + order by p_name + ) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noop(on part + partition by p_mfgr + order by p_name + ) +POSTHOOK: type: QUERY +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_9] + Select Operator [SEL_7] (rows=26 width=239) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + PTF Operator [PTF_6] (rows=26 width=499) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] + Select Operator [SEL_5] (rows=26 width=499) + Output:["_col1","_col2","_col5","_col7"] + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_4] + PartitionCols:_col2 + PTF Operator [PTF_3] (rows=26 width=499) + Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] + Select Operator [SEL_2] (rows=26 width=499) + Output:["_col1","_col2","_col5","_col7"] + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_1] + PartitionCols:p_mfgr + TableScan [TS_0] (rows=26 width=231) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] + +PREHOOK: query: explain +select p_mfgr, p_name, +p_size, p_size - lag(p_size,1,p_size) over (partition by p_mfgr order by p_name) as deltaSz +from noop (on (select p1.* from part p1 join part p2 on p1.p_partkey = p2.p_partkey) j +distribute by j.p_mfgr +sort by j.p_name) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select p_mfgr, p_name, +p_size, p_size - lag(p_size,1,p_size) over (partition by p_mfgr order by p_name) as deltaSz +from noop (on (select p1.* from part p1 join part p2 on p1.p_partkey = p2.p_partkey) j +distribute by j.p_mfgr +sort by j.p_name) +POSTHOOK: type: QUERY +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_16] + Select Operator [SEL_14] (rows=29 width=227) + Output:["_col0","_col1","_col2","_col3"] + PTF Operator [PTF_13] (rows=29 width=223) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] + Select Operator [SEL_12] (rows=29 width=223) + Output:["_col1","_col2","_col5"] + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_11] + PartitionCols:_col2 + PTF Operator [PTF_10] (rows=29 width=223) + Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] + Select Operator [SEL_9] (rows=29 width=223) + Output:["_col1","_col2","_col5"] + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_8] + PartitionCols:_col2 + Map Join Operator [MAPJOIN_19] (rows=29 width=223) + Conds:FIL_17.p_partkey=FIL_17.p_partkey(Inner),Output:["_col1","_col2","_col5"] + <-Filter Operator [FIL_17] (rows=26 width=227) + predicate:p_partkey is not null + TableScan [TS_0] (rows=26 width=227) + default@part,p1,Tbl:COMPLETE,Col:COMPLETE,Output:["p_partkey","p_name","p_mfgr","p_size"] + Map Reduce Local Work + Stage-2 + Map 4 + keys: [HASHTABLESINK_21] + 0p_partkey,1p_partkey + Filter Operator [FIL_18] (rows=26 width=4) + predicate:p_partkey is not null + TableScan [TS_1] (rows=26 width=4) + default@part,p2,Tbl:COMPLETE,Col:COMPLETE,Output:["p_partkey"] + Map Reduce Local Work + +PREHOOK: query: explain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noop(on part + partition by p_mfgr + order by p_name + ) abc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noop(on part + partition by p_mfgr + order by p_name + ) abc +POSTHOOK: type: QUERY +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_9] + Select Operator [SEL_7] (rows=26 width=239) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + PTF Operator [PTF_6] (rows=26 width=499) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] + Select Operator [SEL_5] (rows=26 width=499) + Output:["_col1","_col2","_col5","_col7"] + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_4] + PartitionCols:_col2 + PTF Operator [PTF_3] (rows=26 width=499) + Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] + Select Operator [SEL_2] (rows=26 width=499) + Output:["_col1","_col2","_col5","_col7"] + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_1] + PartitionCols:p_mfgr + TableScan [TS_0] (rows=26 width=231) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] + +PREHOOK: query: explain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +p_size, p_size - lag(p_size,1,p_size) over (partition by p_mfgr order by p_name) as deltaSz +from noop(on part + partition by p_mfgr + order by p_name + ) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +p_size, p_size - lag(p_size,1,p_size) over (partition by p_mfgr order by p_name) as deltaSz +from noop(on part + partition by p_mfgr + order by p_name + ) +POSTHOOK: type: QUERY +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_9] + Select Operator [SEL_7] (rows=26 width=239) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + PTF Operator [PTF_6] (rows=26 width=491) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] + Select Operator [SEL_5] (rows=26 width=491) + Output:["_col1","_col2","_col5"] + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_4] + PartitionCols:_col2 + PTF Operator [PTF_3] (rows=26 width=491) + Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] + Select Operator [SEL_2] (rows=26 width=491) + Output:["_col1","_col2","_col5"] + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_1] + PartitionCols:p_mfgr + TableScan [TS_0] (rows=26 width=223) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] + +PREHOOK: query: explain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +p_size, p_size - lag(p_size,1,p_size) over (partition by p_mfgr order by p_name) as deltaSz +from noop(on part + partition by p_mfgr + order by p_name + ) +group by p_mfgr, p_name, p_size +PREHOOK: type: QUERY +POSTHOOK: query: explain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +p_size, p_size - lag(p_size,1,p_size) over (partition by p_mfgr order by p_name) as deltaSz +from noop(on part + partition by p_mfgr + order by p_name + ) +group by p_mfgr, p_name, p_size +POSTHOOK: type: QUERY +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT) +Reducer 4 <- Reducer 3 (GROUP PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 4 + File Output Operator [FS_14] + Select Operator [SEL_12] (rows=26 width=239) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + PTF Operator [PTF_11] (rows=26 width=223) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col0"}] + Group By Operator [GBY_8] (rows=26 width=223) + Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Reducer 3 [GROUP PARTITION-LEVEL SORT] + GROUP PARTITION-LEVEL SORT [RS_7] + PartitionCols:_col0 + Group By Operator [GBY_6] (rows=26 width=223) + Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 + <-Reducer 2 [GROUP PARTITION-LEVEL SORT] + GROUP PARTITION-LEVEL SORT [RS_5] + PartitionCols:rand() + Select Operator [SEL_4] (rows=26 width=491) + Output:["_col1","_col2","_col5"] + PTF Operator [PTF_3] (rows=26 width=491) + Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] + Select Operator [SEL_2] (rows=26 width=491) + Output:["_col1","_col2","_col5"] + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_1] + PartitionCols:p_mfgr + TableScan [TS_0] (rows=26 width=223) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] + +PREHOOK: query: explain +select abc.* +from noop(on part +partition by p_mfgr +order by p_name +) abc join part p1 on abc.p_partkey = p1.p_partkey +PREHOOK: type: QUERY +POSTHOOK: query: explain +select abc.* +from noop(on part +partition by p_mfgr +order by p_name +) abc join part p1 on abc.p_partkey = p1.p_partkey +POSTHOOK: type: QUERY +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + Map Reduce Local Work + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_2] + PartitionCols:p_mfgr + TableScan [TS_0] (rows=26 width=619) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_brand","p_comment","p_container","p_mfgr","p_name","p_partkey","p_retailprice","p_size","p_type"] + File Output Operator [FS_11] + Map Join Operator [MAPJOIN_14] (rows=29 width=619) + Conds:FIL_12._col0=FIL_12.p_partkey(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] + <-Filter Operator [FIL_12] (rows=26 width=887) + predicate:_col0 is not null + PTF Operator [PTF_4] (rows=26 width=887) + Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] + Select Operator [SEL_3] (rows=26 width=887) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] + <- Please refer to the previous Map 1 [PARTITION-LEVEL SORT] + Stage-2 + Map 3 + keys: [HASHTABLESINK_16] + 0_col0,1p_partkey + Filter Operator [FIL_13] (rows=26 width=4) + predicate:p_partkey is not null + TableScan [TS_1] (rows=26 width=4) + default@part,p1,Tbl:COMPLETE,Col:COMPLETE,Output:["p_partkey"] + Map Reduce Local Work + +PREHOOK: query: explain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name, p_size desc) as r +from noopwithmap(on part +partition by p_mfgr +order by p_name, p_size desc) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name, p_size desc) as r +from noopwithmap(on part +partition by p_mfgr +order by p_name, p_size desc) +POSTHOOK: type: QUERY +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_10] + Select Operator [SEL_8] (rows=26 width=227) + Output:["_col0","_col1","_col2","_col3"] + PTF Operator [PTF_7] (rows=26 width=491) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST, _col5 DESC NULLS LAST","partition by:":"_col2"}] + Select Operator [SEL_6] (rows=26 width=491) + Output:["_col1","_col2","_col5"] + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_5] + PartitionCols:_col2 + PTF Operator [PTF_4] (rows=26 width=491) + Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col1 ASC NULLS FIRST, _col5 DESC NULLS LAST","partition by:":"_col2"}}] + Select Operator [SEL_3] (rows=26 width=491) + Output:["_col1","_col2","_col5"] + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_2] + PartitionCols:p_mfgr + PTF Operator [PTF_1] (rows=26 width=223) + Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"p_name ASC NULLS FIRST, p_size DESC NULLS LAST","partition by:":"p_mfgr"}}] + TableScan [TS_0] (rows=26 width=223) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] + +PREHOOK: query: explain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noopwithmap(on part + partition by p_mfgr + order by p_name) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noopwithmap(on part + partition by p_mfgr + order by p_name) +POSTHOOK: type: QUERY +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_10] + Select Operator [SEL_8] (rows=26 width=239) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + PTF Operator [PTF_7] (rows=26 width=499) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] + Select Operator [SEL_6] (rows=26 width=499) + Output:["_col1","_col2","_col5","_col7"] + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_5] + PartitionCols:_col2 + PTF Operator [PTF_4] (rows=26 width=499) + Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] + Select Operator [SEL_3] (rows=26 width=499) + Output:["_col1","_col2","_col5","_col7"] + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_2] + PartitionCols:p_mfgr + PTF Operator [PTF_1] (rows=26 width=231) + Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"p_name ASC NULLS FIRST","partition by:":"p_mfgr"}}] + TableScan [TS_0] (rows=26 width=231) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size","p_retailprice"] + +PREHOOK: query: explain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noop(on part +partition by p_mfgr +order by p_name) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noop(on part +partition by p_mfgr +order by p_name) +POSTHOOK: type: QUERY +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_9] + Select Operator [SEL_7] (rows=26 width=239) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + PTF Operator [PTF_6] (rows=26 width=499) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] + Select Operator [SEL_5] (rows=26 width=499) + Output:["_col1","_col2","_col5","_col7"] + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_4] + PartitionCols:_col2 + PTF Operator [PTF_3] (rows=26 width=499) + Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] + Select Operator [SEL_2] (rows=26 width=499) + Output:["_col1","_col2","_col5","_col7"] + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_1] + PartitionCols:p_mfgr + TableScan [TS_0] (rows=26 width=231) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] + +PREHOOK: query: explain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noop(on noopwithmap(on noop(on part +partition by p_mfgr +order by p_mfgr DESC, p_name +))) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noop(on noopwithmap(on noop(on part +partition by p_mfgr +order by p_mfgr DESC, p_name +))) +POSTHOOK: type: QUERY +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT) +Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 4 + File Output Operator [FS_13] + Select Operator [SEL_11] (rows=26 width=239) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + PTF Operator [PTF_10] (rows=26 width=499) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] + Select Operator [SEL_9] (rows=26 width=499) + Output:["_col1","_col2","_col5","_col7"] + <-Reducer 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_8] + PartitionCols:_col2 + PTF Operator [PTF_7] (rows=26 width=499) + Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col2 DESC NULLS LAST, _col1 ASC NULLS FIRST","partition by:":"_col2"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 DESC NULLS LAST, _col1 ASC NULLS FIRST","partition by:":"_col2"}}] + Select Operator [SEL_6] (rows=26 width=499) + Output:["_col1","_col2","_col5","_col7"] + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_5] + PartitionCols:_col2 + PTF Operator [PTF_4] (rows=26 width=499) + Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col2 DESC NULLS LAST, _col1 ASC NULLS FIRST","partition by:":"_col2"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 DESC NULLS LAST, _col1 ASC NULLS FIRST","partition by:":"_col2"}}] + PTF Operator [PTF_3] (rows=26 width=499) + Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 DESC NULLS LAST, _col1 ASC NULLS FIRST","partition by:":"_col2"}}] + Select Operator [SEL_2] (rows=26 width=499) + Output:["_col1","_col2","_col5","_col7"] + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_1] + PartitionCols:p_mfgr + TableScan [TS_0] (rows=26 width=231) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] + +PREHOOK: query: explain +select p_mfgr, p_name, +sub1.cd, sub1.s1 +from (select p_mfgr, p_name, +count(p_size) over (partition by p_mfgr order by p_name) as cd, +p_retailprice, +sum(p_retailprice) over w1 as s1 +from noop(on part +partition by p_mfgr +order by p_name) +window w1 as (partition by p_mfgr order by p_name rows between 2 preceding and 2 following) +) sub1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select p_mfgr, p_name, +sub1.cd, sub1.s1 +from (select p_mfgr, p_name, +count(p_size) over (partition by p_mfgr order by p_name) as cd, +p_retailprice, +sum(p_retailprice) over w1 as s1 +from noop(on part +partition by p_mfgr +order by p_name) +window w1 as (partition by p_mfgr order by p_name rows between 2 preceding and 2 following) +) sub1 +POSTHOOK: type: QUERY +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_10] + Select Operator [SEL_7] (rows=26 width=235) + Output:["_col0","_col1","_col2","_col3"] + PTF Operator [PTF_6] (rows=26 width=499) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] + Select Operator [SEL_5] (rows=26 width=499) + Output:["_col1","_col2","_col5","_col7"] + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_4] + PartitionCols:_col2 + PTF Operator [PTF_3] (rows=26 width=499) + Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] + Select Operator [SEL_2] (rows=26 width=499) + Output:["_col1","_col2","_col5","_col7"] + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_1] + PartitionCols:p_mfgr + TableScan [TS_0] (rows=26 width=231) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] + +PREHOOK: query: explain +select abc.p_mfgr, abc.p_name, +rank() over (distribute by abc.p_mfgr sort by abc.p_name) as r, +dense_rank() over (distribute by abc.p_mfgr sort by abc.p_name) as dr, +count(abc.p_name) over (distribute by abc.p_mfgr sort by abc.p_name) as cd, +abc.p_retailprice, sum(abc.p_retailprice) over (distribute by abc.p_mfgr sort by abc.p_name rows between unbounded preceding and current row) as s1, +abc.p_size, abc.p_size - lag(abc.p_size,1,abc.p_size) over (distribute by abc.p_mfgr sort by abc.p_name) as deltaSz +from noop(on part +partition by p_mfgr +order by p_name +) abc join part p1 on abc.p_partkey = p1.p_partkey +PREHOOK: type: QUERY +POSTHOOK: query: explain +select abc.p_mfgr, abc.p_name, +rank() over (distribute by abc.p_mfgr sort by abc.p_name) as r, +dense_rank() over (distribute by abc.p_mfgr sort by abc.p_name) as dr, +count(abc.p_name) over (distribute by abc.p_mfgr sort by abc.p_name) as cd, +abc.p_retailprice, sum(abc.p_retailprice) over (distribute by abc.p_mfgr sort by abc.p_name rows between unbounded preceding and current row) as s1, +abc.p_size, abc.p_size - lag(abc.p_size,1,abc.p_size) over (distribute by abc.p_mfgr sort by abc.p_name) as deltaSz +from noop(on part +partition by p_mfgr +order by p_name +) abc join part p1 on abc.p_partkey = p1.p_partkey +POSTHOOK: type: QUERY +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_15] + Select Operator [SEL_13] (rows=29 width=259) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] + PTF Operator [PTF_12] (rows=29 width=767) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] + Select Operator [SEL_11] (rows=29 width=767) + Output:["_col1","_col2","_col5","_col7"] + <-Reducer 2 [PARTITION-LEVEL SORT] + Map Reduce Local Work + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_2] + PartitionCols:p_mfgr + TableScan [TS_0] (rows=26 width=235) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_partkey","p_retailprice","p_size"] + PARTITION-LEVEL SORT [RS_10] + PartitionCols:_col2 + Map Join Operator [MAPJOIN_18] (rows=29 width=231) + Conds:FIL_16._col0=FIL_16.p_partkey(Inner),Output:["_col1","_col2","_col5","_col7"] + <-Filter Operator [FIL_16] (rows=26 width=503) + predicate:_col0 is not null + PTF Operator [PTF_4] (rows=26 width=503) + Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] + Select Operator [SEL_3] (rows=26 width=503) + Output:["_col0","_col1","_col2","_col5","_col7"] + <- Please refer to the previous Map 1 [PARTITION-LEVEL SORT] + Stage-2 + Map 4 + keys: [HASHTABLESINK_20] + 0_col0,1p_partkey + Filter Operator [FIL_17] (rows=26 width=4) + predicate:p_partkey is not null + TableScan [TS_1] (rows=26 width=4) + default@part,p1,Tbl:COMPLETE,Col:COMPLETE,Output:["p_partkey"] + Map Reduce Local Work + +PREHOOK: query: explain create view IF NOT EXISTS mfgr_price_view as +select p_mfgr, p_brand, +sum(p_retailprice) as s +from part +group by p_mfgr, p_brand +PREHOOK: type: CREATEVIEW +POSTHOOK: query: explain create view IF NOT EXISTS mfgr_price_view as +select p_mfgr, p_brand, +sum(p_retailprice) as s +from part +group by p_mfgr, p_brand +POSTHOOK: type: CREATEVIEW +Plan optimized by CBO. + +Stage-1 + Create View Operator: + name:default.mfgr_price_view,original text:select p_mfgr, p_brand, +sum(p_retailprice) as s +from part +group by p_mfgr, p_brand + +PREHOOK: query: CREATE TABLE part_4( +p_mfgr STRING, +p_name STRING, +p_size INT, +r INT, +dr INT, +s DOUBLE) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_4 +POSTHOOK: query: CREATE TABLE part_4( +p_mfgr STRING, +p_name STRING, +p_size INT, +r INT, +dr INT, +s DOUBLE) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_4 +PREHOOK: query: CREATE TABLE part_5( +p_mfgr STRING, +p_name STRING, +p_size INT, +s2 INT, +r INT, +dr INT, +cud DOUBLE, +fv1 INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_5 +POSTHOOK: query: CREATE TABLE part_5( +p_mfgr STRING, +p_name STRING, +p_size INT, +s2 INT, +r INT, +dr INT, +cud DOUBLE, +fv1 INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_5 +PREHOOK: query: explain +from noop(on part +partition by p_mfgr +order by p_name) +INSERT OVERWRITE TABLE part_4 select p_mfgr, p_name, p_size, +rank() over (distribute by p_mfgr sort by p_name) as r, +dense_rank() over (distribute by p_mfgr sort by p_name) as dr, +sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s +INSERT OVERWRITE TABLE part_5 select p_mfgr,p_name, p_size, +round(sum(p_size) over (distribute by p_mfgr sort by p_size range between 5 preceding and current row),1) as s2, +rank() over (distribute by p_mfgr sort by p_mfgr, p_name) as r, +dense_rank() over (distribute by p_mfgr sort by p_mfgr, p_name) as dr, +cume_dist() over (distribute by p_mfgr sort by p_mfgr, p_name) as cud, +first_value(p_size, true) over w1 as fv1 +window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following) +PREHOOK: type: QUERY +POSTHOOK: query: explain +from noop(on part +partition by p_mfgr +order by p_name) +INSERT OVERWRITE TABLE part_4 select p_mfgr, p_name, p_size, +rank() over (distribute by p_mfgr sort by p_name) as r, +dense_rank() over (distribute by p_mfgr sort by p_name) as dr, +sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s +INSERT OVERWRITE TABLE part_5 select p_mfgr,p_name, p_size, +round(sum(p_size) over (distribute by p_mfgr sort by p_size range between 5 preceding and current row),1) as s2, +rank() over (distribute by p_mfgr sort by p_mfgr, p_name) as r, +dense_rank() over (distribute by p_mfgr sort by p_mfgr, p_name) as dr, +cume_dist() over (distribute by p_mfgr sort by p_mfgr, p_name) as cud, +first_value(p_size, true) over w1 as fv1 +window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following) +POSTHOOK: type: QUERY +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 3 <- Reducer 6 (PARTITION-LEVEL SORT) +Reducer 4 <- Reducer 7 (PARTITION-LEVEL SORT) +Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT) +Reducer 6 <- Map 1 (PARTITION-LEVEL SORT) +Reducer 7 <- Map 1 (PARTITION-LEVEL SORT) + +Stage-3 + Stats-Aggr Operator + Stage-0 + Move Operator + table:{"name:":"default.part_4"} + Stage-2 + Reducer 3 + File Output Operator [FS_9] + table:{"name:":"default.part_4"} + Select Operator [SEL_7] (rows=26 width=239) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + PTF Operator [PTF_6] (rows=26 width=499) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] + Select Operator [SEL_5] (rows=26 width=499) + Output:["_col1","_col2","_col5","_col7"] + <-Reducer 6 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_4] + PartitionCols:_col2 + PTF Operator [PTF_3] (rows=26 width=499) + Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] + Select Operator [SEL_2] (rows=26 width=499) + Output:["_col1","_col2","_col5","_col7"] + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_1] + PartitionCols:p_mfgr + TableScan [TS_0] (rows=26 width=231) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] + Reducer 5 + File Output Operator [FS_20] + table:{"name:":"default.part_5"} + Select Operator [SEL_17] (rows=26 width=247) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] + PTF Operator [PTF_16] (rows=26 width=499) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST, _col2 ASC NULLS FIRST","partition by:":"_col3"}] + Select Operator [SEL_15] (rows=26 width=499) + Output:["_col0","_col2","_col3","_col6"] + <-Reducer 4 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_14] + PartitionCols:_col2 + Select Operator [SEL_13] (rows=26 width=491) + Output:["sum_window_0","_col1","_col2","_col5"] + PTF Operator [PTF_12] (rows=26 width=491) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col5 ASC NULLS FIRST","partition by:":"_col2"}] + Select Operator [SEL_11] (rows=26 width=491) + Output:["_col1","_col2","_col5"] + <-Reducer 7 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_10] + PartitionCols:_col2 + PTF Operator [PTF_3] (rows=26 width=499) + Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] + Select Operator [SEL_2] (rows=26 width=499) + Output:["_col1","_col2","_col5","_col7"] + <- Please refer to the previous Map 1 [PARTITION-LEVEL SORT] +Stage-4 + Stats-Aggr Operator + Stage-1 + Move Operator + table:{"name:":"default.part_5"} + Please refer to the previous Stage-2 + +PREHOOK: query: explain +select p_mfgr, p_name, +rank() over (partition by p_mfgr,p_name) as r, +dense_rank() over (partition by p_mfgr,p_name) as dr, +p_size, sum(p_size) over (partition by p_mfgr,p_name rows between unbounded preceding and current row) as s1 +from noop(on + noopwithmap(on + noop(on + noop(on part + partition by p_mfgr + order by p_mfgr) + ) + partition by p_mfgr,p_name + order by p_mfgr,p_name) + partition by p_mfgr,p_name + order by p_mfgr,p_name) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select p_mfgr, p_name, +rank() over (partition by p_mfgr,p_name) as r, +dense_rank() over (partition by p_mfgr,p_name) as dr, +p_size, sum(p_size) over (partition by p_mfgr,p_name rows between unbounded preceding and current row) as s1 +from noop(on + noopwithmap(on + noop(on + noop(on part + partition by p_mfgr + order by p_mfgr) + ) + partition by p_mfgr,p_name + order by p_mfgr,p_name) + partition by p_mfgr,p_name + order by p_mfgr,p_name) +POSTHOOK: type: QUERY +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT) +Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 4 + File Output Operator [FS_13] + Select Operator [SEL_11] (rows=26 width=239) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + PTF Operator [PTF_10] (rows=26 width=491) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}] + Select Operator [SEL_9] (rows=26 width=491) + Output:["_col1","_col2","_col5"] + <-Reducer 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_8] + PartitionCols:_col2, _col1 + PTF Operator [PTF_7] (rows=26 width=491) + Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}}] + Select Operator [SEL_6] (rows=26 width=491) + Output:["_col1","_col2","_col5"] + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_5] + PartitionCols:_col2, _col1 + PTF Operator [PTF_4] (rows=26 width=491) + Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}}] + PTF Operator [PTF_3] (rows=26 width=491) + Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}}] + Select Operator [SEL_2] (rows=26 width=491) + Output:["_col1","_col2","_col5"] + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_1] + PartitionCols:p_mfgr + TableScan [TS_0] (rows=26 width=223) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] + +PREHOOK: query: explain +select p_mfgr, p_name, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +p_size, sum(p_size) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noop(on + noop(on + noop(on + noop(on part + partition by p_mfgr + order by p_mfgr) + ) + partition by p_mfgr,p_name + order by p_mfgr,p_name) + partition by p_mfgr + order by p_mfgr ) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select p_mfgr, p_name, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +p_size, sum(p_size) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noop(on + noop(on + noop(on + noop(on part + partition by p_mfgr + order by p_mfgr) + ) + partition by p_mfgr,p_name + order by p_mfgr,p_name) + partition by p_mfgr + order by p_mfgr ) +POSTHOOK: type: QUERY +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT) +Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT) +Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 5 + File Output Operator [FS_15] + Select Operator [SEL_13] (rows=26 width=239) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + PTF Operator [PTF_12] (rows=26 width=491) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] + Select Operator [SEL_11] (rows=26 width=491) + Output:["_col1","_col2","_col5"] + <-Reducer 4 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_10] + PartitionCols:_col2 + PTF Operator [PTF_9] (rows=26 width=491) + Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}}] + Select Operator [SEL_8] (rows=26 width=491) + Output:["_col1","_col2","_col5"] + <-Reducer 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_7] + PartitionCols:_col2 + PTF Operator [PTF_6] (rows=26 width=491) + Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}}] + Select Operator [SEL_5] (rows=26 width=491) + Output:["_col1","_col2","_col5"] + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_4] + PartitionCols:_col2, _col1 + PTF Operator [PTF_3] (rows=26 width=491) + Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}}] + Select Operator [SEL_2] (rows=26 width=491) + Output:["_col1","_col2","_col5"] + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_1] + PartitionCols:p_mfgr + TableScan [TS_0] (rows=26 width=223) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] + +PREHOOK: query: explain +select p_mfgr, p_name, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +p_size, sum(p_size) over (partition by p_mfgr order by p_name) as s1 +from noop(on + noop(on + noop(on + noop(on part + partition by p_mfgr,p_name + order by p_mfgr,p_name) + ) + partition by p_mfgr + order by p_mfgr)) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select p_mfgr, p_name, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +p_size, sum(p_size) over (partition by p_mfgr order by p_name) as s1 +from noop(on + noop(on + noop(on + noop(on part + partition by p_mfgr,p_name + order by p_mfgr,p_name) + ) + partition by p_mfgr + order by p_mfgr)) +POSTHOOK: type: QUERY +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT) +Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 4 + File Output Operator [FS_12] + Select Operator [SEL_10] (rows=26 width=239) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + PTF Operator [PTF_9] (rows=26 width=491) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] + Select Operator [SEL_8] (rows=26 width=491) + Output:["_col1","_col2","_col5"] + <-Reducer 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_7] + PartitionCols:_col2 + PTF Operator [PTF_6] (rows=26 width=491) + Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}}] + Select Operator [SEL_5] (rows=26 width=491) + Output:["_col1","_col2","_col5"] + <-Reducer 2 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_4] + PartitionCols:_col2 + PTF Operator [PTF_3] (rows=26 width=491) + Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}}] + Select Operator [SEL_2] (rows=26 width=491) + Output:["_col1","_col2","_col5"] + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_1] + PartitionCols:p_mfgr, p_name + TableScan [TS_0] (rows=26 width=223) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] + +PREHOOK: query: explain select distinct src.* from src +PREHOOK: type: QUERY +POSTHOOK: query: explain select distinct src.* from src +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_7] + Group By Operator [GBY_5] (rows=500 width=178) + Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 + <-Reducer 2 [GROUP] + GROUP [RS_4] + PartitionCols:_col0, _col1 + Group By Operator [GBY_3] (rows=500 width=178) + Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 + <-Map 1 [GROUP PARTITION-LEVEL SORT] + GROUP PARTITION-LEVEL SORT [RS_2] + PartitionCols:rand() + Select Operator [SEL_1] (rows=500 width=178) + Output:["key","value"] + TableScan [TS_0] (rows=500 width=178) + default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + +PREHOOK: query: explain select explode(array('a', 'b')) +PREHOOK: type: QUERY +POSTHOOK: query: explain select explode(array('a', 'b')) +POSTHOOK: type: QUERY +Plan not optimized by CBO. + +Stage-0 + Fetch Operator + limit:-1 + UDTF Operator [UDTF_2] + function name:explode + Select Operator [SEL_1] + Output:["_col0"] + TableScan [TS_0] + +PREHOOK: query: CREATE TABLE T1(key STRING, val STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@T1 +POSTHOOK: query: CREATE TABLE T1(key STRING, val STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@T1 +PREHOOK: query: CREATE TABLE T2(key STRING, val STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@T2 +POSTHOOK: query: CREATE TABLE T2(key STRING, val STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@T2 +PREHOOK: query: CREATE TABLE T3(key STRING, val STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@T3 +POSTHOOK: query: CREATE TABLE T3(key STRING, val STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@T3 +PREHOOK: query: CREATE TABLE T4(key STRING, val STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@T4 +POSTHOOK: query: CREATE TABLE T4(key STRING, val STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@T4 +PREHOOK: query: CREATE TABLE dest_j1(key INT, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: CREATE TABLE dest_j1(key INT, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T1.txt' INTO TABLE T1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@t1 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T1.txt' INTO TABLE T1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@t1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T2.txt' INTO TABLE T2 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@t2 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T2.txt' INTO TABLE T2 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@t2 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T3.txt' INTO TABLE T3 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@t3 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T3.txt' INTO TABLE T3 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@t3 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T1.txt' INTO TABLE T4 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@t4 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T1.txt' INTO TABLE T4 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@t4 +PREHOOK: query: explain +FROM src src1 JOIN src src2 ON (src1.key = src2.key) +INSERT OVERWRITE TABLE dest_j1 select src1.key, src2.value +PREHOOK: type: QUERY +POSTHOOK: query: explain +FROM src src1 JOIN src src2 ON (src1.key = src2.key) +INSERT OVERWRITE TABLE dest_j1 select src1.key, src2.value +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT) + +Stage-2 + Stats-Aggr Operator + Stage-0(CONDITIONAL) + Move Operator + table:{"name:":"default.dest_j1"} + Stage-4(CONDITIONAL CHILD TASKS: Stage-5, Stage-0) + Conditional Operator + Stage-1 + Reducer 2 + File Output Operator [FS_11] + table:{"name:":"default.dest_j1"} + Select Operator [SEL_9] (rows=1219 width=95) + Output:["_col0","_col1"] + Join Operator [JOIN_8] (rows=1219 width=178) + Output:["_col0","_col2"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_6] + PartitionCols:_col0 + Select Operator [SEL_2] (rows=500 width=87) + Output:["_col0"] + Filter Operator [FIL_12] (rows=500 width=87) + predicate:key is not null + TableScan [TS_0] (rows=500 width=87) + default@src,src1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_7] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_13] (rows=500 width=178) + predicate:key is not null + TableScan [TS_3] (rows=500 width=178) + default@src,src2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Stage-3 + Map 4 + File Output Operator [FS_11] + table:{"name:":"default.dest_j1"} + Select Operator [SEL_9] (rows=1219 width=95) + Output:["_col0","_col1"] + Map Join Operator [MAPJOIN_16] + Conds:TS_14.reducesinkkey0=TS_14.reducesinkkey0(Inner),Output:["_col0","_col2"] + <-TableScan [TS_14] + Output:["0_VALUE_0","reducesinkkey0"] + Map Reduce Local Work + Stage-5(CONDITIONAL) + Map 5 + keys: [HASHTABLESINK_18] + 0reducesinkkey0,1reducesinkkey0 + TableScan [TS_15] + Output:["1_VALUE_0","reducesinkkey0"] + Map Reduce Local Work + Please refer to the previous Stage-4(CONDITIONAL CHILD TASKS: Stage-5, Stage-0) + +PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) +INSERT OVERWRITE TABLE dest_j1 select src1.key, src2.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) +INSERT OVERWRITE TABLE dest_j1 select src1.key, src2.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest_j1 +POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain +select /*+ STREAMTABLE(a) */ * +FROM T1 a JOIN T2 b ON a.key = b.key + JOIN T3 c ON b.key = c.key + JOIN T4 d ON c.key = d.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +select /*+ STREAMTABLE(a) */ * +FROM T1 a JOIN T2 b ON a.key = b.key + JOIN T3 c ON b.key = c.key + JOIN T4 d ON c.key = d.key +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Map 1 + File Output Operator [FS_18] + Map Join Operator [MAPJOIN_23] (rows=3 width=33) + Conds:SEL_2._col0=SEL_2._col0(Inner),SEL_2._col0=SEL_2._col0(Inner),SEL_2._col0=SEL_2._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] + <-Select Operator [SEL_2] (rows=1 width=30) + Output:["_col0","_col1"] + Filter Operator [FIL_19] (rows=1 width=30) + predicate:key is not null + TableScan [TS_0] (rows=1 width=30) + default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + Map Reduce Local Work + Stage-2 + Map 2 + keys: [HASHTABLESINK_25] + 0_col0,1_col0,2_col0,3_col0 + Select Operator [SEL_5] (rows=1 width=30) + Output:["_col0","_col1"] + Filter Operator [FIL_20] (rows=1 width=30) + predicate:key is not null + TableScan [TS_3] (rows=1 width=30) + default@t2,b,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + Map Reduce Local Work + Map 3 + keys: [HASHTABLESINK_28] + 0_col0,1_col0,2_col0,3_col0 + Select Operator [SEL_8] (rows=1 width=20) + Output:["_col0","_col1"] + Filter Operator [FIL_21] (rows=1 width=20) + predicate:key is not null + TableScan [TS_6] (rows=1 width=20) + default@t3,c,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + Map Reduce Local Work + Map 4 + keys: [HASHTABLESINK_31] + 0_col0,1_col0,2_col0,3_col0 + Select Operator [SEL_11] (rows=1 width=30) + Output:["_col0","_col1"] + Filter Operator [FIL_22] (rows=1 width=30) + predicate:key is not null + TableScan [TS_9] (rows=1 width=30) + default@t4,d,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + Map Reduce Local Work + +PREHOOK: query: explain +select /*+ STREAMTABLE(a,c) */ * +FROM T1 a JOIN T2 b ON a.key = b.key + JOIN T3 c ON b.key = c.key + JOIN T4 d ON c.key = d.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +select /*+ STREAMTABLE(a,c) */ * +FROM T1 a JOIN T2 b ON a.key = b.key + JOIN T3 c ON b.key = c.key + JOIN T4 d ON c.key = d.key +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Map 1 + File Output Operator [FS_18] + Map Join Operator [MAPJOIN_23] (rows=3 width=33) + Conds:SEL_2._col0=SEL_2._col0(Inner),SEL_2._col0=SEL_2._col0(Inner),SEL_2._col0=SEL_2._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] + <-Select Operator [SEL_2] (rows=1 width=30) + Output:["_col0","_col1"] + Filter Operator [FIL_19] (rows=1 width=30) + predicate:key is not null + TableScan [TS_0] (rows=1 width=30) + default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + Map Reduce Local Work + Stage-2 + Map 2 + keys: [HASHTABLESINK_25] + 0_col0,1_col0,2_col0,3_col0 + Select Operator [SEL_5] (rows=1 width=30) + Output:["_col0","_col1"] + Filter Operator [FIL_20] (rows=1 width=30) + predicate:key is not null + TableScan [TS_3] (rows=1 width=30) + default@t2,b,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + Map Reduce Local Work + Map 3 + keys: [HASHTABLESINK_28] + 0_col0,1_col0,2_col0,3_col0 + Select Operator [SEL_8] (rows=1 width=20) + Output:["_col0","_col1"] + Filter Operator [FIL_21] (rows=1 width=20) + predicate:key is not null + TableScan [TS_6] (rows=1 width=20) + default@t3,c,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + Map Reduce Local Work + Map 4 + keys: [HASHTABLESINK_31] + 0_col0,1_col0,2_col0,3_col0 + Select Operator [SEL_11] (rows=1 width=30) + Output:["_col0","_col1"] + Filter Operator [FIL_22] (rows=1 width=30) + predicate:key is not null + TableScan [TS_9] (rows=1 width=30) + default@t4,d,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + Map Reduce Local Work + +PREHOOK: query: explain FROM T1 a JOIN src c ON c.key+1=a.key select /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) +PREHOOK: type: QUERY +POSTHOOK: query: explain FROM T1 a JOIN src c ON c.key+1=a.key select /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 3 <- Map 2 (GROUP) +Reducer 4 <- Reducer 3 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 4 + File Output Operator [FS_16] + Group By Operator [GBY_14] (rows=1 width=24) + Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] + <-Reducer 3 [GROUP] + GROUP [RS_13] + Group By Operator [GBY_12] (rows=1 width=24) + Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] + <-Map 2 [GROUP] + GROUP [RS_11] + PartitionCols:rand() + Select Operator [SEL_9] (rows=550 width=87) + Output:["_col0","_col1","_col2"] + Map Join Operator [MAPJOIN_19] (rows=550 width=87) + Conds:SEL_5.UDFToDouble(_col0)=SEL_5.(UDFToDouble(_col0) + 1.0)(Inner),Output:["_col0","_col1","_col2"] + <-Select Operator [SEL_5] (rows=500 width=87) + Output:["_col0"] + Filter Operator [FIL_18] (rows=500 width=87) + predicate:key is not null + TableScan [TS_3] (rows=500 width=87) + default@src,c,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Map Reduce Local Work + Stage-2 + Map 1 + keys: [HASHTABLESINK_21] + 0UDFToDouble(_col0),1(UDFToDouble(_col0) + 1.0) + Select Operator [SEL_2] (rows=1 width=30) + Output:["_col0","_col1"] + Filter Operator [FIL_17] (rows=1 width=30) + predicate:key is not null + TableScan [TS_0] (rows=1 width=30) + default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + Map Reduce Local Work + +PREHOOK: query: FROM T1 a JOIN src c ON c.key+1=a.key select /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@t1 +#### A masked pattern was here #### +POSTHOOK: query: FROM T1 a JOIN src c ON c.key+1=a.key select /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@t1 +#### A masked pattern was here #### +198 6274 194 +PREHOOK: query: explain +select * FROM +(select src.* FROM src) x +JOIN +(select src.* FROM src) Y +ON (x.key = Y.key) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * FROM +(select src.* FROM src) x +JOIN +(select src.* FROM src) Y +ON (x.key = Y.key) +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT) + +Stage-0 + Fetch Operator + limit:-1 + Stage-2 + Map 4 + File Output Operator [FS_10] + Map Join Operator [MAPJOIN_15] + Conds:TS_13.reducesinkkey0=TS_13.reducesinkkey0(Inner),Output:["_col0","_col1","_col2","_col3"] + <-TableScan [TS_13] + Output:["0_VALUE_0","0_VALUE_1","reducesinkkey0"] + Map Reduce Local Work + Stage-4(CONDITIONAL) + Map 5 + keys: [HASHTABLESINK_17] + 0reducesinkkey0,1reducesinkkey0 + TableScan [TS_14] + Output:["1_VALUE_0","1_VALUE_1","reducesinkkey0"] + Map Reduce Local Work + Stage-3(CONDITIONAL CHILD TASKS: Stage-4) + Conditional Operator + Stage-1 + Reducer 2 + File Output Operator [FS_10] + Join Operator [JOIN_8] (rows=1219 width=356) + Output:["_col0","_col1","_col2","_col3"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_6] + PartitionCols:_col0 + Select Operator [SEL_2] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_11] (rows=500 width=178) + predicate:key is not null + TableScan [TS_0] (rows=500 width=178) + default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Map 3 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_7] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_12] (rows=500 width=178) + predicate:key is not null + TableScan [TS_3] (rows=500 width=178) + default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + +PREHOOK: query: explain select /*+ mapjoin(k)*/ sum(hash(k.key)), sum(hash(v.val)) from T1 k join T1 v on k.key=v.val +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+ mapjoin(k)*/ sum(hash(k.key)), sum(hash(v.val)) from T1 k join T1 v on k.key=v.val +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Reducer 2 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_16] + Group By Operator [GBY_14] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] + <-Reducer 2 [GROUP] + GROUP [RS_13] + Group By Operator [GBY_12] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] + <-Map 1 [GROUP] + GROUP [RS_11] + PartitionCols:rand() + Select Operator [SEL_9] (rows=1 width=33) + Output:["_col0","_col1"] + Map Join Operator [MAPJOIN_19] (rows=1 width=33) + Conds:SEL_2._col0=SEL_2._col0(Inner),Output:["_col0","_col1"] + <-Select Operator [SEL_2] (rows=1 width=30) + Output:["_col0"] + Filter Operator [FIL_17] (rows=1 width=30) + predicate:key is not null + TableScan [TS_0] (rows=1 width=30) + default@t1,k,Tbl:COMPLETE,Col:NONE,Output:["key"] + Map Reduce Local Work + Stage-2 + Map 4 + keys: [HASHTABLESINK_21] + 0_col0,1_col0 + Select Operator [SEL_5] (rows=1 width=30) + Output:["_col0"] + Filter Operator [FIL_18] (rows=1 width=30) + predicate:val is not null + TableScan [TS_3] (rows=1 width=30) + default@t1,v,Tbl:COMPLETE,Col:NONE,Output:["val"] + Map Reduce Local Work + +PREHOOK: query: explain select sum(hash(k.key)), sum(hash(v.val)) from T1 k join T1 v on k.key=v.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select sum(hash(k.key)), sum(hash(v.val)) from T1 k join T1 v on k.key=v.key +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Reducer 2 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_16] + Group By Operator [GBY_14] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] + <-Reducer 2 [GROUP] + GROUP [RS_13] + Group By Operator [GBY_12] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] + <-Map 1 [GROUP] + GROUP [RS_11] + PartitionCols:rand() + Select Operator [SEL_9] (rows=1 width=33) + Output:["_col0","_col1"] + Map Join Operator [MAPJOIN_19] (rows=1 width=33) + Conds:SEL_2._col0=SEL_2._col0(Inner),Output:["_col0","_col2"] + <-Select Operator [SEL_2] (rows=1 width=30) + Output:["_col0"] + Filter Operator [FIL_17] (rows=1 width=30) + predicate:key is not null + TableScan [TS_0] (rows=1 width=30) + default@t1,k,Tbl:COMPLETE,Col:NONE,Output:["key"] + Map Reduce Local Work + Stage-2 + Map 4 + keys: [HASHTABLESINK_21] + 0_col0,1_col0 + Select Operator [SEL_5] (rows=1 width=30) + Output:["_col0","_col1"] + Filter Operator [FIL_18] (rows=1 width=30) + predicate:key is not null + TableScan [TS_3] (rows=1 width=30) + default@t1,v,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + Map Reduce Local Work + +PREHOOK: query: explain select count(1) from T1 a join T1 b on a.key = b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(1) from T1 a join T1 b on a.key = b.key +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Reducer 2 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_16] + Group By Operator [GBY_14] (rows=1 width=8) + Output:["_col0"],aggregations:["count(VALUE._col0)"] + <-Reducer 2 [GROUP] + GROUP [RS_13] + Group By Operator [GBY_12] (rows=1 width=8) + Output:["_col0"],aggregations:["count(1)"] + <-Map 1 [GROUP] + GROUP [RS_11] + PartitionCols:rand() + Map Join Operator [MAPJOIN_19] (rows=1 width=33) + Conds:SEL_2._col0=SEL_2._col0(Inner) + <-Select Operator [SEL_2] (rows=1 width=30) + Output:["_col0"] + Filter Operator [FIL_17] (rows=1 width=30) + predicate:key is not null + TableScan [TS_0] (rows=1 width=30) + default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key"] + Map Reduce Local Work + Stage-2 + Map 4 + keys: [HASHTABLESINK_21] + 0_col0,1_col0 + Select Operator [SEL_5] (rows=1 width=30) + Output:["_col0"] + Filter Operator [FIL_18] (rows=1 width=30) + predicate:key is not null + TableScan [TS_3] (rows=1 width=30) + default@t1,b,Tbl:COMPLETE,Col:NONE,Output:["key"] + Map Reduce Local Work + +PREHOOK: query: explain FROM T1 a LEFT OUTER JOIN T2 c ON c.key+1=a.key select sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) +PREHOOK: type: QUERY +POSTHOOK: query: explain FROM T1 a LEFT OUTER JOIN T2 c ON c.key+1=a.key select sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Reducer 2 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_14] + Group By Operator [GBY_12] (rows=1 width=24) + Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] + <-Reducer 2 [GROUP] + GROUP [RS_11] + Group By Operator [GBY_10] (rows=1 width=24) + Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] + <-Map 1 [GROUP] + GROUP [RS_9] + PartitionCols:rand() + Select Operator [SEL_7] (rows=1 width=33) + Output:["_col0","_col1","_col2"] + Map Join Operator [MAPJOIN_15] (rows=1 width=33) + Conds:SEL_1.UDFToDouble(_col0)=SEL_1.(UDFToDouble(_col0) + 1.0)(Left Outer),Output:["_col0","_col1","_col2"] + <-Select Operator [SEL_1] (rows=1 width=30) + Output:["_col0","_col1"] + TableScan [TS_0] (rows=1 width=30) + default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + Map Reduce Local Work + Stage-2 + Map 4 + keys: [HASHTABLESINK_17] + 0UDFToDouble(_col0),1(UDFToDouble(_col0) + 1.0) + Select Operator [SEL_3] (rows=1 width=30) + Output:["_col0"] + TableScan [TS_2] (rows=1 width=30) + default@t2,c,Tbl:COMPLETE,Col:NONE,Output:["key"] + Map Reduce Local Work + +PREHOOK: query: explain FROM T1 a RIGHT OUTER JOIN T2 c ON c.key+1=a.key select /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) +PREHOOK: type: QUERY +POSTHOOK: query: explain FROM T1 a RIGHT OUTER JOIN T2 c ON c.key+1=a.key select /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 3 <- Map 2 (GROUP) +Reducer 4 <- Reducer 3 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 4 + File Output Operator [FS_14] + Group By Operator [GBY_12] (rows=1 width=24) + Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] + <-Reducer 3 [GROUP] + GROUP [RS_11] + Group By Operator [GBY_10] (rows=1 width=24) + Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] + <-Map 2 [GROUP] + GROUP [RS_9] + PartitionCols:rand() + Select Operator [SEL_7] (rows=1 width=33) + Output:["_col0","_col1","_col2"] + Map Join Operator [MAPJOIN_15] (rows=1 width=33) + Conds:SEL_3.UDFToDouble(_col0)=SEL_3.(UDFToDouble(_col0) + 1.0)(Right Outer),Output:["_col0","_col1","_col2"] + <-Select Operator [SEL_3] (rows=1 width=30) + Output:["_col0"] + TableScan [TS_2] (rows=1 width=30) + default@t2,c,Tbl:COMPLETE,Col:NONE,Output:["key"] + Map Reduce Local Work + Stage-2 + Map 1 + keys: [HASHTABLESINK_17] + 0UDFToDouble(_col0),1(UDFToDouble(_col0) + 1.0) + Select Operator [SEL_1] (rows=1 width=30) + Output:["_col0","_col1"] + TableScan [TS_0] (rows=1 width=30) + default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + Map Reduce Local Work + +PREHOOK: query: explain FROM T1 a FULL OUTER JOIN T2 c ON c.key+1=a.key select /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) +PREHOOK: type: QUERY +POSTHOOK: query: explain FROM T1 a FULL OUTER JOIN T2 c ON c.key+1=a.key select /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 5 (PARTITION-LEVEL SORT) +Reducer 3 <- Reducer 2 (GROUP) +Reducer 4 <- Reducer 3 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 4 + File Output Operator [FS_14] + Group By Operator [GBY_12] (rows=1 width=24) + Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] + <-Reducer 3 [GROUP] + GROUP [RS_11] + Group By Operator [GBY_10] (rows=1 width=24) + Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] + <-Reducer 2 [GROUP] + GROUP [RS_9] + PartitionCols:rand() + Select Operator [SEL_7] (rows=1 width=33) + Output:["_col0","_col1","_col2"] + Join Operator [JOIN_6] (rows=1 width=33) + Output:["_col0","_col1","_col2"],condition map:[{"":"{\"type\":\"Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"UDFToDouble(_col0)","1":"(UDFToDouble(_col0) + 1.0)"} + <-Map 1 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_4] + PartitionCols:UDFToDouble(_col0) + Select Operator [SEL_1] (rows=1 width=30) + Output:["_col0","_col1"] + TableScan [TS_0] (rows=1 width=30) + default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + <-Map 5 [PARTITION-LEVEL SORT] + PARTITION-LEVEL SORT [RS_5] + PartitionCols:(UDFToDouble(_col0) + 1.0) + Select Operator [SEL_3] (rows=1 width=30) + Output:["_col0"] + TableScan [TS_2] (rows=1 width=30) + default@t2,c,Tbl:COMPLETE,Col:NONE,Output:["key"] + +PREHOOK: query: explain select /*+ mapjoin(v)*/ sum(hash(k.key)), sum(hash(v.val)) from T1 k left outer join T1 v on k.key+1=v.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select /*+ mapjoin(v)*/ sum(hash(k.key)), sum(hash(v.val)) from T1 k left outer join T1 v on k.key+1=v.key +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (GROUP) +Reducer 3 <- Reducer 2 (GROUP) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 + File Output Operator [FS_14] + Group By Operator [GBY_12] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] + <-Reducer 2 [GROUP] + GROUP [RS_11] + Group By Operator [GBY_10] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] + <-Map 1 [GROUP] + GROUP [RS_9] + PartitionCols:rand() + Select Operator [SEL_7] (rows=1 width=33) + Output:["_col0","_col1"] + Map Join Operator [MAPJOIN_15] (rows=1 width=33) + Conds:SEL_1.(UDFToDouble(_col0) + 1.0)=SEL_1.UDFToDouble(_col0)(Left Outer),Output:["_col0","_col2"] + <-Select Operator [SEL_1] (rows=1 width=30) + Output:["_col0"] + TableScan [TS_0] (rows=1 width=30) + default@t1,k,Tbl:COMPLETE,Col:NONE,Output:["key"] + Map Reduce Local Work + Stage-2 + Map 4 + keys: [HASHTABLESINK_17] + 0(UDFToDouble(_col0) + 1.0),1UDFToDouble(_col0) + Select Operator [SEL_3] (rows=1 width=30) + Output:["_col0","_col1"] + TableScan [TS_2] (rows=1 width=30) + default@t1,v,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + Map Reduce Local Work +